Repository: Netflix/conductor Branch: main Commit: 548f386c2c6c Files: 1030 Total size: 5.8 MB Directory structure: gitextract_cp2gulur/ ├── .dockerignore ├── .gitattributes ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── config.yml │ │ ├── documentation.md │ │ └── feature_request.md │ ├── dependabot.yml │ ├── pull_request_template.md │ ├── release-drafter.yml │ └── workflows/ │ ├── ci.yml │ ├── publish.yml │ ├── release_draft.yml │ ├── stale.yml │ └── update-gradle-wrapper.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── OSSMETADATA ├── README.md ├── RELATED.md ├── SECURITY.md ├── USERS.md ├── annotations/ │ ├── README.md │ ├── build.gradle │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── annotations/ │ └── protogen/ │ ├── ProtoEnum.java │ ├── ProtoField.java │ └── ProtoMessage.java ├── annotations-processor/ │ ├── README.md │ ├── build.gradle │ └── src/ │ ├── example/ │ │ └── java/ │ │ └── com/ │ │ └── example/ │ │ └── Example.java │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── annotationsprocessor/ │ │ │ └── protogen/ │ │ │ ├── AbstractMessage.java │ │ │ ├── Enum.java │ │ │ ├── Message.java │ │ │ ├── ProtoFile.java │ │ │ ├── ProtoGen.java │ │ │ ├── ProtoGenTask.java │ │ │ └── types/ │ │ │ ├── AbstractType.java │ │ │ ├── ExternMessageType.java │ │ │ ├── GenericType.java │ │ │ ├── ListType.java │ │ │ ├── MapType.java │ │ │ ├── MessageType.java │ │ │ ├── ScalarType.java │ │ │ ├── TypeMapper.java │ │ │ └── WrappedType.java │ │ └── resources/ │ │ └── templates/ │ │ ├── file.proto │ │ └── message.proto │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── annotationsprocessor/ │ │ └── protogen/ │ │ └── ProtoGenTest.java │ └── resources/ │ └── example.proto.txt ├── awss3-storage/ │ ├── README.md │ ├── build.gradle │ └── src/ │ └── main/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── s3/ │ │ ├── config/ │ │ │ ├── S3Configuration.java │ │ │ └── S3Properties.java │ │ └── storage/ │ │ └── S3PayloadStorage.java │ └── resources/ │ └── META-INF/ │ └── additional-spring-configuration-metadata.json ├── awssqs-event-queue/ │ ├── README.md │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── sqs/ │ │ │ ├── config/ │ │ │ │ ├── SQSEventQueueConfiguration.java │ │ │ │ ├── SQSEventQueueProperties.java │ │ │ │ └── SQSEventQueueProvider.java │ │ │ └── eventqueue/ │ │ │ └── SQSObservableQueue.java │ │ └── resources/ │ │ └── META-INF/ │ │ └── additional-spring-configuration-metadata.json │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── sqs/ │ └── eventqueue/ │ ├── DefaultEventQueueProcessorTest.java │ └── SQSObservableQueueTest.java ├── build.gradle ├── cassandra-persistence/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── cassandra/ │ │ │ ├── config/ │ │ │ │ ├── CassandraConfiguration.java │ │ │ │ ├── CassandraProperties.java │ │ │ │ └── cache/ │ │ │ │ ├── CacheableEventHandlerDAO.java │ │ │ │ ├── CacheableMetadataDAO.java │ │ │ │ └── CachingConfig.java │ │ │ ├── dao/ │ │ │ │ ├── CassandraBaseDAO.java │ │ │ │ ├── CassandraEventHandlerDAO.java │ │ │ │ ├── CassandraExecutionDAO.java │ │ │ │ ├── CassandraMetadataDAO.java │ │ │ │ └── CassandraPollDataDAO.java │ │ │ └── util/ │ │ │ ├── Constants.java │ │ │ └── Statements.java │ │ └── resources/ │ │ └── META-INF/ │ │ └── additional-spring-configuration-metadata.json │ └── test/ │ └── groovy/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── cassandra/ │ ├── dao/ │ │ ├── CassandraEventHandlerDAOSpec.groovy │ │ ├── CassandraExecutionDAOSpec.groovy │ │ ├── CassandraMetadataDAOSpec.groovy │ │ └── CassandraSpec.groovy │ └── util/ │ └── StatementsSpec.groovy ├── client/ │ ├── build.gradle │ ├── spotbugsExclude.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ ├── automator/ │ │ │ ├── PollingSemaphore.java │ │ │ ├── TaskPollExecutor.java │ │ │ └── TaskRunnerConfigurer.java │ │ ├── config/ │ │ │ ├── ConductorClientConfiguration.java │ │ │ ├── DefaultConductorClientConfiguration.java │ │ │ └── PropertyFactory.java │ │ ├── exception/ │ │ │ └── ConductorClientException.java │ │ ├── http/ │ │ │ ├── ClientBase.java │ │ │ ├── ClientRequestHandler.java │ │ │ ├── EventClient.java │ │ │ ├── MetadataClient.java │ │ │ ├── PayloadStorage.java │ │ │ ├── TaskClient.java │ │ │ └── WorkflowClient.java │ │ ├── telemetry/ │ │ │ └── MetricsContainer.java │ │ └── worker/ │ │ └── Worker.java │ └── test/ │ ├── groovy/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ └── http/ │ │ ├── ClientSpecification.groovy │ │ ├── EventClientSpec.groovy │ │ ├── MetadataClientSpec.groovy │ │ ├── TaskClientSpec.groovy │ │ └── WorkflowClientSpec.groovy │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ ├── automator/ │ │ │ ├── PollingSemaphoreTest.java │ │ │ ├── TaskPollExecutorTest.java │ │ │ └── TaskRunnerConfigurerTest.java │ │ ├── config/ │ │ │ └── TestPropertyFactory.java │ │ ├── sample/ │ │ │ ├── Main.java │ │ │ └── SampleWorker.java │ │ ├── testing/ │ │ │ ├── AbstractWorkflowTests.java │ │ │ ├── LoanWorkflowInput.java │ │ │ ├── LoanWorkflowTest.java │ │ │ ├── RegressionTest.java │ │ │ └── SubWorkflowTest.java │ │ └── worker/ │ │ └── TestWorkflowTask.java │ └── resources/ │ ├── config.properties │ ├── tasks.json │ ├── test_data/ │ │ ├── loan_workflow_input.json │ │ └── workflow1_run.json │ └── workflows/ │ ├── PopulationMinMax.json │ ├── calculate_loan_workflow.json │ ├── kitchensink.json │ └── workflow1.json ├── client-spring/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── client/ │ │ │ └── spring/ │ │ │ ├── ClientProperties.java │ │ │ ├── ConductorClientAutoConfiguration.java │ │ │ ├── ConductorWorkerAutoConfiguration.java │ │ │ └── SpringWorkerConfiguration.java │ │ └── resources/ │ │ └── META-INF/ │ │ └── spring.factories │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ └── spring/ │ │ ├── ExampleClient.java │ │ └── Workers.java │ └── resources/ │ └── application.properties ├── common/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── common/ │ │ ├── config/ │ │ │ ├── ObjectMapperBuilderConfiguration.java │ │ │ ├── ObjectMapperConfiguration.java │ │ │ └── ObjectMapperProvider.java │ │ ├── constraints/ │ │ │ ├── NoSemiColonConstraint.java │ │ │ ├── OwnerEmailMandatoryConstraint.java │ │ │ ├── TaskReferenceNameUniqueConstraint.java │ │ │ └── TaskTimeoutConstraint.java │ │ ├── jackson/ │ │ │ └── JsonProtoModule.java │ │ ├── metadata/ │ │ │ ├── Auditable.java │ │ │ ├── BaseDef.java │ │ │ ├── acl/ │ │ │ │ └── Permission.java │ │ │ ├── events/ │ │ │ │ ├── EventExecution.java │ │ │ │ └── EventHandler.java │ │ │ ├── tasks/ │ │ │ │ ├── PollData.java │ │ │ │ ├── Task.java │ │ │ │ ├── TaskDef.java │ │ │ │ ├── TaskExecLog.java │ │ │ │ ├── TaskResult.java │ │ │ │ └── TaskType.java │ │ │ └── workflow/ │ │ │ ├── DynamicForkJoinTask.java │ │ │ ├── DynamicForkJoinTaskList.java │ │ │ ├── RerunWorkflowRequest.java │ │ │ ├── SkipTaskRequest.java │ │ │ ├── StartWorkflowRequest.java │ │ │ ├── SubWorkflowParams.java │ │ │ ├── WorkflowDef.java │ │ │ ├── WorkflowDefSummary.java │ │ │ └── WorkflowTask.java │ │ ├── model/ │ │ │ └── BulkResponse.java │ │ ├── run/ │ │ │ ├── ExternalStorageLocation.java │ │ │ ├── SearchResult.java │ │ │ ├── TaskSummary.java │ │ │ ├── Workflow.java │ │ │ ├── WorkflowSummary.java │ │ │ └── WorkflowTestRequest.java │ │ ├── utils/ │ │ │ ├── ConstraintParamUtil.java │ │ │ ├── EnvUtils.java │ │ │ ├── ExternalPayloadStorage.java │ │ │ ├── SummaryUtil.java │ │ │ └── TaskUtils.java │ │ └── validation/ │ │ ├── ErrorResponse.java │ │ └── ValidationError.java │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── common/ │ ├── config/ │ │ └── TestObjectMapperConfiguration.java │ ├── events/ │ │ └── EventHandlerTest.java │ ├── run/ │ │ └── TaskSummaryTest.java │ ├── tasks/ │ │ ├── TaskDefTest.java │ │ ├── TaskResultTest.java │ │ └── TaskTest.java │ ├── utils/ │ │ ├── ConstraintParamUtilTest.java │ │ └── SummaryUtilTest.java │ └── workflow/ │ ├── SubWorkflowParamsTest.java │ ├── WorkflowDefValidatorTest.java │ └── WorkflowTaskTest.java ├── core/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ ├── annotations/ │ │ │ │ ├── Audit.java │ │ │ │ ├── Trace.java │ │ │ │ └── VisibleForTesting.java │ │ │ ├── core/ │ │ │ │ ├── LifecycleAwareComponent.java │ │ │ │ ├── WorkflowContext.java │ │ │ │ ├── config/ │ │ │ │ │ ├── ConductorCoreConfiguration.java │ │ │ │ │ ├── ConductorProperties.java │ │ │ │ │ └── SchedulerConfiguration.java │ │ │ │ ├── dal/ │ │ │ │ │ └── ExecutionDAOFacade.java │ │ │ │ ├── event/ │ │ │ │ │ ├── WorkflowCreationEvent.java │ │ │ │ │ └── WorkflowEvaluationEvent.java │ │ │ │ ├── events/ │ │ │ │ │ ├── ActionProcessor.java │ │ │ │ │ ├── DefaultEventProcessor.java │ │ │ │ │ ├── DefaultEventQueueManager.java │ │ │ │ │ ├── EventQueueManager.java │ │ │ │ │ ├── EventQueueProvider.java │ │ │ │ │ ├── EventQueues.java │ │ │ │ │ ├── ScriptEvaluator.java │ │ │ │ │ ├── SimpleActionProcessor.java │ │ │ │ │ └── queue/ │ │ │ │ │ ├── ConductorEventQueueProvider.java │ │ │ │ │ ├── ConductorObservableQueue.java │ │ │ │ │ ├── DefaultEventQueueProcessor.java │ │ │ │ │ ├── Message.java │ │ │ │ │ └── ObservableQueue.java │ │ │ │ ├── exception/ │ │ │ │ │ ├── ConflictException.java │ │ │ │ │ ├── NonTransientException.java │ │ │ │ │ ├── NotFoundException.java │ │ │ │ │ ├── TerminateWorkflowException.java │ │ │ │ │ └── TransientException.java │ │ │ │ ├── execution/ │ │ │ │ │ ├── AsyncSystemTaskExecutor.java │ │ │ │ │ ├── DeciderService.java │ │ │ │ │ ├── StartWorkflowInput.java │ │ │ │ │ ├── WorkflowExecutor.java │ │ │ │ │ ├── evaluators/ │ │ │ │ │ │ ├── Evaluator.java │ │ │ │ │ │ ├── JavascriptEvaluator.java │ │ │ │ │ │ └── ValueParamEvaluator.java │ │ │ │ │ ├── mapper/ │ │ │ │ │ │ ├── DecisionTaskMapper.java │ │ │ │ │ │ ├── DoWhileTaskMapper.java │ │ │ │ │ │ ├── DynamicTaskMapper.java │ │ │ │ │ │ ├── EventTaskMapper.java │ │ │ │ │ │ ├── ExclusiveJoinTaskMapper.java │ │ │ │ │ │ ├── ForkJoinDynamicTaskMapper.java │ │ │ │ │ │ ├── ForkJoinTaskMapper.java │ │ │ │ │ │ ├── HTTPTaskMapper.java │ │ │ │ │ │ ├── HumanTaskMapper.java │ │ │ │ │ │ ├── InlineTaskMapper.java │ │ │ │ │ │ ├── JoinTaskMapper.java │ │ │ │ │ │ ├── JsonJQTransformTaskMapper.java │ │ │ │ │ │ ├── KafkaPublishTaskMapper.java │ │ │ │ │ │ ├── LambdaTaskMapper.java │ │ │ │ │ │ ├── NoopTaskMapper.java │ │ │ │ │ │ ├── SetVariableTaskMapper.java │ │ │ │ │ │ ├── SimpleTaskMapper.java │ │ │ │ │ │ ├── StartWorkflowTaskMapper.java │ │ │ │ │ │ ├── SubWorkflowTaskMapper.java │ │ │ │ │ │ ├── SwitchTaskMapper.java │ │ │ │ │ │ ├── TaskMapper.java │ │ │ │ │ │ ├── TaskMapperContext.java │ │ │ │ │ │ ├── TerminateTaskMapper.java │ │ │ │ │ │ ├── UserDefinedTaskMapper.java │ │ │ │ │ │ └── WaitTaskMapper.java │ │ │ │ │ └── tasks/ │ │ │ │ │ ├── Decision.java │ │ │ │ │ ├── DoWhile.java │ │ │ │ │ ├── Event.java │ │ │ │ │ ├── ExclusiveJoin.java │ │ │ │ │ ├── ExecutionConfig.java │ │ │ │ │ ├── Fork.java │ │ │ │ │ ├── Human.java │ │ │ │ │ ├── Inline.java │ │ │ │ │ ├── IsolatedTaskQueueProducer.java │ │ │ │ │ ├── Join.java │ │ │ │ │ ├── Lambda.java │ │ │ │ │ ├── Noop.java │ │ │ │ │ ├── SetVariable.java │ │ │ │ │ ├── StartWorkflow.java │ │ │ │ │ ├── SubWorkflow.java │ │ │ │ │ ├── Switch.java │ │ │ │ │ ├── SystemTaskRegistry.java │ │ │ │ │ ├── SystemTaskWorker.java │ │ │ │ │ ├── SystemTaskWorkerCoordinator.java │ │ │ │ │ ├── Terminate.java │ │ │ │ │ ├── Wait.java │ │ │ │ │ └── WorkflowSystemTask.java │ │ │ │ ├── index/ │ │ │ │ │ ├── NoopIndexDAO.java │ │ │ │ │ └── NoopIndexDAOConfiguration.java │ │ │ │ ├── listener/ │ │ │ │ │ ├── TaskStatusListener.java │ │ │ │ │ ├── TaskStatusListenerStub.java │ │ │ │ │ ├── WorkflowStatusListener.java │ │ │ │ │ └── WorkflowStatusListenerStub.java │ │ │ │ ├── metadata/ │ │ │ │ │ └── MetadataMapperService.java │ │ │ │ ├── operation/ │ │ │ │ │ ├── StartWorkflowOperation.java │ │ │ │ │ └── WorkflowOperation.java │ │ │ │ ├── reconciliation/ │ │ │ │ │ ├── WorkflowReconciler.java │ │ │ │ │ ├── WorkflowRepairService.java │ │ │ │ │ └── WorkflowSweeper.java │ │ │ │ ├── storage/ │ │ │ │ │ └── DummyPayloadStorage.java │ │ │ │ ├── sync/ │ │ │ │ │ ├── Lock.java │ │ │ │ │ ├── local/ │ │ │ │ │ │ ├── LocalOnlyLock.java │ │ │ │ │ │ └── LocalOnlyLockConfiguration.java │ │ │ │ │ └── noop/ │ │ │ │ │ └── NoopLock.java │ │ │ │ └── utils/ │ │ │ │ ├── DateTimeUtils.java │ │ │ │ ├── ExternalPayloadStorageUtils.java │ │ │ │ ├── IDGenerator.java │ │ │ │ ├── JsonUtils.java │ │ │ │ ├── ParametersUtils.java │ │ │ │ ├── QueueUtils.java │ │ │ │ ├── SemaphoreUtil.java │ │ │ │ └── Utils.java │ │ │ ├── dao/ │ │ │ │ ├── ConcurrentExecutionLimitDAO.java │ │ │ │ ├── EventHandlerDAO.java │ │ │ │ ├── ExecutionDAO.java │ │ │ │ ├── IndexDAO.java │ │ │ │ ├── MetadataDAO.java │ │ │ │ ├── PollDataDAO.java │ │ │ │ ├── QueueDAO.java │ │ │ │ └── RateLimitingDAO.java │ │ │ ├── metrics/ │ │ │ │ ├── Monitors.java │ │ │ │ └── WorkflowMonitor.java │ │ │ ├── model/ │ │ │ │ ├── TaskModel.java │ │ │ │ └── WorkflowModel.java │ │ │ ├── service/ │ │ │ │ ├── AdminService.java │ │ │ │ ├── AdminServiceImpl.java │ │ │ │ ├── EventService.java │ │ │ │ ├── EventServiceImpl.java │ │ │ │ ├── ExecutionLockService.java │ │ │ │ ├── ExecutionService.java │ │ │ │ ├── MetadataService.java │ │ │ │ ├── MetadataServiceImpl.java │ │ │ │ ├── TaskService.java │ │ │ │ ├── TaskServiceImpl.java │ │ │ │ ├── WorkflowBulkService.java │ │ │ │ ├── WorkflowBulkServiceImpl.java │ │ │ │ ├── WorkflowService.java │ │ │ │ ├── WorkflowServiceImpl.java │ │ │ │ └── WorkflowTestService.java │ │ │ └── validations/ │ │ │ ├── ValidationContext.java │ │ │ └── WorkflowTaskTypeConstraint.java │ │ └── resources/ │ │ └── META-INF/ │ │ ├── additional-spring-configuration-metadata.json │ │ ├── validation/ │ │ │ └── constraints.xml │ │ └── validation.xml │ └── test/ │ ├── groovy/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ ├── core/ │ │ │ ├── execution/ │ │ │ │ ├── AsyncSystemTaskExecutorTest.groovy │ │ │ │ └── tasks/ │ │ │ │ ├── DoWhileSpec.groovy │ │ │ │ ├── EventSpec.groovy │ │ │ │ ├── IsolatedTaskQueueProducerSpec.groovy │ │ │ │ └── StartWorkflowSpec.groovy │ │ │ └── operation/ │ │ │ └── StartWorkflowOperationSpec.groovy │ │ └── model/ │ │ ├── TaskModelSpec.groovy │ │ └── WorkflowModelSpec.groovy │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ ├── TestUtils.java │ │ ├── core/ │ │ │ ├── dal/ │ │ │ │ └── ExecutionDAOFacadeTest.java │ │ │ ├── events/ │ │ │ │ ├── MockObservableQueue.java │ │ │ │ ├── MockQueueProvider.java │ │ │ │ ├── TestDefaultEventProcessor.java │ │ │ │ ├── TestScriptEval.java │ │ │ │ └── TestSimpleActionProcessor.java │ │ │ ├── execution/ │ │ │ │ ├── TestDeciderOutcomes.java │ │ │ │ ├── TestDeciderService.java │ │ │ │ ├── TestWorkflowDef.java │ │ │ │ ├── TestWorkflowExecutor.java │ │ │ │ ├── WorkflowSystemTaskStub.java │ │ │ │ ├── mapper/ │ │ │ │ │ ├── DecisionTaskMapperTest.java │ │ │ │ │ ├── DoWhileTaskMapperTest.java │ │ │ │ │ ├── DynamicTaskMapperTest.java │ │ │ │ │ ├── EventTaskMapperTest.java │ │ │ │ │ ├── ForkJoinDynamicTaskMapperTest.java │ │ │ │ │ ├── ForkJoinTaskMapperTest.java │ │ │ │ │ ├── HTTPTaskMapperTest.java │ │ │ │ │ ├── HumanTaskMapperTest.java │ │ │ │ │ ├── InlineTaskMapperTest.java │ │ │ │ │ ├── JoinTaskMapperTest.java │ │ │ │ │ ├── JsonJQTransformTaskMapperTest.java │ │ │ │ │ ├── KafkaPublishTaskMapperTest.java │ │ │ │ │ ├── LambdaTaskMapperTest.java │ │ │ │ │ ├── NoopTaskMapperTest.java │ │ │ │ │ ├── SetVariableTaskMapperTest.java │ │ │ │ │ ├── SimpleTaskMapperTest.java │ │ │ │ │ ├── SubWorkflowTaskMapperTest.java │ │ │ │ │ ├── SwitchTaskMapperTest.java │ │ │ │ │ ├── TerminateTaskMapperTest.java │ │ │ │ │ ├── UserDefinedTaskMapperTest.java │ │ │ │ │ └── WaitTaskMapperTest.java │ │ │ │ └── tasks/ │ │ │ │ ├── EventQueueResolutionTest.java │ │ │ │ ├── InlineTest.java │ │ │ │ ├── TestLambda.java │ │ │ │ ├── TestNoop.java │ │ │ │ ├── TestSubWorkflow.java │ │ │ │ ├── TestSystemTaskWorker.java │ │ │ │ ├── TestSystemTaskWorkerCoordinator.java │ │ │ │ └── TestTerminate.java │ │ │ ├── metadata/ │ │ │ │ └── MetadataMapperServiceTest.java │ │ │ ├── reconciliation/ │ │ │ │ ├── TestWorkflowRepairService.java │ │ │ │ └── TestWorkflowSweeper.java │ │ │ ├── storage/ │ │ │ │ └── DummyPayloadStorageTest.java │ │ │ ├── sync/ │ │ │ │ └── local/ │ │ │ │ └── LocalOnlyLockTest.java │ │ │ └── utils/ │ │ │ ├── ExternalPayloadStorageUtilsTest.java │ │ │ ├── JsonUtilsTest.java │ │ │ ├── ParametersUtilsTest.java │ │ │ ├── QueueUtilsTest.java │ │ │ └── SemaphoreUtilTest.java │ │ ├── dao/ │ │ │ ├── ExecutionDAOTest.java │ │ │ └── PollDataDAOTest.java │ │ ├── metrics/ │ │ │ └── WorkflowMonitorTest.java │ │ ├── service/ │ │ │ ├── EventServiceTest.java │ │ │ ├── ExecutionServiceTest.java │ │ │ ├── MetadataServiceTest.java │ │ │ ├── TaskServiceTest.java │ │ │ ├── WorkflowBulkServiceTest.java │ │ │ └── WorkflowServiceTest.java │ │ └── validations/ │ │ ├── WorkflowDefConstraintTest.java │ │ └── WorkflowTaskTypeConstraintTest.java │ └── resources/ │ ├── completed.json │ ├── conditional_flow.json │ ├── conditional_flow_with_switch.json │ ├── payload.json │ └── test.json ├── dependencies.gradle ├── docker/ │ ├── README.md │ ├── ci/ │ │ └── Dockerfile │ ├── docker-compose-mysql.yaml │ ├── docker-compose-postgres.yaml │ ├── docker-compose.yaml │ ├── server/ │ │ ├── Dockerfile │ │ ├── config/ │ │ │ ├── config-mysql.properties │ │ │ ├── config-postgres.properties │ │ │ ├── config-redis.properties │ │ │ ├── config.properties │ │ │ ├── log4j-file-appender.properties │ │ │ ├── log4j.properties │ │ │ └── redis.conf │ │ └── nginx/ │ │ └── nginx.conf │ └── ui/ │ ├── Dockerfile │ └── README.md ├── docs/ │ ├── docs/ │ │ ├── apispec.md │ │ ├── architecture/ │ │ │ ├── overview.md │ │ │ └── tasklifecycle.md │ │ ├── bestpractices.md │ │ ├── configuration/ │ │ │ ├── eventhandlers.md │ │ │ ├── isolationgroups.md │ │ │ ├── sysoperator.md │ │ │ ├── systask.md │ │ │ ├── taskdef.md │ │ │ ├── taskdomains.md │ │ │ ├── workerdef.md │ │ │ └── workflowdef.md │ │ ├── css/ │ │ │ └── custom.css │ │ ├── extend.md │ │ ├── externalpayloadstorage.md │ │ ├── faq.md │ │ ├── gettingstarted/ │ │ │ ├── basicconcepts.md │ │ │ ├── client.md │ │ │ ├── docker.md │ │ │ ├── hosted.md │ │ │ ├── intro.md │ │ │ ├── source.md │ │ │ ├── startworkflow.md │ │ │ └── steps.md │ │ ├── googleba55068fa3e0e553.html │ │ ├── how-tos/ │ │ │ ├── Monitoring/ │ │ │ │ └── Conductor-LogLevel.md │ │ │ ├── Tasks/ │ │ │ │ ├── creating-tasks.md │ │ │ │ ├── dynamic-vs-switch-tasks.md │ │ │ │ ├── extending-system-tasks.md │ │ │ │ ├── monitoring-task-queues.md │ │ │ │ ├── reusing-tasks.md │ │ │ │ ├── task-configurations.md │ │ │ │ ├── task-inputs.md │ │ │ │ ├── task-timeouts.md │ │ │ │ └── updating-tasks.md │ │ │ ├── Test/ │ │ │ │ └── testing-workflows.md │ │ │ ├── Workers/ │ │ │ │ ├── build-a-golang-task-worker.md │ │ │ │ ├── build-a-java-task-worker.md │ │ │ │ └── build-a-python-task-worker.md │ │ │ ├── Workflows/ │ │ │ │ ├── debugging-workflows.md │ │ │ │ ├── handling-errors.md │ │ │ │ ├── searching-workflows.md │ │ │ │ ├── starting-workflows.md │ │ │ │ ├── updating-workflows.md │ │ │ │ ├── versioning-workflows.md │ │ │ │ └── view-workflow-executions.md │ │ │ ├── clojure-sdk.md │ │ │ ├── csharp-sdk.md │ │ │ ├── go-sdk.md │ │ │ ├── java-sdk.md │ │ │ └── python-sdk.md │ │ ├── index.md │ │ ├── labs/ │ │ │ ├── beginner.md │ │ │ ├── eventhandlers.md │ │ │ ├── kitchensink.md │ │ │ └── running-first-workflow.md │ │ ├── metrics/ │ │ │ ├── client.md │ │ │ └── server.md │ │ ├── reference-docs/ │ │ │ ├── annotation-processor.md │ │ │ ├── archival-of-workflows.md │ │ │ ├── azureblob-storage.md │ │ │ ├── directed-acyclic-graph.md │ │ │ ├── do-while-task.md │ │ │ ├── dynamic-fork-task.md │ │ │ ├── dynamic-task.md │ │ │ ├── event-task.md │ │ │ ├── fork-task.md │ │ │ ├── http-task.md │ │ │ ├── human-task.md │ │ │ ├── inline-task.md │ │ │ ├── join-task.md │ │ │ ├── json-jq-transform-task.md │ │ │ ├── kafka-publish-task.md │ │ │ ├── redis.md │ │ │ ├── set-variable-task.md │ │ │ ├── start-workflow-task.md │ │ │ ├── sub-workflow-task.md │ │ │ ├── switch-task.md │ │ │ ├── terminate-task.md │ │ │ └── wait-task.md │ │ ├── resources/ │ │ │ ├── code-of-conduct.md │ │ │ ├── contributing.md │ │ │ ├── license.md │ │ │ └── related.md │ │ └── technicaldetails.md │ ├── kitchensink.json │ ├── mkdocs.yml │ └── theme/ │ ├── main.html │ ├── toc-sub.html │ └── toc.html ├── es6-persistence/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── es6/ │ │ │ ├── config/ │ │ │ │ ├── ElasticSearchConditions.java │ │ │ │ ├── ElasticSearchProperties.java │ │ │ │ ├── ElasticSearchV6Configuration.java │ │ │ │ ├── IsHttpProtocol.java │ │ │ │ └── IsTcpProtocol.java │ │ │ └── dao/ │ │ │ ├── index/ │ │ │ │ ├── BulkRequestBuilderWrapper.java │ │ │ │ ├── BulkRequestWrapper.java │ │ │ │ ├── ElasticSearchBaseDAO.java │ │ │ │ ├── ElasticSearchDAOV6.java │ │ │ │ └── ElasticSearchRestDAOV6.java │ │ │ └── query/ │ │ │ └── parser/ │ │ │ ├── Expression.java │ │ │ ├── FilterProvider.java │ │ │ ├── GroupedExpression.java │ │ │ ├── NameValue.java │ │ │ └── internal/ │ │ │ ├── AbstractNode.java │ │ │ ├── BooleanOp.java │ │ │ ├── ComparisonOp.java │ │ │ ├── ConstValue.java │ │ │ ├── FunctionThrowingException.java │ │ │ ├── ListConst.java │ │ │ ├── Name.java │ │ │ ├── ParserException.java │ │ │ └── Range.java │ │ └── resources/ │ │ ├── mappings_docType_task.json │ │ ├── mappings_docType_workflow.json │ │ ├── template_event.json │ │ ├── template_message.json │ │ └── template_task_log.json │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── es6/ │ │ ├── dao/ │ │ │ ├── index/ │ │ │ │ ├── ElasticSearchDaoBaseTest.java │ │ │ │ ├── ElasticSearchRestDaoBaseTest.java │ │ │ │ ├── ElasticSearchTest.java │ │ │ │ ├── TestElasticSearchDAOV6.java │ │ │ │ ├── TestElasticSearchDAOV6Batch.java │ │ │ │ ├── TestElasticSearchRestDAOV6.java │ │ │ │ └── TestElasticSearchRestDAOV6Batch.java │ │ │ └── query/ │ │ │ └── parser/ │ │ │ ├── TestExpression.java │ │ │ └── internal/ │ │ │ ├── TestAbstractParser.java │ │ │ ├── TestBooleanOp.java │ │ │ ├── TestComparisonOp.java │ │ │ ├── TestConstValue.java │ │ │ └── TestName.java │ │ └── utils/ │ │ └── TestUtils.java │ └── resources/ │ ├── expected_template_task_log.json │ ├── task_summary.json │ └── workflow_summary.json ├── family.properties ├── gradle/ │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── grpc/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── grpc/ │ │ │ ├── AbstractProtoMapper.java │ │ │ └── ProtoMapper.java │ │ └── proto/ │ │ ├── grpc/ │ │ │ ├── event_service.proto │ │ │ ├── metadata_service.proto │ │ │ ├── search.proto │ │ │ ├── task_service.proto │ │ │ └── workflow_service.proto │ │ └── model/ │ │ ├── dynamicforkjointask.proto │ │ ├── dynamicforkjointasklist.proto │ │ ├── eventexecution.proto │ │ ├── eventhandler.proto │ │ ├── polldata.proto │ │ ├── rerunworkflowrequest.proto │ │ ├── skiptaskrequest.proto │ │ ├── startworkflowrequest.proto │ │ ├── subworkflowparams.proto │ │ ├── task.proto │ │ ├── taskdef.proto │ │ ├── taskexeclog.proto │ │ ├── taskresult.proto │ │ ├── tasksummary.proto │ │ ├── workflow.proto │ │ ├── workflowdef.proto │ │ ├── workflowdefsummary.proto │ │ ├── workflowsummary.proto │ │ └── workflowtask.proto │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── grpc/ │ └── TestProtoMapper.java ├── grpc-client/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ └── grpc/ │ │ ├── ClientBase.java │ │ ├── EventClient.java │ │ ├── MetadataClient.java │ │ ├── TaskClient.java │ │ └── WorkflowClient.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── client/ │ │ └── grpc/ │ │ ├── EventClientTest.java │ │ ├── TaskClientTest.java │ │ └── WorkflowClientTest.java │ └── resources/ │ └── mockito-extensions/ │ └── org.mockito.plugins.MockMaker ├── grpc-server/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── grpc/ │ │ └── server/ │ │ ├── GRPCServer.java │ │ ├── GRPCServerProperties.java │ │ ├── GrpcConfiguration.java │ │ └── service/ │ │ ├── EventServiceImpl.java │ │ ├── GRPCHelper.java │ │ ├── HealthServiceImpl.java │ │ ├── MetadataServiceImpl.java │ │ ├── TaskServiceImpl.java │ │ └── WorkflowServiceImpl.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── grpc/ │ │ └── server/ │ │ └── service/ │ │ ├── HealthServiceImplTest.java │ │ ├── TaskServiceImplTest.java │ │ └── WorkflowServiceImplTest.java │ └── resources/ │ └── log4j.properties ├── http-task/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── tasks/ │ │ │ └── http/ │ │ │ ├── HttpTask.java │ │ │ └── providers/ │ │ │ ├── DefaultRestTemplateProvider.java │ │ │ └── RestTemplateProvider.java │ │ └── resources/ │ │ └── META-INF/ │ │ └── additional-spring-configuration-metadata.json │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── tasks/ │ └── http/ │ ├── HttpTaskTest.java │ └── providers/ │ └── DefaultRestTemplateProviderTest.java ├── java-sdk/ │ ├── README.md │ ├── build.gradle │ ├── example/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── sdk/ │ │ │ └── example/ │ │ │ └── shipment/ │ │ │ ├── Order.java │ │ │ ├── Shipment.java │ │ │ ├── ShipmentState.java │ │ │ ├── ShipmentWorkers.java │ │ │ ├── ShipmentWorkflow.java │ │ │ └── User.java │ │ └── resources/ │ │ └── script.js │ ├── src/ │ │ ├── main/ │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── netflix/ │ │ │ │ └── conductor/ │ │ │ │ └── sdk/ │ │ │ │ ├── healthcheck/ │ │ │ │ │ └── HealthCheckClient.java │ │ │ │ ├── testing/ │ │ │ │ │ ├── LocalServerRunner.java │ │ │ │ │ └── WorkflowTestRunner.java │ │ │ │ └── workflow/ │ │ │ │ ├── def/ │ │ │ │ │ ├── ConductorWorkflow.java │ │ │ │ │ ├── ValidationError.java │ │ │ │ │ ├── WorkflowBuilder.java │ │ │ │ │ └── tasks/ │ │ │ │ │ ├── DoWhile.java │ │ │ │ │ ├── Dynamic.java │ │ │ │ │ ├── DynamicFork.java │ │ │ │ │ ├── DynamicForkInput.java │ │ │ │ │ ├── Event.java │ │ │ │ │ ├── ForkJoin.java │ │ │ │ │ ├── Http.java │ │ │ │ │ ├── JQ.java │ │ │ │ │ ├── Javascript.java │ │ │ │ │ ├── Join.java │ │ │ │ │ ├── SetVariable.java │ │ │ │ │ ├── SimpleTask.java │ │ │ │ │ ├── SubWorkflow.java │ │ │ │ │ ├── Switch.java │ │ │ │ │ ├── Task.java │ │ │ │ │ ├── TaskRegistry.java │ │ │ │ │ ├── Terminate.java │ │ │ │ │ └── Wait.java │ │ │ │ ├── executor/ │ │ │ │ │ ├── WorkflowExecutor.java │ │ │ │ │ └── task/ │ │ │ │ │ ├── AnnotatedWorker.java │ │ │ │ │ ├── AnnotatedWorkerExecutor.java │ │ │ │ │ ├── DynamicForkWorker.java │ │ │ │ │ ├── NonRetryableException.java │ │ │ │ │ ├── TaskContext.java │ │ │ │ │ └── WorkerConfiguration.java │ │ │ │ ├── task/ │ │ │ │ │ ├── InputParam.java │ │ │ │ │ ├── OutputParam.java │ │ │ │ │ └── WorkerTask.java │ │ │ │ └── utils/ │ │ │ │ ├── InputOutputGetter.java │ │ │ │ ├── MapBuilder.java │ │ │ │ └── ObjectMapperProvider.java │ │ │ └── resources/ │ │ │ └── test-server.properties │ │ └── test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── sdk/ │ │ │ └── workflow/ │ │ │ ├── def/ │ │ │ │ ├── TaskConversionsTests.java │ │ │ │ ├── WorkflowCreationTests.java │ │ │ │ ├── WorkflowDefTaskTests.java │ │ │ │ └── WorkflowState.java │ │ │ ├── executor/ │ │ │ │ └── task/ │ │ │ │ ├── AnnotatedWorkerTests.java │ │ │ │ └── TestWorkerConfig.java │ │ │ └── testing/ │ │ │ ├── Task1Input.java │ │ │ ├── TestWorkflowInput.java │ │ │ └── WorkflowTestFrameworkTests.java │ │ └── resources/ │ │ ├── application-integrationtest.properties │ │ ├── log4j2.xml │ │ ├── script.js │ │ ├── simple_workflow.json │ │ └── tasks.json │ ├── testing_framework.md │ ├── worker_sdk.md │ └── workflow_sdk.md ├── json-jq-task/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── tasks/ │ │ └── json/ │ │ └── JsonJqTransform.java │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── tasks/ │ └── json/ │ └── JsonJqTransformTest.java ├── licenseheader.txt ├── polyglot-clients/ │ └── README.md ├── redis-concurrency-limit/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── redis/ │ │ └── limit/ │ │ ├── RedisConcurrentExecutionLimitDAO.java │ │ └── config/ │ │ ├── RedisConcurrentExecutionLimitConfiguration.java │ │ └── RedisConcurrentExecutionLimitProperties.java │ └── test/ │ └── groovy/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── redis/ │ └── limit/ │ └── RedisConcurrentExecutionLimitDAOSpec.groovy ├── redis-lock/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── redislock/ │ │ │ ├── config/ │ │ │ │ ├── RedisLockConfiguration.java │ │ │ │ └── RedisLockProperties.java │ │ │ └── lock/ │ │ │ └── RedisLock.java │ │ └── resources/ │ │ └── META-INF/ │ │ └── additional-spring-configuration-metadata.json │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── redis/ │ └── lock/ │ └── RedisLockTest.java ├── redis-persistence/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── redis/ │ │ ├── config/ │ │ │ ├── AnyRedisCondition.java │ │ │ ├── DynomiteClusterConfiguration.java │ │ │ ├── InMemoryRedisConfiguration.java │ │ │ ├── JedisCommandsConfigurer.java │ │ │ ├── RedisClusterConfiguration.java │ │ │ ├── RedisCommonConfiguration.java │ │ │ ├── RedisProperties.java │ │ │ ├── RedisSentinelConfiguration.java │ │ │ └── RedisStandaloneConfiguration.java │ │ ├── dao/ │ │ │ ├── BaseDynoDAO.java │ │ │ ├── DynoQueueDAO.java │ │ │ ├── RedisEventHandlerDAO.java │ │ │ ├── RedisExecutionDAO.java │ │ │ ├── RedisMetadataDAO.java │ │ │ ├── RedisPollDataDAO.java │ │ │ └── RedisRateLimitingDAO.java │ │ ├── dynoqueue/ │ │ │ ├── ConfigurationHostSupplier.java │ │ │ ├── LocalhostHostSupplier.java │ │ │ └── RedisQueuesShardingStrategyProvider.java │ │ └── jedis/ │ │ ├── JedisCluster.java │ │ ├── JedisMock.java │ │ ├── JedisProxy.java │ │ ├── JedisSentinel.java │ │ └── JedisStandalone.java │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── redis/ │ ├── config/ │ │ └── utils/ │ │ └── RedisQueuesShardingStrategyProviderTest.java │ ├── dao/ │ │ ├── BaseDynoDAOTest.java │ │ ├── DynoQueueDAOTest.java │ │ ├── RedisEventHandlerDAOTest.java │ │ ├── RedisExecutionDAOTest.java │ │ ├── RedisMetadataDAOTest.java │ │ ├── RedisPollDataDAOTest.java │ │ └── RedisRateLimitDAOTest.java │ └── jedis/ │ ├── ConfigurationHostSupplierTest.java │ ├── JedisClusterTest.java │ └── JedisSentinelTest.java ├── rest/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── rest/ │ │ │ ├── config/ │ │ │ │ ├── RequestMappingConstants.java │ │ │ │ └── RestConfiguration.java │ │ │ ├── controllers/ │ │ │ │ ├── AdminResource.java │ │ │ │ ├── ApplicationExceptionMapper.java │ │ │ │ ├── EventResource.java │ │ │ │ ├── HealthCheckResource.java │ │ │ │ ├── MetadataResource.java │ │ │ │ ├── QueueAdminResource.java │ │ │ │ ├── TaskResource.java │ │ │ │ ├── ValidationExceptionMapper.java │ │ │ │ ├── WorkflowBulkResource.java │ │ │ │ └── WorkflowResource.java │ │ │ └── startup/ │ │ │ └── KitchenSinkInitializer.java │ │ └── resources/ │ │ ├── kitchensink/ │ │ │ ├── kitchenSink-ephemeralWorkflowWithEphemeralTasks.json │ │ │ ├── kitchenSink-ephemeralWorkflowWithStoredTasks.json │ │ │ ├── kitchensink.json │ │ │ ├── sub_flow_1.json │ │ │ ├── wf1.json │ │ │ └── wf2.json │ │ └── static/ │ │ └── index.html │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── rest/ │ └── controllers/ │ ├── AdminResourceTest.java │ ├── EventResourceTest.java │ ├── MetadataResourceTest.java │ ├── TaskResourceTest.java │ └── WorkflowResourceTest.java ├── server/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── netflix/ │ │ │ └── conductor/ │ │ │ └── Conductor.java │ │ └── resources/ │ │ ├── META-INF/ │ │ │ └── additional-spring-configuration-metadata.json │ │ ├── application.properties │ │ ├── banner.txt │ │ └── log4j2.xml │ └── test/ │ └── java/ │ └── com/ │ └── netflix/ │ └── conductor/ │ └── common/ │ └── config/ │ └── ConductorObjectMapperTest.java ├── settings.gradle ├── springboot-bom-overrides.gradle ├── test-harness/ │ ├── build.gradle │ └── src/ │ └── test/ │ ├── groovy/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ └── test/ │ │ ├── base/ │ │ │ ├── AbstractResiliencySpecification.groovy │ │ │ └── AbstractSpecification.groovy │ │ ├── integration/ │ │ │ ├── DecisionTaskSpec.groovy │ │ │ ├── DoWhileSpec.groovy │ │ │ ├── DynamicForkJoinSpec.groovy │ │ │ ├── EventTaskSpec.groovy │ │ │ ├── ExclusiveJoinSpec.groovy │ │ │ ├── ExternalPayloadStorageSpec.groovy │ │ │ ├── FailureWorkflowSpec.groovy │ │ │ ├── ForkJoinSpec.groovy │ │ │ ├── HierarchicalForkJoinSubworkflowRerunSpec.groovy │ │ │ ├── HierarchicalForkJoinSubworkflowRestartSpec.groovy │ │ │ ├── HierarchicalForkJoinSubworkflowRetrySpec.groovy │ │ │ ├── JsonJQTransformSpec.groovy │ │ │ ├── LambdaAndTerminateTaskSpec.groovy │ │ │ ├── NestedForkJoinSubWorkflowSpec.groovy │ │ │ ├── SetVariableTaskSpec.groovy │ │ │ ├── SimpleWorkflowSpec.groovy │ │ │ ├── StartWorkflowSpec.groovy │ │ │ ├── SubWorkflowRerunSpec.groovy │ │ │ ├── SubWorkflowRestartSpec.groovy │ │ │ ├── SubWorkflowRetrySpec.groovy │ │ │ ├── SubWorkflowSpec.groovy │ │ │ ├── SwitchTaskSpec.groovy │ │ │ ├── SystemTaskSpec.groovy │ │ │ ├── TaskLimitsWorkflowSpec.groovy │ │ │ ├── TestWorkflowSpec.groovy │ │ │ ├── WaitTaskSpec.groovy │ │ │ └── WorkflowAndTaskConfigurationSpec.groovy │ │ ├── resiliency/ │ │ │ ├── QueueResiliencySpec.groovy │ │ │ └── TaskResiliencySpec.groovy │ │ └── util/ │ │ └── WorkflowTestUtil.groovy │ ├── java/ │ │ └── com/ │ │ └── netflix/ │ │ └── conductor/ │ │ ├── ConductorTestApp.java │ │ └── test/ │ │ ├── integration/ │ │ │ ├── AbstractEndToEndTest.java │ │ │ ├── grpc/ │ │ │ │ ├── AbstractGrpcEndToEndTest.java │ │ │ │ └── GrpcEndToEndTest.java │ │ │ └── http/ │ │ │ ├── AbstractHttpEndToEndTest.java │ │ │ └── HttpEndToEndTest.java │ │ └── utils/ │ │ ├── MockExternalPayloadStorage.java │ │ └── UserTask.java │ └── resources/ │ ├── application-integrationtest.properties │ ├── concurrency_limited_task_workflow_integration_test.json │ ├── conditional_switch_task_workflow_integration_test.json │ ├── conditional_system_task_workflow_integration_test.json │ ├── conditional_task_workflow_integration_test.json │ ├── decision_and_fork_join_integration_test.json │ ├── decision_and_terminate_integration_test.json │ ├── do_while_as_subtask_integration_test.json │ ├── do_while_five_loop_over_integration_test.json │ ├── do_while_integration_test.json │ ├── do_while_iteration_fix_test.json │ ├── do_while_multiple_integration_test.json │ ├── do_while_set_variable_fix.json │ ├── do_while_sub_workflow_integration_test.json │ ├── do_while_system_tasks.json │ ├── do_while_with_decision_task.json │ ├── dynamic_fork_join_integration_test.json │ ├── event_workflow_integration_test.json │ ├── exclusive_join_integration_test.json │ ├── failure_workflow_for_terminate_task_workflow.json │ ├── fork_join_integration_test.json │ ├── fork_join_sub_workflow.json │ ├── fork_join_with_no_task_retry_integration_test.json │ ├── fork_join_with_optional_sub_workflow_forks_integration_test.json │ ├── hierarchical_fork_join_swf.json │ ├── input.json │ ├── json_jq_transform_result_integration_test.json │ ├── nested_fork_join_integration_test.json │ ├── nested_fork_join_swf.json │ ├── nested_fork_join_with_sub_workflow_integration_test.json │ ├── output.json │ ├── rate_limited_simple_task_workflow_integration_test.json │ ├── rate_limited_system_task_workflow_integration_test.json │ ├── sequential_json_jq_transform_integration_test.json │ ├── set_variable_workflow_integration_test.json │ ├── simple_decision_task_integration_test.json │ ├── simple_json_jq_transform_integration_test.json │ ├── simple_lambda_workflow_integration_test.json │ ├── simple_one_task_sub_workflow_integration_test.json │ ├── simple_set_variable_workflow_integration_test.json │ ├── simple_switch_task_integration_test.json │ ├── simple_wait_task_workflow_integration_test.json │ ├── simple_workflow_1_input_template_integration_test.json │ ├── simple_workflow_1_integration_test.json │ ├── simple_workflow_3_integration_test.json │ ├── simple_workflow_with_async_complete_system_task_integration_test.json │ ├── simple_workflow_with_optional_task_integration_test.json │ ├── simple_workflow_with_resp_time_out_integration_test.json │ ├── simple_workflow_with_sub_workflow_inline_def_integration_test.json │ ├── start_workflow_input.json │ ├── switch_and_fork_join_integration_test.json │ ├── switch_and_terminate_integration_test.json │ ├── switch_with_no_default_case_integration_test.json │ ├── terminate_task_completed_workflow_integration_test.json │ ├── terminate_task_failed_workflow_integration.json │ ├── terminate_task_parent_workflow.json │ ├── terminate_task_sub_workflow.json │ ├── test_task_failed_parent_workflow.json │ ├── test_task_failed_sub_workflow.json │ ├── wait_workflow_integration_test.json │ ├── workflow_that_starts_another_workflow.json │ ├── workflow_with_sub_workflow_1_integration_test.json │ └── workflow_with_synchronous_system_task.json └── ui/ ├── .eslintrc ├── .gitignore ├── .prettierignore ├── .prettierrc.json ├── README.md ├── cypress/ │ ├── e2e/ │ │ └── spec.cy.js │ ├── fixtures/ │ │ ├── doWhile/ │ │ │ └── doWhileSwitch.json │ │ ├── dynamicFork/ │ │ │ ├── externalizedInput.json │ │ │ ├── noneSpawned.json │ │ │ ├── notExecuted.json │ │ │ ├── oneFailed.json │ │ │ └── success.json │ │ ├── dynamicFork.json │ │ ├── metadataTasks.json │ │ ├── metadataWorkflow.json │ │ ├── taskSearch.json │ │ └── workflowSearch.json │ └── support/ │ ├── commands.ts │ ├── component-index.html │ ├── component.ts │ └── e2e.ts ├── cypress.config.ts ├── package.json ├── public/ │ ├── index.html │ └── robots.txt ├── src/ │ ├── App.jsx │ ├── components/ │ │ ├── Banner.jsx │ │ ├── Button.jsx │ │ ├── ButtonGroup.jsx │ │ ├── ConfirmChoiceDialog.jsx │ │ ├── CustomButtons.jsx │ │ ├── DataTable.jsx │ │ ├── DateRangePicker.jsx │ │ ├── Dropdown.jsx │ │ ├── DropdownButton.jsx │ │ ├── Heading.jsx │ │ ├── Input.jsx │ │ ├── KeyValueTable.jsx │ │ ├── LinearProgress.jsx │ │ ├── NavLink.jsx │ │ ├── Paper.jsx │ │ ├── Pill.jsx │ │ ├── PrimaryButton.jsx │ │ ├── ReactJson.jsx │ │ ├── SecondaryButton.jsx │ │ ├── Select.jsx │ │ ├── SplitButton.jsx │ │ ├── StatusBadge.jsx │ │ ├── Tabs.jsx │ │ ├── TaskLink.jsx │ │ ├── TaskNameInput.jsx │ │ ├── TertiaryButton.jsx │ │ ├── Text.jsx │ │ ├── WorkflowNameInput.jsx │ │ ├── definitionList/ │ │ │ └── DefinitionList.jsx │ │ ├── diagram/ │ │ │ ├── TaskPointer.d.ts │ │ │ ├── TaskResult.d.ts │ │ │ ├── WorkflowDAG.js │ │ │ ├── WorkflowGraph.jsx │ │ │ ├── WorkflowGraph.test.cy.js │ │ │ └── diagram.scss │ │ ├── formik/ │ │ │ ├── FormikCronEditor.jsx │ │ │ ├── FormikDropdown.jsx │ │ │ ├── FormikInput.jsx │ │ │ ├── FormikJsonInput.jsx │ │ │ ├── FormikSwitch.jsx │ │ │ ├── FormikVersionDropdown.jsx │ │ │ ├── FormikWorkflowNameInput.jsx │ │ │ └── cron.css │ │ └── index.js │ ├── data/ │ │ ├── actions.js │ │ ├── bulkactions.js │ │ ├── common.js │ │ ├── misc.js │ │ ├── task.js │ │ └── workflow.js │ ├── hooks/ │ │ └── useTime.js │ ├── index.css │ ├── index.js │ ├── pages/ │ │ ├── definition/ │ │ │ ├── EventHandler.jsx │ │ │ ├── ResetConfirmationDialog.jsx │ │ │ ├── SaveTaskDialog.jsx │ │ │ ├── SaveWorkflowDialog.jsx │ │ │ ├── TaskDefinition.jsx │ │ │ └── WorkflowDefinition.jsx │ │ ├── definitions/ │ │ │ ├── EventHandler.jsx │ │ │ ├── Header.jsx │ │ │ ├── Task.jsx │ │ │ └── Workflow.jsx │ │ ├── execution/ │ │ │ ├── ActionModule.jsx │ │ │ ├── Execution.jsx │ │ │ ├── ExecutionInputOutput.jsx │ │ │ ├── ExecutionJson.jsx │ │ │ ├── ExecutionSummary.jsx │ │ │ ├── Legend.jsx │ │ │ ├── RightPanel.jsx │ │ │ ├── TaskDetails.jsx │ │ │ ├── TaskList.jsx │ │ │ ├── TaskLogs.jsx │ │ │ ├── TaskPollData.jsx │ │ │ ├── TaskSummary.jsx │ │ │ ├── Timeline.jsx │ │ │ └── timeline.scss │ │ ├── executions/ │ │ │ ├── BulkActionModule.jsx │ │ │ ├── ResultsTable.jsx │ │ │ ├── SearchTabs.jsx │ │ │ ├── TaskResultsTable.jsx │ │ │ ├── TaskSearch.jsx │ │ │ ├── WorkflowSearch.jsx │ │ │ └── executionsStyles.js │ │ ├── kitchensink/ │ │ │ ├── DataTableDemo.jsx │ │ │ ├── DiagramTest.jsx │ │ │ ├── Dropdown.jsx │ │ │ ├── EnhancedTable.jsx │ │ │ ├── Examples.jsx │ │ │ ├── Gantt.jsx │ │ │ ├── KitchenSink.jsx │ │ │ └── sampleMovieData.js │ │ ├── misc/ │ │ │ └── TaskQueue.jsx │ │ ├── styles.js │ │ └── workbench/ │ │ ├── ExecutionHistory.jsx │ │ ├── RunHistory.tsx │ │ ├── Workbench.jsx │ │ └── WorkbenchForm.jsx │ ├── plugins/ │ │ ├── AppBarModules.jsx │ │ ├── AppLogo.jsx │ │ ├── CustomAppBarButtons.jsx │ │ ├── CustomRoutes.jsx │ │ ├── constants.js │ │ ├── customTypeRenderers.jsx │ │ ├── env.js │ │ └── fetch.js │ ├── react-app-env.d.ts │ ├── schema/ │ │ ├── task.js │ │ └── workflow.js │ ├── serviceWorker.js │ ├── setupProxy.js │ ├── setupTests.js │ ├── theme/ │ │ ├── colorOverrides.js │ │ ├── colors.js │ │ ├── index.js │ │ ├── provider.jsx │ │ ├── theme.js │ │ └── variables.js │ └── utils/ │ ├── constants.js │ ├── helpers.js │ ├── localstorage.ts │ └── path.js ├── test-karbon.sh └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ # General # Backend server/build/libs # UI **/node_modules ui/build ================================================ FILE: .gitattributes ================================================ gradlew eol=lf *.gradle eol=lf *.java eol=lf *.groovy eol=lf spring.factories eol=lf *.sh eol=lf docs/* linguist-documentation server/src/main/resources/swagger-ui/* linguist-vendored ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: "" labels: 'type: bug' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **Details** Conductor version: Persistence implementation: Cassandra, Postgres, MySQL, Dynomite etc Queue implementation: Postgres, MySQL, Dynoqueues etc Lock: Redis or Zookeeper? Workflow definition: Task definition: Event handler definition: **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false ================================================ FILE: .github/ISSUE_TEMPLATE/documentation.md ================================================ --- name: Documentation about: Something in the documentation that needs improvement title: "[DOC]: " labels: 'type: docs' assignees: '' --- ## What are you missing in the docs ## Proposed text ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Propose a new feature title: "[FEATURE]: " labels: 'type: feature' assignees: '' --- Please read our [contributor guide](https://github.com/Netflix/conductor/blob/main/CONTRIBUTING.md) before creating an issue. Also consider discussing your idea on the [discussion forum](https://github.com/Netflix/conductor/discussions) first. ## Describe the Feature Request _A clear and concise description of what the feature request is._ ## Describe Preferred Solution _A clear and concise description of what you want to happen._ ## Describe Alternatives _A clear and concise description of any alternative solutions or features you've considered._ ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "gradle" directory: "/" schedule: interval: "weekly" reviewers: - "aravindanr" - "jxu-nflx" - "apanicker-nflx" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/pull_request_template.md ================================================ Pull Request type ---- - [ ] Bugfix - [ ] Feature - [ ] Refactoring (no functional changes, no api changes) - [ ] Build related changes (Please run `./gradlew generateLock saveLock` to refresh dependencies) - [ ] WHOSUSING.md - [ ] Other (please describe): **NOTE**: Please remember to run `./gradlew spotlessApply` to fix any format violations. Changes in this PR ---- _Describe the new behavior from this PR, and why it's needed_ Issue # Alternatives considered ---- _Describe alternative implementation you have considered_ ================================================ FILE: .github/release-drafter.yml ================================================ template: | ## What’s Changed $CHANGES name-template: 'v$RESOLVED_VERSION' tag-template: 'v$RESOLVED_VERSION' categories: - title: 'IMPORTANT' label: 'type: important' - title: 'New' label: 'type: feature' - title: 'Bug Fixes' label: 'type: bug' - title: 'Refactor' label: 'type: maintenance' - title: 'Documentation' label: 'type: docs' - title: 'Dependency Updates' label: 'type: dependencies' version-resolver: minor: labels: - 'type: important' patch: labels: - 'type: bug' - 'type: maintenance' - 'type: docs' - 'type: dependencies' - 'type: feature' exclude-labels: - 'skip-changelog' - 'gradle-wrapper' - 'github_actions' ================================================ FILE: .github/workflows/ci.yml ================================================ name: CI on: [ push, pull_request ] jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Gradle wrapper validation uses: gradle/wrapper-validation-action@v1 - name: Set up Zulu JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' java-version: '17' - name: Cache SonarCloud packages uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Cache Gradle packages uses: actions/cache@v3 with: path: | ~/.gradle/caches ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - name: Build with Gradle if: github.ref != 'refs/heads/main' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | ./gradlew build --scan - name: Build and Publish snapshot if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' run: | echo "Running build for commit ${{ github.sha }}" ./gradlew build snapshot --scan env: NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() with: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload build artifacts uses: actions/upload-artifact@v3 with: name: build-artifacts path: '**/build/reports' - name: Store Buildscan URL uses: actions/upload-artifact@v3 with: name: build-scan path: 'buildscan.log' build-ui: runs-on: ubuntu-latest container: cypress/browsers:node14.17.6-chrome100-ff98 defaults: run: working-directory: ui steps: - uses: actions/checkout@v3 - name: Install Dependencies run: yarn install - name: Build UI run: yarn run build - name: Run E2E Tests uses: cypress-io/github-action@v4 with: working-directory: ui install: false start: yarn run serve-build wait-on: 'http://localhost:5000' - name: Run Component Tests uses: cypress-io/github-action@v4 with: working-directory: ui install: false component: true - name: Archive test screenshots uses: actions/upload-artifact@v2 if: failure() with: name: cypress-screenshots path: ui/cypress/screenshots - name: Archive test videos uses: actions/upload-artifact@v2 if: always() with: name: cypress-videos path: ui/cypress/videos ================================================ FILE: .github/workflows/publish.yml ================================================ name: Publish to NetflixOSS and Maven Central on: release: types: - released - prereleased permissions: contents: read jobs: publish: runs-on: ubuntu-latest name: Gradle Build and Publish steps: - uses: actions/checkout@v3 - name: Set up Zulu JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' java-version: '17' - name: Cache Gradle packages uses: actions/cache@v3 with: path: | ~/.gradle/caches ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle- - name: Publish candidate if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-rc.') run: ./gradlew -Prelease.useLastTag=true candidate --scan env: NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} - name: Publish release if: startsWith(github.ref, 'refs/tags/v') && (!contains(github.ref, '-rc.')) run: ./gradlew -Prelease.useLastTag=true final --scan env: NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} - name: Publish tag to community repo if: startsWith(github.ref, 'refs/tags/v') run: | export TAG=$(git describe --tags --abbrev=0) echo "Current release version is $TAG" echo "Triggering community build" curl \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: Bearer ${{ secrets.COMMUNITY_REPO_TRIGGER }}" \ -X POST https://api.github.com/repos/Netflix/conductor-community/dispatches \ -d '{"event_type": "publish_build","client_payload": {"tag":"'"$TAG"'"}}' - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails with: report_paths: '**/build/test-results/test/TEST-*.xml' ================================================ FILE: .github/workflows/release_draft.yml ================================================ name: Release Drafter on: push: branches: - main permissions: contents: read jobs: update_release_draft: permissions: contents: write # for release-drafter/release-drafter to create a github release pull-requests: write # for release-drafter/release-drafter to add label to PR runs-on: ubuntu-latest steps: - uses: release-drafter/release-drafter@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/stale.yml ================================================ name: Close stale issues and pull requests on: schedule: - cron: "0 0 * * *" permissions: contents: read jobs: stale: permissions: issues: write # for actions/stale to close stale issues pull-requests: write # for actions/stale to close stale PRs runs-on: ubuntu-latest steps: - uses: actions/stale@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale, because it has been open for 45 days with no activity. Remove the stale label or comment, or this will be closed in 7 days.' close-issue-message: 'This issue was closed, because it has been stalled for 7 days with no activity.' stale-pr-message: 'This PR is stale, because it has been open for 45 days with no activity. Remove the stale label or comment, or this will be closed in 7 days.' close-pr-message: 'This PR was closed, because it has been stalled for 7 days with no activity.' days-before-issue-stale: 45 days-before-issue-close: 7 days-before-pr-stale: 45 days-before-pr-close: 7 exempt-issue-labels: 'type: bug,enhancement,work_in_progress,help_wanted' ================================================ FILE: .github/workflows/update-gradle-wrapper.yml ================================================ name: Update Gradle Wrapper on: schedule: - cron: "0 0 * * *" workflow_dispatch: jobs: update-gradle-wrapper: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Zulu JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' java-version: '17' - name: Update Gradle Wrapper uses: gradle-update/update-gradle-wrapper-action@v1 ================================================ FILE: .gitignore ================================================ # Java Build .gradle .classpath dump.rdb out bin target buildscan.log /docs/site # Python /polyglot-clients/python/conductor.egg-info *.pyc # OS & IDE .DS_Store .settings .vscode .idea .project *.iml # JS & UI Related node_modules /ui/build /ui/public/monaco-editor # publishing secrets secrets/signing-key # local builds lib/ build/ */build/ # asdf version file .tool-versions ================================================ FILE: CHANGELOG.md ================================================ Conductor has been upgraded to use the SpringBoot framework and requires Java11 or above. #### NOTE: The java clients (conductor-client, conductor-client-spring, conductor-grpc-client) are still compiled using Java8 to ensure backward compatibility and smoother migration. ## Removals/Deprecations - Removed support for EmbeddedElasticSearch - Removed deprecated constructors in DynoQueueDAO - Removed deprecated methods in the Worker interface - Removed OAuth Support in HTTP task (Looking for contributions for OAuth/OAuth2.0) - Removed deprecated fields and methods in the Workflow object - Removed deprecated fields and methods in the Task object - Removed deprecated fields and methods in the WorkflowTask object Removed unused methods from QueueDAO: - List pop(String, int, int, long) - List pollMessages(String, int, int, long) Removed APIs: - GET /tasks/in_progress/{tasktype} - GET /tasks/in_progress/{workflowId}/{taskRefName} - POST /tasks/{taskId}/ack - POST /tasks/queue/requeue - DELETE /queue/{taskType}/{taskId} - GET /event/queues - GET /event/queues/providers - void restart(String) in workflow client - List getPendingTasksByType(String, String, Integer) in task client - Task getPendingTaskForWorkflow(String, String) in task client - boolean preAck(Task) in Worker - int getPollCount() in Worker ## What's changed Changes to configurations: ### `azureblob-storage` module: | Old | New | Default | | --- | --- | --- | | workflow.external.payload.storage.azure_blob.connection_string | conductor.external-payload-storage.azureblob.connectionString | null | | workflow.external.payload.storage.azure_blob.container_name | conductor.external-payload-storage.azureblob.containerName | conductor-payloads | | workflow.external.payload.storage.azure_blob.endpoint | conductor.external-payload-storage.azureblob.endpoint | null | | workflow.external.payload.storage.azure_blob.sas_token | conductor.external-payload-storage.azureblob.sasToken | null | | workflow.external.payload.storage.azure_blob.signedurlexpirationseconds | conductor.external-payload-storage.azureblob.signedUrlExpirationDuration | 5s | | workflow.external.payload.storage.azure_blob.workflow_input_path | conductor.external-payload-storage.azureblob.workflowInputPath | workflow/input/ | | workflow.external.payload.storage.azure_blob.workflow_output_path | conductor.external-payload-storage.azureblob.workflowOutputPath | workflow/output/ | | workflow.external.payload.storage.azure_blob.task_input_path | conductor.external-payload-storage.azureblob.taskInputPath | task/input/ | | workflow.external.payload.storage.azure_blob.task_output_path | conductor.external-payload-storage.azureblob.taskOutputPath | task/output/ | ### `cassandra-persistence` module: | Old | New | Default | | --- | --- | --- | | workflow.cassandra.host | conductor.cassandra.hostAddress | 127.0.0.1 | | workflow.cassandra.port | conductor.cassandra.port | 9142 | | workflow.cassandra.cluster | conductor.cassandra.cluster | "" | | workflow.cassandra.keyspace | conductor.cassandra.keyspace | conductor | | workflow.cassandra.shard.size | conductor.cassandra.shardSize | 100 | | workflow.cassandra.replication.strategy | conductor.cassandra.replicationStrategy | SimpleStrategy | | workflow.cassandra.replication.factor.key | conductor.cassandra.replicationFactorKey | replication_factor | | workflow.cassandra.replication.factor.value | conductor.cassandra.replicationFactorValue | 3 | | workflow.cassandra.read.consistency.level | conductor.cassandra.readConsistencyLevel | LOCAL_QUORUM | | workflow.cassandra.write.consistency.level | conductor.cassandra.writeConsistencyLevel | LOCAL_QUORUM | | conductor.taskdef.cache.refresh.time.seconds | conductor.cassandra.taskDefCacheRefreshInterval | 60s | | conductor.eventhandler.cache.refresh.time.seconds | conductor.cassandra.eventHandlerCacheRefreshInterval | 60s | | workflow.event.execution.persistence.ttl.seconds | conductor.cassandra.eventExecutionPersistenceTTL | 0s | ### `contribs` module: | Old | New | Default | | --- | --- | --- | | workflow.archival.ttl.seconds | conductor.workflow-status-listener.archival.ttlDuration | 0s | | workflow.archival.delay.queue.worker.thread.count | conductor.workflow-status-listener.archival.delayQueueWorkerThreadCount | 5 | | workflow.archival.delay.seconds | conductor.workflow-status-listener.archival.delaySeconds | 60 | | | | | workflowstatuslistener.publisher.success.queue | conductor.workflow-status-listener.queue-publisher.successQueue | _callbackSuccessQueue | | workflowstatuslistener.publisher.failure.queue | conductor.workflow-status-listener.queue-publisher.failureQueue | _callbackFailureQueue | | | | | | com.netflix.conductor.contribs.metrics.LoggingMetricsModule.reportPeriodSeconds | conductor.metrics-logger.reportInterval | 30s | | | | | | workflow.event.queues.amqp.batchSize | conductor.event-queues.amqp.batchSize | 1 | | workflow.event.queues.amqp.pollTimeInMs | conductor.event-queues.amqp.pollTimeDuration | 100ms | | workflow.event.queues.amqp.hosts | conductor.event-queues.amqp.hosts | localhost | | workflow.event.queues.amqp.username | conductor.event-queues.amqp.username | guest | | workflow.event.queues.amqp.password | conductor.event-queues.amqp.password | guest | | workflow.event.queues.amqp.virtualHost | conductor.event-queues.amqp.virtualHost | / | | workflow.event.queues.amqp.port | conductor.event-queues.amqp.port.port | 5672 | | workflow.event.queues.amqp.connectionTimeout | conductor.event-queues.amqp.connectionTimeout | 60000ms | | workflow.event.queues.amqp.useNio | conductor.event-queues.amqp.useNio | false | | workflow.event.queues.amqp.durable | conductor.event-queues.amqp.durable | true | | workflow.event.queues.amqp.exclusive | conductor.event-queues.amqp.exclusive | false | | workflow.event.queues.amqp.autoDelete | conductor.event-queues.amqp.autoDelete | false | | workflow.event.queues.amqp.contentType | conductor.event-queues.amqp.contentType | application/json | | workflow.event.queues.amqp.contentEncoding | conductor.event-queues.amqp.contentEncoding | UTF-8 | | workflow.event.queues.amqp.amqp_exchange | conductor.event-queues.amqp.exchangeType | topic | | workflow.event.queues.amqp.deliveryMode | conductor.event-queues.amqp.deliveryMode | 2 | | workflow.listener.queue.useExchange | conductor.event-queues.amqp.useExchange | true | | workflow.listener.queue.prefix | conductor.event-queues.amqp.listenerQueuePrefix | "" | | | | | | io.nats.streaming.clusterId | conductor.event-queues.nats-stream.clusterId | test-cluster | | io.nats.streaming.durableName | conductor.event-queues.nats-stream.durableName | null | | io.nats.streaming.url | conductor.event-queues.nats-stream.url | nats://localhost:4222 | | | | | | workflow.event.queues.sqs.batchSize | conductor.event-queues.sqs.batchSize | 1 | | workflow.event.queues.sqs.pollTimeInMS | conductor.event-queues.sqs.pollTimeDuration | 100ms | | workflow.event.queues.sqs.visibilityTimeoutInSeconds | conductor.event-queues.sqs.visibilityTimeout | 60s | | workflow.listener.queue.prefix | conductor.event-queues.sqs.listenerQueuePrefix | "" | | workflow.listener.queue.authorizedAccounts | conductor.event-queues.sqs.authorizedAccounts | "" | | | | | | workflow.external.payload.storage.s3.bucket | conductor.external-payload-storage.s3.bucketName | conductor_payloads | | workflow.external.payload.storage.s3.signedurlexpirationseconds | conductor.external-payload-storage.s3.signedUrlExpirationDuration | 5s | | workflow.external.payload.storage.s3.region | conductor.external-payload-storage.s3.region | us-east-1 | | | | | | http.task.read.timeout | conductor.tasks.http.readTimeout | 150ms | | http.task.connect.timeout | conductor.tasks.http.connectTimeout | 100ms | | | | | | kafka.publish.request.timeout.ms | conductor.tasks.kafka-publish.requestTimeout | 100ms | | kafka.publish.max.block.ms | conductor.tasks.kafka-publish.maxBlock | 500ms | | kafka.publish.producer.cache.size | conductor.tasks.kafka-publish.cacheSize | 10 | | kafka.publish.producer.cache.time.ms | conductor.tasks.kafka-publish.cacheTime | 120000ms | ### `core` module: | Old | New | Default | | --- | --- | --- | | environment | _removed_ | | | STACK | conductor.app.stack | test | | APP_ID | conductor.app.appId | conductor | | workflow.executor.service.max.threads | conductor.app.executorServiceMaxThreadCount | 50 | | decider.sweep.frequency.seconds | conductor.app.sweepFrequency | 30s | | workflow.sweeper.thread.count | conductor.app.sweeperThreadCount | 5 | | - | conductor.app.sweeperWorkflowPollTimeout | 2000ms | | workflow.event.processor.thread.count | conductor.app.eventProcessorThreadCount | 2 | | workflow.event.message.indexing.enabled | conductor.app.eventMessageIndexingEnabled | true | | workflow.event.execution.indexing.enabled | conductor.app.eventExecutionIndexingEnabled | true | | workflow.decider.locking.enabled | conductor.app.workflowExecutionLockEnabled | false | | workflow.locking.lease.time.ms | conductor.app.lockLeaseTime | 60000ms | | workflow.locking.time.to.try.ms | conductor.app.lockTimeToTry | 500ms | | tasks.active.worker.lastpoll | conductor.app.activeWorkerLastPollTimeout | 10s | | task.queue.message.postponeSeconds | conductor.app.taskExecutionPostponeDuration | 60s | | workflow.taskExecLog.indexing.enabled | conductor.app.taskExecLogIndexingEnabled | true | | async.indexing.enabled | conductor.app.asyncIndexingEnabled | false | | workflow.system.task.worker.thread.count | conductor.app.systemTaskWorkerThreadCount | # available processors * 2 | | workflow.system.task.worker.callback.seconds | conductor.app.systemTaskWorkerCallbackDuration | 30s | | workflow.system.task.worker.poll.interval | conductor.app.systemTaskWorkerPollInterval | 50s | | workflow.system.task.worker.executionNameSpace | conductor.app.systemTaskWorkerExecutionNamespace | "" | | workflow.isolated.system.task.worker.thread.count | conductor.app.isolatedSystemTaskWorkerThreadCount | 1 | | workflow.system.task.queue.pollCount | conductor.app.systemTaskMaxPollCount | 1 | | async.update.short.workflow.duration.seconds | conductor.app.asyncUpdateShortRunningWorkflowDuration | 30s | | async.update.delay.seconds | conductor.app.asyncUpdateDelay | 60s | | summary.input.output.json.serialization.enabled | conductor.app.summary-input-output-json-serialization.enabled | false | | workflow.owner.email.mandatory | conductor.app.ownerEmailMandatory | true | | workflow.repairservice.enabled | conductor.app.workflowRepairServiceEnabled | false | | workflow.event.queue.scheduler.poll.thread.count | conductor.app.eventSchedulerPollThreadCount | # CPU cores | | workflow.dyno.queues.pollingInterval | conductor.app.eventQueuePollInterval | 100ms | | workflow.dyno.queues.pollCount | conductor.app.eventQueuePollCount | 10 | | workflow.dyno.queues.longPollTimeout | conductor.app.eventQueueLongPollTimeout | 1000ms | | conductor.workflow.input.payload.threshold.kb | conductor.app.workflowInputPayloadSizeThreshold | 5120KB | | conductor.max.workflow.input.payload.threshold.kb | conductor.app.maxWorkflowInputPayloadSizeThreshold | 10240KB | | conductor.workflow.output.payload.threshold.kb | conductor.app.workflowOutputPayloadSizeThreshold | 5120KB | | conductor.max.workflow.output.payload.threshold.kb | conductor.app.maxWorkflowOutputPayloadSizeThreshold | 10240KB | | conductor.task.input.payload.threshold.kb | conductor.app.taskInputPayloadSizeThreshold | 3072KB | | conductor.max.task.input.payload.threshold.kb | conductor.app.maxTaskInputPayloadSizeThreshold | 10240KB | | conductor.task.output.payload.threshold.kb | conductor.app.taskOutputPayloadSizeThreshold | 3072KB | | conductor.max.task.output.payload.threshold.kb | conductor.app.maxTaskOutputPayloadSizeThreshold | 10240KB | | conductor.max.workflow.variables.payload.threshold.kb | conductor.app.maxWorkflowVariablesPayloadSizeThreshold | 256KB | | | | | | workflow.isolated.system.task.enable | conductor.app.isolatedSystemTaskEnabled | false | | workflow.isolated.system.task.poll.time.secs | conductor.app.isolatedSystemTaskQueuePollInterval | 10s | | | | | | workflow.task.pending.time.threshold.minutes | conductor.app.taskPendingTimeThreshold | 60m | | | | | | workflow.monitor.metadata.refresh.counter | conductor.workflow-monitor.metadataRefreshInterval | 10 | | workflow.monitor.stats.freq.seconds | conductor.workflow-monitor.statsFrequency | 60s | ### `es6-persistence` module: | Old | New | Default | | --- | --- | --- | | workflow.elasticsearch.version | conductor.elasticsearch.version | 6 | | workflow.elasticsearch.url | conductor.elasticsearch.url | localhost:9300 | | workflow.elasticsearch.index.name | conductor.elasticsearch.indexPrefix | conductor | | workflow.elasticsearch.tasklog.index.name | _removed_ | | | workflow.elasticsearch.cluster.health.color | conductor.elasticsearch.clusterHealthColor | green | | workflow.elasticsearch.archive.search.batchSize | _removed_ | | | workflow.elasticsearch.index.batchSize | conductor.elasticsearch.indexBatchSize | 1 | | workflow.elasticsearch.async.dao.worker.queue.size | conductor.elasticsearch.asyncWorkerQueueSize | 100 | | workflow.elasticsearch.async.dao.max.pool.size | conductor.elasticsearch.asyncMaxPoolSize | 12 | | workflow.elasticsearch.async.buffer.flush.timeout.seconds | conductor.elasticsearch.asyncBufferFlushTimeout | 10s | | workflow.elasticsearch.index.shard.count | conductor.elasticsearch.indexShardCount | 5 | | workflow.elasticsearch.index.replicas.count | conductor.elasticsearch.indexReplicasCount | 1 | | tasklog.elasticsearch.query.size | conductor.elasticsearch.taskLogResultLimit | 10 | | workflow.elasticsearch.rest.client.connectionRequestTimeout.milliseconds | conductor.elasticsearch.restClientConnectionRequestTimeout | -1 | | workflow.elasticsearch.auto.index.management.enabled | conductor.elasticsearch.autoIndexManagementEnabled | true | | workflow.elasticsearch.document.type.override | conductor.elasticsearch.documentTypeOverride | "" | ### `es7-persistence` module: | Old | New | Default | | --- | --- | --- | | workflow.elasticsearch.version | conductor.elasticsearch.version | 7 | | workflow.elasticsearch.url | conductor.elasticsearch.url | localhost:9300 | | workflow.elasticsearch.index.name | conductor.elasticsearch.indexPrefix | conductor | | workflow.elasticsearch.tasklog.index.name | _removed_ | | | workflow.elasticsearch.cluster.health.color | conductor.elasticsearch.clusterHealthColor | green | | workflow.elasticsearch.archive.search.batchSize | _removed_ | | | workflow.elasticsearch.index.batchSize | conductor.elasticsearch.indexBatchSize | 1 | | workflow.elasticsearch.async.dao.worker.queue.size | conductor.elasticsearch.asyncWorkerQueueSize | 100 | | workflow.elasticsearch.async.dao.max.pool.size | conductor.elasticsearch.asyncMaxPoolSize | 12 | | workflow.elasticsearch.async.buffer.flush.timeout.seconds | conductor.elasticsearch.asyncBufferFlushTimeout | 10s | | workflow.elasticsearch.index.shard.count | conductor.elasticsearch.indexShardCount | 5 | | workflow.elasticsearch.index.replicas.count | conductor.elasticsearch.indexReplicasCount | 1 | | tasklog.elasticsearch.query.size | conductor.elasticsearch.taskLogResultLimit | 10 | | workflow.elasticsearch.rest.client.connectionRequestTimeout.milliseconds | conductor.elasticsearch.restClientConnectionRequestTimeout | -1 | | workflow.elasticsearch.auto.index.management.enabled | conductor.elasticsearch.autoIndexManagementEnabled | true | | workflow.elasticsearch.document.type.override | conductor.elasticsearch.documentTypeOverride | "" | | workflow.elasticsearch.basic.auth.username | conductor.elasticsearch.username | "" | | workflow.elasticsearch.basic.auth.password | conductor.elasticsearch.password | "" | ### `grpc-server` module: | Old | New | Default | | --- | --- | --- | | conductor.grpc.server.port | conductor.grpc-server.port | 8090 | | conductor.grpc.server.reflectionEnabled | conductor.grpc-server.reflectionEnabled | true | ### `mysql-persistence` module (v3.0.0 - v3.0.5): | Old | New | Default | | --- | --- | --- | | jdbc.url | conductor.mysql.jdbcUrl | jdbc:mysql://localhost:3306/conductor | | jdbc.username | conductor.mysql.jdbcUsername | conductor | | jdbc.password | conductor.mysql.jdbcPassword | password | | flyway.enabled | conductor.mysql.flywayEnabled | true | | flyway.table | conductor.mysql.flywayTable | null | | conductor.mysql.connection.pool.size.max | conductor.mysql.connectionPoolMaxSize | -1 | | conductor.mysql.connection.pool.idle.min | conductor.mysql.connectionPoolMinIdle | -1 | | conductor.mysql.connection.lifetime.max | conductor.mysql.connectionMaxLifetime | 30m | | conductor.mysql.connection.idle.timeout | conductor.mysql.connectionIdleTimeout | 10m | | conductor.mysql.connection.timeout | conductor.mysql.connectionTimeout | 30s | | conductor.mysql.transaction.isolation.level | conductor.mysql.transactionIsolationLevel | "" | | conductor.mysql.autocommit | conductor.mysql.autoCommit | false | | conductor.taskdef.cache.refresh.time.seconds | conductor.mysql.taskDefCacheRefreshInterval | 60s | ### `mysql-persistence` module (v3.0.5+): | Old | New | | --- | --- | | jdbc.url | spring.datasource.url | | jdbc.username | spring.datasource.username | | jdbc.password | spring.datasource.password | | flyway.enabled | spring.flyway.enabled | | flyway.table | spring.flyway.table | | conductor.mysql.connection.pool.size.max | spring.datasource.hikari.maximum-pool-size | | conductor.mysql.connection.pool.idle.min | spring.datasource.hikari.minimum-idle | | conductor.mysql.connection.lifetime.max | spring.datasource.hikari.max-lifetime | | conductor.mysql.connection.idle.timeout | spring.datasource.hikari.idle-timeout | | conductor.mysql.connection.timeout | spring.datasource.hikari.connection-timeout | | conductor.mysql.transaction.isolation.level | spring.datasource.hikari.transaction-isolation | | conductor.mysql.autocommit | spring.datasource.hikari.auto-commit | | conductor.taskdef.cache.refresh.time.seconds | conductor.mysql.taskDefCacheRefreshInterval | * for more properties and default values: https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#application-properties.data.spring.datasource.hikari ### `postgres-persistence` module (v3.0.0 - v3.0.5): | Old | New | Default | | --- | --- | --- | | jdbc.url | conductor.postgres.jdbcUrl | jdbc:postgresql://localhost:5432/conductor | | jdbc.username | conductor.postgres.jdbcUsername | conductor | | jdbc.password | conductor.postgres.jdbcPassword | password | | flyway.enabled | conductor.postgres.flywayEnabled | true | | flyway.table | conductor.postgres.flywayTable | null | | conductor.postgres.connection.pool.size.max | conductor.postgres.connectionPoolMaxSize | -1 | | conductor.postgres.connection.pool.idle.min | conductor.postgres.connectionPoolMinIdle | -1 | | conductor.postgres.connection.lifetime.max | conductor.postgres.connectionMaxLifetime | 30m | | conductor.postgres.connection.idle.timeout | conductor.postgres.connectionIdleTimeout | 10m | | conductor.postgres.connection.timeout | conductor.postgres.connectionTimeout | 30s | | conductor.postgres.transaction.isolation.level | conductor.postgres.transactionIsolationLevel | "" | | conductor.postgres.autocommit | conductor.postgres.autoCommit | false | | conductor.taskdef.cache.refresh.time.seconds | conductor.postgres.taskDefCacheRefreshInterval | 60s | ### `postgres-persistence` module (v3.0.5+): | Old | New | | --- | --- | | jdbc.url | spring.datasource.url | | jdbc.username | spring.datasource.username | | jdbc.password | spring.datasource.password | | flyway.enabled | spring.flyway.enabled | | flyway.table | spring.flyway.table | | conductor.postgres.connection.pool.size.max | spring.datasource.hikari.maximum-pool-size | | conductor.postgres.connection.pool.idle.min | spring.datasource.hikari.minimum-idle | | conductor.postgres.connection.lifetime.max | spring.datasource.hikari.max-lifetime | | conductor.postgres.connection.idle.timeout | spring.datasource.hikari.idle-timeout | | conductor.postgres.connection.timeout | spring.datasource.hikari.connection-timeout | | conductor.postgres.transaction.isolation.level | spring.datasource.hikari.transaction-isolation | | conductor.postgres.autocommit | spring.datasource.hikari.auto-commit | | conductor.taskdef.cache.refresh.time.seconds | conductor.postgres.taskDefCacheRefreshInterval | * for more properties and default values: https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#application-properties.data.spring.datasource.hikari ### `redis-lock` module: | Old | New | Default | | --- | --- | --- | | workflow.redis.locking.server.type | conductor.redis-lock.serverType | single | | workflow.redis.locking.server.address | conductor.redis-lock.serverAddress | redis://127.0.0.1:6379 | | workflow.redis.locking.server.password | conductor.redis-lock.serverPassword | null | | workflow.redis.locking.server.master.name | conductor.redis-lock.serverMasterName | master | | workflow.decider.locking.namespace | conductor.redis-lock.namespace | "" | | workflow.decider.locking.exceptions.ignore | conductor.redis-lock.ignoreLockingExceptions | false | ### `redis-persistence` module: | Old | New | Default | | --- | --- | --- | | EC2_REGION | conductor.redis.dataCenterRegion | us-east-1 | | EC2_AVAILABILITY_ZONE | conductor.redis.availabilityZone | us-east-1c | | workflow.dynomite.cluster | _removed_ | | workflow.dynomite.cluster.name | conductor.redis.clusterName | "" | | workflow.dynomite.cluster.hosts | conductor.redis.hosts | null | | workflow.namespace.prefix | conductor.redis.workflowNamespacePrefix | null | | workflow.namespace.queue.prefix | conductor.redis.queueNamespacePrefix | null | | workflow.dyno.keyspace.domain | conductor.redis.keyspaceDomain | null | | workflow.dynomite.connection.maxConnsPerHost | conductor.redis.maxConnectionsPerHost | 10 | | workflow.dynomite.connection.max.retry.attempt | conductor.redis.maxRetryAttempts | 0 | | workflow.dynomite.connection.max.timeout.exhausted.ms | conductor.redis.maxTimeoutWhenExhausted | 800ms | | queues.dynomite.nonQuorum.port | conductor.redis.queuesNonQuorumPort | 22122 | | workflow.dyno.queue.sharding.strategy | conductor.redis.queueShardingStrategy | roundRobin | | conductor.taskdef.cache.refresh.time.seconds | conductor.redis.taskDefCacheRefreshInterval | 60s | | workflow.event.execution.persistence.ttl.seconds | conductor.redis.eventExecutionPersistenceTTL | 60s | ### `zookeeper-lock` module: | Old | New | Default | | --- | --- | --- | | workflow.zookeeper.lock.connection | conductor.zookeeper-lock.connectionString | localhost:2181 | | workflow.zookeeper.lock.sessionTimeoutMs | conductor.zookeeper-lock.sessionTimeout | 60000ms | | workflow.zookeeper.lock.connectionTimeoutMs | conductor.zookeeper-lock.connectionTimeout | 15000ms | | workflow.decider.locking.namespace | conductor.zookeeper-lock.namespace | "" | ### Component configuration: | Old | New | Default | | --- | --- | --- | | db | conductor.db.type | "" | | workflow.indexing.enabled | conductor.indexing.enabled | true | | conductor.disable.async.workers | conductor.system-task-workers.enabled | true | | decider.sweep.disable | conductor.workflow-reconciler.enabled | true | | conductor.grpc.server.enabled | conductor.grpc-server.enabled | false | | workflow.external.payload.storage | conductor.external-payload-storage.type | dummy | | workflow.default.event.processor.enabled | conductor.default-event-processor.enabled | true | | workflow.events.default.queue.type | conductor.default-event-queue.type | sqs | | workflow.status.listener.type | conductor.workflow-status-listener.type | stub | | - | conductor.task-status-listener.type | stub | | workflow.decider.locking.server | conductor.workflow-execution-lock.type | noop_lock | | | | | | workflow.default.event.queue.enabled | conductor.event-queues.default.enabled | true | | workflow.sqs.event.queue.enabled | conductor.event-queues.sqs.enabled | false | | workflow.amqp.event.queue.enabled | conductor.event-queues.amqp.enabled | false | | workflow.nats.event.queue.enabled | conductor.event-queues.nats.enabled | false | | workflow.nats_stream.event.queue.enabled | conductor.event-queues.nats-stream.enabled | false | | | | | | - | conductor.metrics-logger.enabled | false | | - | conductor.metrics-prometheus.enabled | false | | - | conductor.metrics-datadog.enable | false | | - | conductor.metrics-datadog.api-key | | ================================================ FILE: CODE_OF_CONDUCT.md ================================================ [Code of Conduct](docs/docs/resources/code-of-conduct.md) ================================================ FILE: CONTRIBUTING.md ================================================ [Contributing](docs/docs/resources/contributing.md) ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} Netflix, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: OSSMETADATA ================================================ osslifecycle=active ================================================ FILE: README.md ================================================ ![Conductor](docs/docs/img/logo.png) ## Announcement > Effective **December 13, 2023**, Netflix will discontinue maintenance of Conductor OSS on GitHub. This strategic decision, while difficult, is essential for realigning our resources to better serve our business objectives with our internal Conductor fork. > > We are *deeply grateful* for your support and contributions over the years. While Netflix will no longer be maintaining this repo, members of the Conductor community have been active in promoting alternative forks of this project, we’ll leave the code as is and trust that the health of the community will remain strong and continue to develop moving forward. # Conductor [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/conductor.svg)]() [![Github release](https://img.shields.io/github/v/release/Netflix/conductor.svg)](https://GitHub.com/Netflix/conductor/releases) [![License](https://img.shields.io/github/license/Netflix/conductor.svg)](http://www.apache.org/licenses/LICENSE-2.0) [![GitHub stars](https://img.shields.io/github/stars/Netflix/conductor.svg?style=social&label=Star&maxAge=2592000)](https://GitHub.com/Netflix/conductor/stargazers/) [![GitHub forks](https://img.shields.io/github/forks/Netflix/conductor.svg?style=social&label=Fork&maxAge=2592000)](https://GitHub.com/Netflix/conductor/network/) Conductor is a platform created by Netflix to orchestrate workflows that span across microservices. ## Releases The final release is [![Github release](https://img.shields.io/github/v/release/Netflix/conductor.svg)](https://GitHub.com/Netflix/conductor/releases) ## Workflow Creation in Code Conductor supports creating workflows using JSON and Code. SDK support for creating workflows using code is available in multiple languages and can be found at https://github.com/conductor-sdk ## Community Contributions The modules contributed by the community are housed at [conductor-community](https://github.com/Netflix/conductor-community). Compatible versions of the community modules are released simultaneously with releases of the main modules. [Discussion Forum](https://github.com/Netflix/conductor/discussions): Please use the forum for questions and discussing ideas and join the community. [List of Conductor community projects](/docs/docs/resources/related.md): Backup tool, Cron like workflow starter, Docker containers and more. ## Getting Started - Building & Running Conductor ### Using Docker: The easiest way to get started is with Docker containers. Please follow the instructions [here](https://conductor.netflix.com/devguide/running/docker.html). ### From Source: Conductor Server is a [Spring Boot](https://spring.io/projects/spring-boot) project and follows all applicable conventions. See instructions [here](https://conductor.netflix.com/devguide/running/source.html). ## Published Artifacts Binaries are available from the [Maven Central Repository](https://search.maven.org/search?q=g:com.netflix.conductor). | Artifact | Description | |---------------------------------|-------------------------------------------------------------------------------------------------| | conductor-common | Common models used by various conductor modules | | conductor-core | Core Conductor module | | conductor-redis-persistence | Persistence and queue using Redis/Dynomite | | conductor-cassandra-persistence | Persistence using Cassandra | | conductor-es6-persistence | Indexing using Elasticsearch 6.X | | conductor-rest | Spring MVC resources for the core services | | conductor-ui | node.js based UI for Conductor | | conductor-client | Java client for Conductor that includes helpers for running worker tasks | | conductor-client-spring | Client starter kit for Spring | | conductor-java-sdk | SDK for writing workflows in code | | conductor-server | Spring Boot Web Application | | conductor-redis-lock | Workflow execution lock implementation using Redis | | conductor-awss3-storage | External payload storage implementation using AWS S3 | | conductor-awssqs-event-queue | Event queue implementation using AWS SQS | | conductor-http-task | Workflow system task implementation to send make requests | | conductor-json-jq-task | Workflow system task implementation to evaluate JSON using [jq](https://stedolan.github.io/jq/) | | conductor-grpc | Protobuf models used by the server and client | | conductor-grpc-client | gRPC client to interact with the gRPC server | | conductor-grpc-server | gRPC server Application | | conductor-test-harness | Integration and regression tests | ## Database Requirements * The default persistence used is Redis * The indexing backend is [Elasticsearch](https://www.elastic.co/) (6.x) ## Other Requirements * JDK 17+ * UI requires Node 14 to build. Earlier Node versions may work but is untested. ## Get Support There are several ways to get in touch with us: * [Slack Community](https://join.slack.com/t/orkes-conductor/shared_invite/zt-xyxqyseb-YZ3hwwAgHJH97bsrYRnSZg) * [GitHub Discussion Forum](https://github.com/Netflix/conductor/discussions) ## Contributions Whether it is a small documentation correction, bug fix or a new feature, contributions are highly appreciated. We just ask you to follow standard OSS guidelines. The [Discussion Forum](https://github.com/Netflix/conductor/discussions) is a good place to ask questions, discuss new features and explore ideas. Please check with us before spending too much time, only to find out later that someone else is already working on a similar feature. `main` branch is the current working branch. Please send your PR's to `main` branch, making sure that it builds on your local system successfully. Also, please make sure all the conflicts are resolved. ## License Copyright 2022 Netflix, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: RELATED.md ================================================ [Related Projects](docs/docs/resources/related.md) ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions | Version | Supported | | ------- | ------------------ | | 3.x.x | :white_check_mark: | | 2.x.x | :x: | | 1.x.x | :x: | ## Reporting a Vulnerability Please email conductor@netflix.com to report vulnerabilities. ================================================ FILE: USERS.md ================================================ ## Who uses Conductor? We would like to keep track of whose using Conductor. Please send a pull request with your company name and Github handle. * [Netflix](https://www.netflix.com/) [[@aravindanr](https://github.com/aravindanr)] * [Florida Blue](http://bcbsfl.com/) [[@rickfish](https://github.com/rickfish)] * [UWM](https://www.uwm.com/) [[@zergrushjoe](https://github.com/ZergRushJoe)] * [Deutsche Telekom Digital Labs](https://dtdl.in) [[@jas34](https://github.com/jas34)] [[@deoramanas](https://github.com/deoramanas)] * [VMware](https://www.vmware.com/) [[@taojwmware](https://github.com/taojwmware)] [[@venkag](https://github.com/venkag)] * [JP Morgan Chase](https://www.chase.com/) [[@maheshyaddanapudi](https://github.com/maheshyaddanapudi)] * [Orkes](https://orkes.io/) [[@CherishSantoshi](https://github.com/CherishSantoshi)] * [313X](https://313x.com.br) [[@dalmoveras](https://github.com/dalmoveras)] * [Supercharge](https://supercharge.io) [[@team-supercharge](https://github.com/team-supercharge)] * [GE Healthcare](https://www.gehealthcare.com/) [[@flavioschuindt](https://github.com/flavioschuindt)] * [ReliaQuest](https://www.reliaquest.com/) [[@rq-dbrady](https://github.com/rq-dbrady)] [[@alexmay48](https://github.com/alexmay48)] * [Clari](https://www.clari.com/) [[@TeamJOF](https://github.com/clari)] * [Atlassian](https://www.atlassian.com/) [[@LuisLainez](https://github.com/LuisLainez)] [[@aradu](https://github.com/aradu-atlassian)] ================================================ FILE: annotations/README.md ================================================ # Annotations - `protogen` Annotations - Original Author: Vicent Martí - https://github.com/vmg - Original Repo: https://github.com/vmg/protogen ================================================ FILE: annotations/build.gradle ================================================ dependencies { } ================================================ FILE: annotations/src/main/java/com/netflix/conductor/annotations/protogen/ProtoEnum.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations.protogen; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * ProtoEnum annotates an enum type that will be exposed via the GRPC API as a native Protocol * Buffers enum. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface ProtoEnum {} ================================================ FILE: annotations/src/main/java/com/netflix/conductor/annotations/protogen/ProtoField.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations.protogen; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * ProtoField annotates a field inside an struct with metadata on how to expose it on its * corresponding Protocol Buffers struct. For a field to be exposed in a ProtoBuf struct, the * containing struct must also be annotated with a {@link ProtoMessage} or {@link ProtoEnum} tag. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) public @interface ProtoField { /** * Mandatory. Sets the Protocol Buffer ID for this specific field. Once a field has been * annotated with a given ID, the ID can never change to a different value or the resulting * Protocol Buffer struct will not be backwards compatible. * * @return the numeric ID for the field */ int id(); } ================================================ FILE: annotations/src/main/java/com/netflix/conductor/annotations/protogen/ProtoMessage.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations.protogen; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * ProtoMessage annotates a given Java class so it becomes exposed via the GRPC API as a native * Protocol Buffers struct. The annotated class must be a POJO. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface ProtoMessage { /** * Sets whether the generated mapping code will contain a helper to translate the POJO for this * class into the equivalent ProtoBuf object. * * @return whether this class will generate a mapper to ProtoBuf objects */ boolean toProto() default true; /** * Sets whether the generated mapping code will contain a helper to translate the ProtoBuf * object for this class into the equivalent POJO. * * @return whether this class will generate a mapper from ProtoBuf objects */ boolean fromProto() default true; /** * Sets whether this is a wrapper class that will be used to encapsulate complex nested type * interfaces. Wrapper classes are not directly exposed by the ProtoBuf API and must be mapped * manually. * * @return whether this is a wrapper class */ boolean wrapper() default false; } ================================================ FILE: annotations-processor/README.md ================================================ [Annotations Processor](docs/docs/reference-docs/annotations-processor.md) ================================================ FILE: annotations-processor/build.gradle ================================================ sourceSets { example } dependencies { implementation project(':conductor-annotations') api 'com.google.guava:guava:32.1.2-jre' api 'com.squareup:javapoet:1.13.+' api 'com.github.jknack:handlebars:4.3.+' api 'com.google.protobuf:protobuf-java:3.21.12' api 'javax.annotation:javax.annotation-api:1.3.2' api gradleApi() exampleImplementation sourceSets.main.output exampleImplementation project(':conductor-annotations') } task exampleJar(type: Jar) { archiveFileName = 'example.jar' from sourceSets.example.output.classesDirs } testClasses.finalizedBy(exampleJar) ================================================ FILE: annotations-processor/src/example/java/com/example/Example.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.example; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; @ProtoMessage public class Example { @ProtoField(id = 1) public String name; @ProtoField(id = 2) public Long count; } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/AbstractMessage.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.util.ArrayList; import java.util.List; import java.util.Set; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.annotationsprocessor.protogen.types.MessageType; import com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeSpec; public abstract class AbstractMessage { protected Class clazz; protected MessageType type; protected List fields = new ArrayList(); protected List nested = new ArrayList<>(); public AbstractMessage(Class cls, MessageType parentType) { assert cls.isAnnotationPresent(ProtoMessage.class) || cls.isAnnotationPresent(ProtoEnum.class); this.clazz = cls; this.type = TypeMapper.INSTANCE.declare(cls, parentType); for (Class nested : clazz.getDeclaredClasses()) { if (nested.isEnum()) addNestedEnum(nested); else addNestedClass(nested); } } private void addNestedEnum(Class cls) { ProtoEnum ann = (ProtoEnum) cls.getAnnotation(ProtoEnum.class); if (ann != null) { nested.add(new Enum(cls, this.type)); } } private void addNestedClass(Class cls) { ProtoMessage ann = (ProtoMessage) cls.getAnnotation(ProtoMessage.class); if (ann != null) { nested.add(new Message(cls, this.type)); } } public abstract String getProtoClass(); protected abstract void javaMapToProto(TypeSpec.Builder builder); protected abstract void javaMapFromProto(TypeSpec.Builder builder); public void generateJavaMapper(TypeSpec.Builder builder) { javaMapToProto(builder); javaMapFromProto(builder); for (AbstractMessage abstractMessage : this.nested) { abstractMessage.generateJavaMapper(builder); } } public void generateAbstractMethods(Set specs) { for (Field field : fields) { field.generateAbstractMethods(specs); } for (AbstractMessage elem : nested) { elem.generateAbstractMethods(specs); } } public void findDependencies(Set dependencies) { for (Field field : fields) { field.getDependencies(dependencies); } for (AbstractMessage elem : nested) { elem.findDependencies(dependencies); } } public List getNested() { return nested; } public List getFields() { return fields; } public String getName() { return clazz.getSimpleName(); } public abstract static class Field { protected int protoIndex; protected java.lang.reflect.Field field; protected Field(int index, java.lang.reflect.Field field) { this.protoIndex = index; this.field = field; } public abstract String getProtoTypeDeclaration(); public int getProtoIndex() { return protoIndex; } public String getName() { return field.getName(); } public String getProtoName() { return field.getName().toUpperCase(); } public void getDependencies(Set deps) {} public void generateAbstractMethods(Set specs) {} } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/Enum.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import javax.lang.model.element.Modifier; import com.netflix.conductor.annotationsprocessor.protogen.types.MessageType; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; public class Enum extends AbstractMessage { public enum MapType { FROM_PROTO("fromProto"), TO_PROTO("toProto"); private final String methodName; MapType(String m) { methodName = m; } public String getMethodName() { return methodName; } } public Enum(Class cls, MessageType parent) { super(cls, parent); int protoIndex = 0; for (java.lang.reflect.Field field : cls.getDeclaredFields()) { if (field.isEnumConstant()) fields.add(new EnumField(protoIndex++, field)); } } @Override public String getProtoClass() { return "enum"; } private MethodSpec javaMap(MapType mt, TypeName from, TypeName to) { MethodSpec.Builder method = MethodSpec.methodBuilder(mt.getMethodName()); method.addModifiers(Modifier.PUBLIC); method.returns(to); method.addParameter(from, "from"); method.addStatement("$T to", to); method.beginControlFlow("switch (from)"); for (Field field : fields) { String fromName = (mt == MapType.TO_PROTO) ? field.getName() : field.getProtoName(); String toName = (mt == MapType.TO_PROTO) ? field.getProtoName() : field.getName(); method.addStatement("case $L: to = $T.$L; break", fromName, to, toName); } method.addStatement( "default: throw new $T(\"Unexpected enum constant: \" + from)", IllegalArgumentException.class); method.endControlFlow(); method.addStatement("return to"); return method.build(); } @Override protected void javaMapFromProto(TypeSpec.Builder type) { type.addMethod( javaMap( MapType.FROM_PROTO, this.type.getJavaProtoType(), TypeName.get(this.clazz))); } @Override protected void javaMapToProto(TypeSpec.Builder type) { type.addMethod( javaMap(MapType.TO_PROTO, TypeName.get(this.clazz), this.type.getJavaProtoType())); } public class EnumField extends Field { protected EnumField(int index, java.lang.reflect.Field field) { super(index, field); } @Override public String getProtoTypeDeclaration() { return String.format("%s = %d", getProtoName(), getProtoIndex()); } } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/Message.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.lang.model.element.Modifier; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.annotationsprocessor.protogen.types.AbstractType; import com.netflix.conductor.annotationsprocessor.protogen.types.MessageType; import com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeSpec; public class Message extends AbstractMessage { public Message(Class cls, MessageType parent) { super(cls, parent); for (java.lang.reflect.Field field : clazz.getDeclaredFields()) { ProtoField ann = field.getAnnotation(ProtoField.class); if (ann == null) continue; fields.add(new MessageField(ann.id(), field)); } } protected ProtoMessage getAnnotation() { return (ProtoMessage) this.clazz.getAnnotation(ProtoMessage.class); } @Override public String getProtoClass() { return "message"; } @Override protected void javaMapToProto(TypeSpec.Builder type) { if (!getAnnotation().toProto() || getAnnotation().wrapper()) return; ClassName javaProtoType = (ClassName) this.type.getJavaProtoType(); MethodSpec.Builder method = MethodSpec.methodBuilder("toProto"); method.addModifiers(Modifier.PUBLIC); method.returns(javaProtoType); method.addParameter(this.clazz, "from"); method.addStatement( "$T to = $T.newBuilder()", javaProtoType.nestedClass("Builder"), javaProtoType); for (Field field : this.fields) { if (field instanceof MessageField) { AbstractType fieldType = ((MessageField) field).getAbstractType(); fieldType.mapToProto(field.getName(), method); } } method.addStatement("return to.build()"); type.addMethod(method.build()); } @Override protected void javaMapFromProto(TypeSpec.Builder type) { if (!getAnnotation().fromProto() || getAnnotation().wrapper()) return; MethodSpec.Builder method = MethodSpec.methodBuilder("fromProto"); method.addModifiers(Modifier.PUBLIC); method.returns(this.clazz); method.addParameter(this.type.getJavaProtoType(), "from"); method.addStatement("$T to = new $T()", this.clazz, this.clazz); for (Field field : this.fields) { if (field instanceof MessageField) { AbstractType fieldType = ((MessageField) field).getAbstractType(); fieldType.mapFromProto(field.getName(), method); } } method.addStatement("return to"); type.addMethod(method.build()); } public static class MessageField extends Field { protected AbstractType type; protected MessageField(int index, java.lang.reflect.Field field) { super(index, field); } public AbstractType getAbstractType() { if (type == null) { type = TypeMapper.INSTANCE.get(field.getGenericType()); } return type; } private static Pattern CAMEL_CASE_RE = Pattern.compile("(?<=[a-z])[A-Z]"); private static String toUnderscoreCase(String input) { Matcher m = CAMEL_CASE_RE.matcher(input); StringBuilder sb = new StringBuilder(); while (m.find()) { m.appendReplacement(sb, "_" + m.group()); } m.appendTail(sb); return sb.toString().toLowerCase(); } @Override public String getProtoTypeDeclaration() { return String.format( "%s %s = %d", getAbstractType().getProtoType(), toUnderscoreCase(getName()), getProtoIndex()); } @Override public void getDependencies(Set deps) { getAbstractType().getDependencies(deps); } @Override public void generateAbstractMethods(Set specs) { getAbstractType().generateAbstractMethods(specs); } } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/ProtoFile.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.util.HashSet; import java.util.Set; import com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper; import com.squareup.javapoet.ClassName; public class ProtoFile { public static String PROTO_SUFFIX = "Pb"; private ClassName baseClass; private AbstractMessage message; private String filePath; private String protoPackageName; private String javaPackageName; private String goPackageName; public ProtoFile( Class object, String protoPackageName, String javaPackageName, String goPackageName) { this.protoPackageName = protoPackageName; this.javaPackageName = javaPackageName; this.goPackageName = goPackageName; String className = object.getSimpleName() + PROTO_SUFFIX; this.filePath = "model/" + object.getSimpleName().toLowerCase() + ".proto"; this.baseClass = ClassName.get(this.javaPackageName, className); this.message = new Message(object, TypeMapper.INSTANCE.baseClass(baseClass, filePath)); } public String getJavaClassName() { return baseClass.simpleName(); } public String getFilePath() { return filePath; } public String getProtoPackageName() { return protoPackageName; } public String getJavaPackageName() { return javaPackageName; } public String getGoPackageName() { return goPackageName; } public AbstractMessage getMessage() { return message; } public Set getIncludes() { Set includes = new HashSet<>(); message.findDependencies(includes); includes.remove(this.getFilePath()); return includes; } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/ProtoGen.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.net.URL; import java.net.URLClassLoader; import java.util.*; import javax.annotation.Generated; import javax.lang.model.element.Modifier; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.github.jknack.handlebars.EscapingStrategy; import com.github.jknack.handlebars.Handlebars; import com.github.jknack.handlebars.Template; import com.github.jknack.handlebars.io.ClassPathTemplateLoader; import com.github.jknack.handlebars.io.TemplateLoader; import com.google.common.reflect.ClassPath; import com.squareup.javapoet.AnnotationSpec; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeSpec; public class ProtoGen { private static final String GENERATOR_NAME = "com.netflix.conductor.annotationsprocessor.protogen"; private String protoPackageName; private String javaPackageName; private String goPackageName; private List protoFiles = new ArrayList<>(); public ProtoGen(String protoPackageName, String javaPackageName, String goPackageName) { this.protoPackageName = protoPackageName; this.javaPackageName = javaPackageName; this.goPackageName = goPackageName; } public void writeMapper(File root, String mapperPackageName) throws IOException { TypeSpec.Builder protoMapper = TypeSpec.classBuilder("AbstractProtoMapper") .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT) .addAnnotation( AnnotationSpec.builder(Generated.class) .addMember("value", "$S", GENERATOR_NAME) .build()); Set abstractMethods = new HashSet<>(); protoFiles.sort( new Comparator() { public int compare(ProtoFile p1, ProtoFile p2) { String n1 = p1.getMessage().getName(); String n2 = p2.getMessage().getName(); return n1.compareTo(n2); } }); for (ProtoFile protoFile : protoFiles) { AbstractMessage elem = protoFile.getMessage(); elem.generateJavaMapper(protoMapper); elem.generateAbstractMethods(abstractMethods); } protoMapper.addMethods(abstractMethods); JavaFile javaFile = JavaFile.builder(mapperPackageName, protoMapper.build()).indent(" ").build(); File filename = new File(root, "AbstractProtoMapper.java"); try (Writer writer = new FileWriter(filename.toString())) { System.out.printf("protogen: writing '%s'...\n", filename); javaFile.writeTo(writer); } } public void writeProtos(File root) throws IOException { TemplateLoader loader = new ClassPathTemplateLoader("/templates", ".proto"); Handlebars handlebars = new Handlebars(loader) .infiniteLoops(true) .prettyPrint(true) .with(EscapingStrategy.NOOP); Template protoFile = handlebars.compile("file"); for (ProtoFile file : protoFiles) { File filename = new File(root, file.getFilePath()); try (Writer writer = new FileWriter(filename)) { System.out.printf("protogen: writing '%s'...\n", filename); protoFile.apply(file, writer); } } } public void processPackage(File jarFile, String packageName) throws IOException { if (!jarFile.isFile()) throw new IOException("missing Jar file " + jarFile); URL[] urls = new URL[] {jarFile.toURI().toURL()}; ClassLoader loader = new URLClassLoader(urls, Thread.currentThread().getContextClassLoader()); ClassPath cp = ClassPath.from(loader); System.out.printf("protogen: processing Jar '%s'\n", jarFile); for (ClassPath.ClassInfo info : cp.getTopLevelClassesRecursive(packageName)) { try { processClass(info.load()); } catch (NoClassDefFoundError ignored) { } } } public void processClass(Class obj) { if (obj.isAnnotationPresent(ProtoMessage.class)) { System.out.printf("protogen: found %s\n", obj.getCanonicalName()); protoFiles.add(new ProtoFile(obj, protoPackageName, javaPackageName, goPackageName)); } } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/ProtoGenTask.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.io.File; import java.io.IOException; public class ProtoGenTask { private String protoPackage; private String javaPackage; private String goPackage; private File protosDir; private File mapperDir; private String mapperPackage; private File sourceJar; private String sourcePackage; public String getProtoPackage() { return protoPackage; } public void setProtoPackage(String protoPackage) { this.protoPackage = protoPackage; } public String getJavaPackage() { return javaPackage; } public void setJavaPackage(String javaPackage) { this.javaPackage = javaPackage; } public String getGoPackage() { return goPackage; } public void setGoPackage(String goPackage) { this.goPackage = goPackage; } public File getProtosDir() { return protosDir; } public void setProtosDir(File protosDir) { this.protosDir = protosDir; } public File getMapperDir() { return mapperDir; } public void setMapperDir(File mapperDir) { this.mapperDir = mapperDir; } public String getMapperPackage() { return mapperPackage; } public void setMapperPackage(String mapperPackage) { this.mapperPackage = mapperPackage; } public File getSourceJar() { return sourceJar; } public void setSourceJar(File sourceJar) { this.sourceJar = sourceJar; } public String getSourcePackage() { return sourcePackage; } public void setSourcePackage(String sourcePackage) { this.sourcePackage = sourcePackage; } public void generate() { ProtoGen generator = new ProtoGen(protoPackage, javaPackage, goPackage); try { generator.processPackage(sourceJar, sourcePackage); generator.writeMapper(mapperDir, mapperPackage); generator.writeProtos(protosDir); } catch (IOException e) { System.err.printf("protogen: failed with %s\n", e); } } public static void main(String[] args) { if (args == null || args.length < 8) { throw new RuntimeException( "protogen configuration incomplete, please provide all required (8) inputs"); } ProtoGenTask task = new ProtoGenTask(); int argsId = 0; task.setProtoPackage(args[argsId++]); task.setJavaPackage(args[argsId++]); task.setGoPackage(args[argsId++]); task.setProtosDir(new File(args[argsId++])); task.setMapperDir(new File(args[argsId++])); task.setMapperPackage(args[argsId++]); task.setSourceJar(new File(args[argsId++])); task.setSourcePackage(args[argsId]); System.out.println("Running protogen with arguments: " + task); task.generate(); System.out.println("protogen completed."); } @Override public String toString() { return "ProtoGenTask{" + "protoPackage='" + protoPackage + '\'' + ", javaPackage='" + javaPackage + '\'' + ", goPackage='" + goPackage + '\'' + ", protosDir=" + protosDir + ", mapperDir=" + mapperDir + ", mapperPackage='" + mapperPackage + '\'' + ", sourceJar=" + sourceJar + ", sourcePackage='" + sourcePackage + '\'' + '}'; } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/AbstractType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.Set; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; public abstract class AbstractType { Type javaType; TypeName javaProtoType; AbstractType(Type javaType, TypeName javaProtoType) { this.javaType = javaType; this.javaProtoType = javaProtoType; } public Type getJavaType() { return javaType; } public TypeName getJavaProtoType() { return javaProtoType; } public abstract String getProtoType(); public abstract TypeName getRawJavaType(); public abstract void mapToProto(String field, MethodSpec.Builder method); public abstract void mapFromProto(String field, MethodSpec.Builder method); public abstract void getDependencies(Set deps); public abstract void generateAbstractMethods(Set specs); protected String javaMethodName(String m, String field) { String fieldName = field.substring(0, 1).toUpperCase() + field.substring(1); return m + fieldName; } private static class ProtoCase { static String convert(String s) { StringBuilder out = new StringBuilder(s.length()); final int len = s.length(); int i = 0; int j = -1; while ((j = findWordBoundary(s, ++j)) != -1) { out.append(normalizeWord(s.substring(i, j))); if (j < len && s.charAt(j) == '_') j++; i = j; } if (i == 0) return normalizeWord(s); if (i < len) out.append(normalizeWord(s.substring(i))); return out.toString(); } private static boolean isWordBoundary(char c) { return (c >= 'A' && c <= 'Z'); } private static int findWordBoundary(CharSequence sequence, int start) { int length = sequence.length(); if (start >= length) return -1; if (isWordBoundary(sequence.charAt(start))) { int i = start; while (i < length && isWordBoundary(sequence.charAt(i))) i++; return i; } else { for (int i = start; i < length; i++) { final char c = sequence.charAt(i); if (c == '_' || isWordBoundary(c)) return i; } return -1; } } private static String normalizeWord(String word) { if (word.length() < 2) return word.toUpperCase(); return word.substring(0, 1).toUpperCase() + word.substring(1).toLowerCase(); } } protected String protoMethodName(String m, String field) { return m + ProtoCase.convert(field); } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/ExternMessageType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.Set; import javax.lang.model.element.Modifier; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; public class ExternMessageType extends MessageType { private String externProtoType; public ExternMessageType( Type javaType, ClassName javaProtoType, String externProtoType, String protoFilePath) { super(javaType, javaProtoType, protoFilePath); this.externProtoType = externProtoType; } @Override public String getProtoType() { return externProtoType; } @Override public void generateAbstractMethods(Set specs) { MethodSpec fromProto = MethodSpec.methodBuilder("fromProto") .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT) .returns(this.getJavaType()) .addParameter(this.getJavaProtoType(), "in") .build(); MethodSpec toProto = MethodSpec.methodBuilder("toProto") .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT) .returns(this.getJavaProtoType()) .addParameter(this.getJavaType(), "in") .build(); specs.add(fromProto); specs.add(toProto); } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/GenericType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.Set; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; abstract class GenericType extends AbstractType { public GenericType(Type type) { super(type, null); } protected Class getRawType() { ParameterizedType tt = (ParameterizedType) this.getJavaType(); return (Class) tt.getRawType(); } protected AbstractType resolveGenericParam(int idx) { ParameterizedType tt = (ParameterizedType) this.getJavaType(); Type[] types = tt.getActualTypeArguments(); AbstractType abstractType = TypeMapper.INSTANCE.get(types[idx]); if (abstractType instanceof GenericType) { return WrappedType.wrap((GenericType) abstractType); } return abstractType; } public abstract String getWrapperSuffix(); public abstract AbstractType getValueType(); public abstract TypeName resolveJavaProtoType(); @Override public TypeName getRawJavaType() { return ClassName.get(getRawType()); } @Override public void getDependencies(Set deps) { getValueType().getDependencies(deps); } @Override public void generateAbstractMethods(Set specs) { getValueType().generateAbstractMethods(specs); } @Override public TypeName getJavaProtoType() { if (javaProtoType == null) { javaProtoType = resolveJavaProtoType(); } return javaProtoType; } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/ListType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.stream.Collectors; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeName; public class ListType extends GenericType { private AbstractType valueType; public ListType(Type type) { super(type); } @Override public String getWrapperSuffix() { return "List"; } @Override public AbstractType getValueType() { if (valueType == null) { valueType = resolveGenericParam(0); } return valueType; } @Override public void mapToProto(String field, MethodSpec.Builder method) { AbstractType subtype = getValueType(); if (subtype instanceof ScalarType) { method.addStatement( "to.$L( from.$L() )", protoMethodName("addAll", field), javaMethodName("get", field)); } else { method.beginControlFlow( "for ($T elem : from.$L())", subtype.getJavaType(), javaMethodName("get", field)); method.addStatement("to.$L( toProto(elem) )", protoMethodName("add", field)); method.endControlFlow(); } } @Override public void mapFromProto(String field, MethodSpec.Builder method) { AbstractType subtype = getValueType(); Type entryType = subtype.getJavaType(); Class collector = TypeMapper.PROTO_LIST_TYPES.get(getRawType()); if (subtype instanceof ScalarType) { if (entryType.equals(String.class)) { method.addStatement( "to.$L( from.$L().stream().collect($T.toCollection($T::new)) )", javaMethodName("set", field), protoMethodName("get", field) + "List", Collectors.class, collector); } else { method.addStatement( "to.$L( from.$L() )", javaMethodName("set", field), protoMethodName("get", field) + "List"); } } else { method.addStatement( "to.$L( from.$L().stream().map(this::fromProto).collect($T.toCollection($T::new)) )", javaMethodName("set", field), protoMethodName("get", field) + "List", Collectors.class, collector); } } @Override public TypeName resolveJavaProtoType() { return ParameterizedTypeName.get( (ClassName) getRawJavaType(), getValueType().getJavaProtoType()); } @Override public String getProtoType() { return "repeated " + getValueType().getProtoType(); } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/MapType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeName; public class MapType extends GenericType { private AbstractType keyType; private AbstractType valueType; public MapType(Type type) { super(type); } @Override public String getWrapperSuffix() { return "Map"; } @Override public AbstractType getValueType() { if (valueType == null) { valueType = resolveGenericParam(1); } return valueType; } public AbstractType getKeyType() { if (keyType == null) { keyType = resolveGenericParam(0); } return keyType; } @Override public void mapToProto(String field, MethodSpec.Builder method) { AbstractType valueType = getValueType(); if (valueType instanceof ScalarType) { method.addStatement( "to.$L( from.$L() )", protoMethodName("putAll", field), javaMethodName("get", field)); } else { TypeName typeName = ParameterizedTypeName.get( Map.Entry.class, getKeyType().getJavaType(), getValueType().getJavaType()); method.beginControlFlow( "for ($T pair : from.$L().entrySet())", typeName, javaMethodName("get", field)); method.addStatement( "to.$L( pair.getKey(), toProto( pair.getValue() ) )", protoMethodName("put", field)); method.endControlFlow(); } } @Override public void mapFromProto(String field, MethodSpec.Builder method) { AbstractType valueType = getValueType(); if (valueType instanceof ScalarType) { method.addStatement( "to.$L( from.$L() )", javaMethodName("set", field), protoMethodName("get", field) + "Map"); } else { Type keyType = getKeyType().getJavaType(); Type valueTypeJava = getValueType().getJavaType(); TypeName valueTypePb = getValueType().getJavaProtoType(); ParameterizedTypeName entryType = ParameterizedTypeName.get( ClassName.get(Map.Entry.class), TypeName.get(keyType), valueTypePb); ParameterizedTypeName mapType = ParameterizedTypeName.get(Map.class, keyType, valueTypeJava); ParameterizedTypeName hashMapType = ParameterizedTypeName.get(HashMap.class, keyType, valueTypeJava); String mapName = field + "Map"; method.addStatement("$T $L = new $T()", mapType, mapName, hashMapType); method.beginControlFlow( "for ($T pair : from.$L().entrySet())", entryType, protoMethodName("get", field) + "Map"); method.addStatement("$L.put( pair.getKey(), fromProto( pair.getValue() ) )", mapName); method.endControlFlow(); method.addStatement("to.$L($L)", javaMethodName("set", field), mapName); } } @Override public TypeName resolveJavaProtoType() { return ParameterizedTypeName.get( (ClassName) getRawJavaType(), getKeyType().getJavaProtoType(), getValueType().getJavaProtoType()); } @Override public String getProtoType() { AbstractType keyType = getKeyType(); AbstractType valueType = getValueType(); if (!(keyType instanceof ScalarType)) { throw new IllegalArgumentException( "cannot map non-scalar map key: " + this.getJavaType()); } return String.format("map<%s, %s>", keyType.getProtoType(), valueType.getProtoType()); } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/MessageType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.List; import java.util.Set; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; public class MessageType extends AbstractType { private String protoFilePath; public MessageType(Type javaType, ClassName javaProtoType, String protoFilePath) { super(javaType, javaProtoType); this.protoFilePath = protoFilePath; } @Override public String getProtoType() { List classes = ((ClassName) getJavaProtoType()).simpleNames(); return String.join(".", classes.subList(1, classes.size())); } public String getProtoFilePath() { return protoFilePath; } @Override public TypeName getRawJavaType() { return getJavaProtoType(); } @Override public void mapToProto(String field, MethodSpec.Builder method) { final String getter = javaMethodName("get", field); method.beginControlFlow("if (from.$L() != null)", getter); method.addStatement("to.$L( toProto( from.$L() ) )", protoMethodName("set", field), getter); method.endControlFlow(); } private boolean isEnum() { Type clazz = getJavaType(); return (clazz instanceof Class) && ((Class) clazz).isEnum(); } @Override public void mapFromProto(String field, MethodSpec.Builder method) { if (!isEnum()) method.beginControlFlow("if (from.$L())", protoMethodName("has", field)); method.addStatement( "to.$L( fromProto( from.$L() ) )", javaMethodName("set", field), protoMethodName("get", field)); if (!isEnum()) method.endControlFlow(); } @Override public void getDependencies(Set deps) { deps.add(protoFilePath); } @Override public void generateAbstractMethods(Set specs) {} } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/ScalarType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.Set; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; public class ScalarType extends AbstractType { private String protoType; public ScalarType(Type javaType, TypeName javaProtoType, String protoType) { super(javaType, javaProtoType); this.protoType = protoType; } @Override public String getProtoType() { return protoType; } @Override public TypeName getRawJavaType() { return getJavaProtoType(); } @Override public void mapFromProto(String field, MethodSpec.Builder method) { method.addStatement( "to.$L( from.$L() )", javaMethodName("set", field), protoMethodName("get", field)); } private boolean isNullableType() { final Type jt = getJavaType(); return jt.equals(Boolean.class) || jt.equals(Byte.class) || jt.equals(Character.class) || jt.equals(Short.class) || jt.equals(Integer.class) || jt.equals(Long.class) || jt.equals(Double.class) || jt.equals(Float.class) || jt.equals(String.class); } @Override public void mapToProto(String field, MethodSpec.Builder method) { final boolean nullable = isNullableType(); String getter = (getJavaType().equals(boolean.class) || getJavaType().equals(Boolean.class)) ? javaMethodName("is", field) : javaMethodName("get", field); if (nullable) method.beginControlFlow("if (from.$L() != null)", getter); method.addStatement("to.$L( from.$L() )", protoMethodName("set", field), getter); if (nullable) method.endControlFlow(); } @Override public void getDependencies(Set deps) {} @Override public void generateAbstractMethods(Set specs) {} } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/TypeMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.*; import com.google.protobuf.Any; import com.squareup.javapoet.ClassName; import com.squareup.javapoet.TypeName; public class TypeMapper { static Map PROTO_LIST_TYPES = new HashMap<>(); static { PROTO_LIST_TYPES.put(List.class, ArrayList.class); PROTO_LIST_TYPES.put(Set.class, HashSet.class); PROTO_LIST_TYPES.put(LinkedList.class, LinkedList.class); } public static TypeMapper INSTANCE = new TypeMapper(); private Map types = new HashMap<>(); public void addScalarType(Type t, String protoType) { types.put(t, new ScalarType(t, TypeName.get(t), protoType)); } public void addMessageType(Class t, MessageType message) { types.put(t, message); } public TypeMapper() { addScalarType(int.class, "int32"); addScalarType(Integer.class, "int32"); addScalarType(long.class, "int64"); addScalarType(Long.class, "int64"); addScalarType(String.class, "string"); addScalarType(boolean.class, "bool"); addScalarType(Boolean.class, "bool"); addMessageType( Object.class, new ExternMessageType( Object.class, ClassName.get("com.google.protobuf", "Value"), "google.protobuf.Value", "google/protobuf/struct.proto")); addMessageType( Any.class, new ExternMessageType( Any.class, ClassName.get(Any.class), "google.protobuf.Any", "google/protobuf/any.proto")); } public AbstractType get(Type t) { if (!types.containsKey(t)) { if (t instanceof ParameterizedType) { Type raw = ((ParameterizedType) t).getRawType(); if (PROTO_LIST_TYPES.containsKey(raw)) { types.put(t, new ListType(t)); } else if (raw.equals(Map.class)) { types.put(t, new MapType(t)); } } } if (!types.containsKey(t)) { throw new IllegalArgumentException("Cannot map type: " + t); } return types.get(t); } public MessageType get(String className) { for (Map.Entry pair : types.entrySet()) { AbstractType t = pair.getValue(); if (t instanceof MessageType) { if (((Class) t.getJavaType()).getSimpleName().equals(className)) return (MessageType) t; } } return null; } public MessageType declare(Class type, MessageType parent) { return declare(type, (ClassName) parent.getJavaProtoType(), parent.getProtoFilePath()); } public MessageType declare(Class type, ClassName parentType, String protoFilePath) { String simpleName = type.getSimpleName(); MessageType t = new MessageType(type, parentType.nestedClass(simpleName), protoFilePath); if (types.containsKey(type)) { throw new IllegalArgumentException("duplicate type declaration: " + type); } types.put(type, t); return t; } public MessageType baseClass(ClassName className, String protoFilePath) { return new MessageType(Object.class, className, protoFilePath); } } ================================================ FILE: annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/WrappedType.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen.types; import java.lang.reflect.Type; import java.util.Set; import javax.lang.model.element.Modifier; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; public class WrappedType extends AbstractType { private AbstractType realType; private MessageType wrappedType; public static WrappedType wrap(GenericType realType) { Type valueType = realType.getValueType().getJavaType(); if (!(valueType instanceof Class)) throw new IllegalArgumentException("cannot wrap primitive type: " + valueType); String className = ((Class) valueType).getSimpleName() + realType.getWrapperSuffix(); MessageType wrappedType = TypeMapper.INSTANCE.get(className); if (wrappedType == null) throw new IllegalArgumentException("missing wrapper class: " + className); return new WrappedType(realType, wrappedType); } public WrappedType(AbstractType realType, MessageType wrappedType) { super(realType.getJavaType(), wrappedType.getJavaProtoType()); this.realType = realType; this.wrappedType = wrappedType; } @Override public String getProtoType() { return wrappedType.getProtoType(); } @Override public TypeName getRawJavaType() { return realType.getRawJavaType(); } @Override public void mapToProto(String field, MethodSpec.Builder method) { wrappedType.mapToProto(field, method); } @Override public void mapFromProto(String field, MethodSpec.Builder method) { wrappedType.mapFromProto(field, method); } @Override public void getDependencies(Set deps) { this.realType.getDependencies(deps); this.wrappedType.getDependencies(deps); } @Override public void generateAbstractMethods(Set specs) { MethodSpec fromProto = MethodSpec.methodBuilder("fromProto") .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT) .returns(this.realType.getJavaType()) .addParameter(this.wrappedType.getJavaProtoType(), "in") .build(); MethodSpec toProto = MethodSpec.methodBuilder("toProto") .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT) .returns(this.wrappedType.getJavaProtoType()) .addParameter(this.realType.getJavaType(), "in") .build(); specs.add(fromProto); specs.add(toProto); } } ================================================ FILE: annotations-processor/src/main/resources/templates/file.proto ================================================ syntax = "proto3"; package {{protoPackageName}}; {{#includes}} import "{{this}}"; {{/includes}} option java_package = "{{javaPackageName}}"; option java_outer_classname = "{{javaClassName}}"; option go_package = "{{goPackageName}}"; {{#message}} {{>message}} {{/message}} ================================================ FILE: annotations-processor/src/main/resources/templates/message.proto ================================================ {{protoClass}} {{name}} { {{#nested}} {{>message}} {{/nested}} {{#fields}} {{protoTypeDeclaration}}; {{/fields}} } ================================================ FILE: annotations-processor/src/test/java/com/netflix/conductor/annotationsprocessor/protogen/ProtoGenTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotationsprocessor.protogen; import java.io.File; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.List; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import com.google.common.collect.Lists; import com.google.common.io.Files; import com.google.common.io.Resources; import static org.junit.Assert.*; public class ProtoGenTest { private static final Charset charset = StandardCharsets.UTF_8; @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test public void happyPath() throws Exception { File rootDir = folder.getRoot(); String protoPackage = "protoPackage"; String javaPackage = "abc.protogen.example"; String goPackage = "goPackage"; String sourcePackage = "com.example"; String mapperPackage = "mapperPackage"; File jarFile = new File("./build/libs/example.jar"); assertTrue(jarFile.exists()); File mapperDir = new File(rootDir, "mapperDir"); mapperDir.mkdirs(); File protosDir = new File(rootDir, "protosDir"); protosDir.mkdirs(); File modelDir = new File(protosDir, "model"); modelDir.mkdirs(); ProtoGen generator = new ProtoGen(protoPackage, javaPackage, goPackage); generator.processPackage(jarFile, sourcePackage); generator.writeMapper(mapperDir, mapperPackage); generator.writeProtos(protosDir); List models = Lists.newArrayList(modelDir.listFiles()); assertEquals(1, models.size()); File exampleProtoFile = models.stream().filter(f -> f.getName().equals("example.proto")).findFirst().get(); assertTrue(exampleProtoFile.length() > 0); assertEquals( Resources.asCharSource(Resources.getResource("example.proto.txt"), charset).read(), Files.asCharSource(exampleProtoFile, charset).read()); } } ================================================ FILE: annotations-processor/src/test/resources/example.proto.txt ================================================ syntax = "proto3"; package protoPackage; option java_package = "abc.protogen.example"; option java_outer_classname = "ExamplePb"; option go_package = "goPackage"; message Example { string name = 1; int64 count = 2; } ================================================ FILE: awss3-storage/README.md ================================================ ================================================ FILE: awss3-storage/build.gradle ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "com.amazonaws:aws-java-sdk-s3:${revAwsSdk}" implementation "org.apache.commons:commons-lang3" } ================================================ FILE: awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.s3.config; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.s3.storage.S3PayloadStorage; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @Configuration @EnableConfigurationProperties(S3Properties.class) @ConditionalOnProperty(name = "conductor.external-payload-storage.type", havingValue = "s3") public class S3Configuration { @Bean public ExternalPayloadStorage s3ExternalPayloadStorage( IDGenerator idGenerator, S3Properties properties, AmazonS3 s3Client) { return new S3PayloadStorage(idGenerator, properties, s3Client); } @ConditionalOnProperty( name = "conductor.external-payload-storage.s3.use_default_client", havingValue = "true", matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); } } ================================================ FILE: awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.s3.config; import java.time.Duration; import java.time.temporal.ChronoUnit; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; @ConfigurationProperties("conductor.external-payload-storage.s3") public class S3Properties { /** The s3 bucket name where the payloads will be stored */ private String bucketName = "conductor_payloads"; /** The time (in seconds) for which the signed url will be valid */ @DurationUnit(ChronoUnit.SECONDS) private Duration signedUrlExpirationDuration = Duration.ofSeconds(5); /** The AWS region of the s3 bucket */ private String region = "us-east-1"; public String getBucketName() { return bucketName; } public void setBucketName(String bucketName) { this.bucketName = bucketName; } public Duration getSignedUrlExpirationDuration() { return signedUrlExpirationDuration; } public void setSignedUrlExpirationDuration(Duration signedUrlExpirationDuration) { this.signedUrlExpirationDuration = signedUrlExpirationDuration; } public String getRegion() { return region; } public void setRegion(String region) { this.region = region; } } ================================================ FILE: awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.s3.storage; import java.io.InputStream; import java.net.URISyntaxException; import java.util.Date; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.s3.config.S3Properties; import com.amazonaws.HttpMethod; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.*; /** * An implementation of {@link ExternalPayloadStorage} using AWS S3 for storing large JSON payload * data. * *

NOTE: The S3 client assumes that access to S3 is configured on the instance. * * @see DefaultAWSCredentialsProviderChain */ public class S3PayloadStorage implements ExternalPayloadStorage { private static final Logger LOGGER = LoggerFactory.getLogger(S3PayloadStorage.class); private static final String CONTENT_TYPE = "application/json"; private final IDGenerator idGenerator; private final AmazonS3 s3Client; private final String bucketName; private final long expirationSec; public S3PayloadStorage(IDGenerator idGenerator, S3Properties properties, AmazonS3 s3Client) { this.idGenerator = idGenerator; this.s3Client = s3Client; bucketName = properties.getBucketName(); expirationSec = properties.getSignedUrlExpirationDuration().getSeconds(); } /** * @param operation the type of {@link Operation} to be performed * @param payloadType the {@link PayloadType} that is being accessed * @return a {@link ExternalStorageLocation} object which contains the pre-signed URL and the s3 * object key for the json payload */ @Override public ExternalStorageLocation getLocation( Operation operation, PayloadType payloadType, String path) { try { ExternalStorageLocation externalStorageLocation = new ExternalStorageLocation(); Date expiration = new Date(); long expTimeMillis = expiration.getTime() + 1000 * expirationSec; expiration.setTime(expTimeMillis); HttpMethod httpMethod = HttpMethod.GET; if (operation == Operation.WRITE) { httpMethod = HttpMethod.PUT; } String objectKey; if (StringUtils.isNotBlank(path)) { objectKey = path; } else { objectKey = getObjectKey(payloadType); } externalStorageLocation.setPath(objectKey); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey) .withMethod(httpMethod) .withExpiration(expiration); externalStorageLocation.setUri( s3Client.generatePresignedUrl(generatePresignedUrlRequest) .toURI() .toASCIIString()); return externalStorageLocation; } catch (SdkClientException e) { String msg = String.format( "Error communicating with S3 - operation:%s, payloadType: %s, path: %s", operation, payloadType, path); LOGGER.error(msg, e); throw new TransientException(msg, e); } catch (URISyntaxException e) { String msg = "Invalid URI Syntax"; LOGGER.error(msg, e); throw new NonTransientException(msg, e); } } /** * Uploads the payload to the given s3 object key. It is expected that the caller retrieves the * object key using {@link #getLocation(Operation, PayloadType, String)} before making this * call. * * @param path the s3 key of the object to be uploaded * @param payload an {@link InputStream} containing the json payload which is to be uploaded * @param payloadSize the size of the json payload in bytes */ @Override public void upload(String path, InputStream payload, long payloadSize) { try { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentType(CONTENT_TYPE); objectMetadata.setContentLength(payloadSize); PutObjectRequest request = new PutObjectRequest(bucketName, path, payload, objectMetadata); s3Client.putObject(request); } catch (SdkClientException e) { String msg = String.format( "Error uploading to S3 - path:%s, payloadSize: %d", path, payloadSize); LOGGER.error(msg, e); throw new TransientException(msg, e); } } /** * Downloads the payload stored in the s3 object. * * @param path the S3 key of the object * @return an input stream containing the contents of the object Caller is expected to close the * input stream. */ @Override public InputStream download(String path) { try { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, path)); return s3Object.getObjectContent(); } catch (SdkClientException e) { String msg = String.format("Error downloading from S3 - path:%s", path); LOGGER.error(msg, e); throw new TransientException(msg, e); } } private String getObjectKey(PayloadType payloadType) { StringBuilder stringBuilder = new StringBuilder(); switch (payloadType) { case WORKFLOW_INPUT: stringBuilder.append("workflow/input/"); break; case WORKFLOW_OUTPUT: stringBuilder.append("workflow/output/"); break; case TASK_INPUT: stringBuilder.append("task/input/"); break; case TASK_OUTPUT: stringBuilder.append("task/output/"); break; } stringBuilder.append(idGenerator.generate()).append(".json"); return stringBuilder.toString(); } } ================================================ FILE: awss3-storage/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "hints": [ { "name": "conductor.external-payload-storage.type", "values": [ { "value": "s3", "description": "Use AWS S3 as the external payload storage." } ] } ] } ================================================ FILE: awssqs-event-queue/README.md ================================================ ================================================ FILE: awssqs-event-queue/build.gradle ================================================ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "org.apache.commons:commons-lang3" // SBMTODO: remove guava dep implementation "com.google.guava:guava:${revGuava}" implementation "com.amazonaws:aws-java-sdk-sqs:${revAwsSdk}" implementation "io.reactivex:rxjava:${revRxJava}" testImplementation 'org.springframework.boot:spring-boot-starter' testImplementation project(':conductor-common').sourceSets.test.output } ================================================ FILE: awssqs-event-queue/src/main/java/com/netflix/conductor/sqs/config/SQSEventQueueConfiguration.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.config; import java.util.HashMap; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.EventQueueProvider; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.sqs.eventqueue.SQSObservableQueue.Builder; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import rx.Scheduler; @Configuration @EnableConfigurationProperties(SQSEventQueueProperties.class) @ConditionalOnProperty(name = "conductor.event-queues.sqs.enabled", havingValue = "true") public class SQSEventQueueConfiguration { @Autowired private SQSEventQueueProperties sqsProperties; private static final Logger LOGGER = LoggerFactory.getLogger(SQSEventQueueConfiguration.class); @Bean AWSCredentialsProvider createAWSCredentialsProvider() { return new DefaultAWSCredentialsProviderChain(); } @ConditionalOnMissingBean @Bean public AmazonSQS getSQSClient(AWSCredentialsProvider credentialsProvider) { AmazonSQSClientBuilder builder = AmazonSQSClientBuilder.standard().withCredentials(credentialsProvider); if (!sqsProperties.getEndpoint().isEmpty()) { LOGGER.info("Setting custom SQS endpoint to {}", sqsProperties.getEndpoint()); builder.withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( sqsProperties.getEndpoint(), System.getenv("AWS_REGION"))); } return builder.build(); } @Bean public EventQueueProvider sqsEventQueueProvider( AmazonSQS sqsClient, SQSEventQueueProperties properties, Scheduler scheduler) { return new SQSEventQueueProvider(sqsClient, properties, scheduler); } @ConditionalOnProperty( name = "conductor.default-event-queue.type", havingValue = "sqs", matchIfMissing = true) @Bean public Map getQueues( ConductorProperties conductorProperties, SQSEventQueueProperties properties, AmazonSQS sqsClient) { String stack = ""; if (conductorProperties.getStack() != null && conductorProperties.getStack().length() > 0) { stack = conductorProperties.getStack() + "_"; } Status[] statuses = new Status[] {Status.COMPLETED, Status.FAILED}; Map queues = new HashMap<>(); for (Status status : statuses) { String queuePrefix = StringUtils.isBlank(properties.getListenerQueuePrefix()) ? conductorProperties.getAppId() + "_sqs_notify_" + stack : properties.getListenerQueuePrefix(); String queueName = queuePrefix + status.name(); Builder builder = new Builder().withClient(sqsClient).withQueueName(queueName); String auth = properties.getAuthorizedAccounts(); String[] accounts = auth.split(","); for (String accountToAuthorize : accounts) { accountToAuthorize = accountToAuthorize.trim(); if (accountToAuthorize.length() > 0) { builder.addAccountToAuthorize(accountToAuthorize.trim()); } } ObservableQueue queue = builder.build(); queues.put(status, queue); } return queues; } } ================================================ FILE: awssqs-event-queue/src/main/java/com/netflix/conductor/sqs/config/SQSEventQueueProperties.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.config; import java.time.Duration; import java.time.temporal.ChronoUnit; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; @ConfigurationProperties("conductor.event-queues.sqs") public class SQSEventQueueProperties { /** The maximum number of messages to be fetched from the queue in a single request */ private int batchSize = 1; /** The polling interval (in milliseconds) */ private Duration pollTimeDuration = Duration.ofMillis(100); /** The visibility timeout (in seconds) for the message on the queue */ @DurationUnit(ChronoUnit.SECONDS) private Duration visibilityTimeout = Duration.ofSeconds(60); /** The prefix to be used for the default listener queues */ private String listenerQueuePrefix = ""; /** The AWS account Ids authorized to send messages to the queues */ private String authorizedAccounts = ""; /** The endpoint to use to connect to a local SQS server for testing */ private String endpoint = ""; public int getBatchSize() { return batchSize; } public void setBatchSize(int batchSize) { this.batchSize = batchSize; } public Duration getPollTimeDuration() { return pollTimeDuration; } public void setPollTimeDuration(Duration pollTimeDuration) { this.pollTimeDuration = pollTimeDuration; } public Duration getVisibilityTimeout() { return visibilityTimeout; } public void setVisibilityTimeout(Duration visibilityTimeout) { this.visibilityTimeout = visibilityTimeout; } public String getListenerQueuePrefix() { return listenerQueuePrefix; } public void setListenerQueuePrefix(String listenerQueuePrefix) { this.listenerQueuePrefix = listenerQueuePrefix; } public String getAuthorizedAccounts() { return authorizedAccounts; } public void setAuthorizedAccounts(String authorizedAccounts) { this.authorizedAccounts = authorizedAccounts; } public String getEndpoint() { return endpoint; } public void setEndpoint(String endpoint) { this.endpoint = endpoint; } } ================================================ FILE: awssqs-event-queue/src/main/java/com/netflix/conductor/sqs/config/SQSEventQueueProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.config; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.springframework.lang.NonNull; import com.netflix.conductor.core.events.EventQueueProvider; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.sqs.eventqueue.SQSObservableQueue; import com.amazonaws.services.sqs.AmazonSQS; import rx.Scheduler; public class SQSEventQueueProvider implements EventQueueProvider { private final Map queues = new ConcurrentHashMap<>(); private final AmazonSQS client; private final int batchSize; private final long pollTimeInMS; private final int visibilityTimeoutInSeconds; private final Scheduler scheduler; public SQSEventQueueProvider( AmazonSQS client, SQSEventQueueProperties properties, Scheduler scheduler) { this.client = client; this.batchSize = properties.getBatchSize(); this.pollTimeInMS = properties.getPollTimeDuration().toMillis(); this.visibilityTimeoutInSeconds = (int) properties.getVisibilityTimeout().getSeconds(); this.scheduler = scheduler; } @Override public String getQueueType() { return "sqs"; } @Override @NonNull public ObservableQueue getQueue(String queueURI) { return queues.computeIfAbsent( queueURI, q -> new SQSObservableQueue.Builder() .withBatchSize(this.batchSize) .withClient(client) .withPollTimeInMS(this.pollTimeInMS) .withQueueName(queueURI) .withVisibilityTimeout(this.visibilityTimeoutInSeconds) .withScheduler(scheduler) .build()); } } ================================================ FILE: awssqs-event-queue/src/main/java/com/netflix/conductor/sqs/eventqueue/SQSObservableQueue.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.eventqueue; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.metrics.Monitors; import com.amazonaws.auth.policy.Action; import com.amazonaws.auth.policy.Policy; import com.amazonaws.auth.policy.Principal; import com.amazonaws.auth.policy.Resource; import com.amazonaws.auth.policy.Statement; import com.amazonaws.auth.policy.Statement.Effect; import com.amazonaws.auth.policy.actions.SQSActions; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.model.BatchResultErrorEntry; import com.amazonaws.services.sqs.model.ChangeMessageVisibilityRequest; import com.amazonaws.services.sqs.model.CreateQueueRequest; import com.amazonaws.services.sqs.model.CreateQueueResult; import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest; import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; import com.amazonaws.services.sqs.model.DeleteMessageBatchResult; import com.amazonaws.services.sqs.model.GetQueueAttributesResult; import com.amazonaws.services.sqs.model.ListQueuesRequest; import com.amazonaws.services.sqs.model.ListQueuesResult; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; import com.amazonaws.services.sqs.model.ReceiveMessageResult; import com.amazonaws.services.sqs.model.SendMessageBatchRequest; import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry; import com.amazonaws.services.sqs.model.SendMessageBatchResult; import com.amazonaws.services.sqs.model.SetQueueAttributesResult; import rx.Observable; import rx.Observable.OnSubscribe; import rx.Scheduler; public class SQSObservableQueue implements ObservableQueue { private static final Logger LOGGER = LoggerFactory.getLogger(SQSObservableQueue.class); private static final String QUEUE_TYPE = "sqs"; private final String queueName; private final int visibilityTimeoutInSeconds; private final int batchSize; private final AmazonSQS client; private final long pollTimeInMS; private final String queueURL; private final Scheduler scheduler; private volatile boolean running; private SQSObservableQueue( String queueName, AmazonSQS client, int visibilityTimeoutInSeconds, int batchSize, long pollTimeInMS, List accountsToAuthorize, Scheduler scheduler) { this.queueName = queueName; this.client = client; this.visibilityTimeoutInSeconds = visibilityTimeoutInSeconds; this.batchSize = batchSize; this.pollTimeInMS = pollTimeInMS; this.queueURL = getOrCreateQueue(); this.scheduler = scheduler; addPolicy(accountsToAuthorize); } @Override public Observable observe() { OnSubscribe subscriber = getOnSubscribe(); return Observable.create(subscriber); } @Override public List ack(List messages) { return delete(messages); } @Override public void publish(List messages) { publishMessages(messages); } @Override public long size() { GetQueueAttributesResult attributes = client.getQueueAttributes( queueURL, Collections.singletonList("ApproximateNumberOfMessages")); String sizeAsStr = attributes.getAttributes().get("ApproximateNumberOfMessages"); try { return Long.parseLong(sizeAsStr); } catch (Exception e) { return -1; } } @Override public void setUnackTimeout(Message message, long unackTimeout) { int unackTimeoutInSeconds = (int) (unackTimeout / 1000); ChangeMessageVisibilityRequest request = new ChangeMessageVisibilityRequest( queueURL, message.getReceipt(), unackTimeoutInSeconds); client.changeMessageVisibility(request); } @Override public String getType() { return QUEUE_TYPE; } @Override public String getName() { return queueName; } @Override public String getURI() { return queueURL; } public long getPollTimeInMS() { return pollTimeInMS; } public int getBatchSize() { return batchSize; } public int getVisibilityTimeoutInSeconds() { return visibilityTimeoutInSeconds; } @Override public void start() { LOGGER.info("Started listening to {}:{}", getClass().getSimpleName(), queueName); running = true; } @Override public void stop() { LOGGER.info("Stopped listening to {}:{}", getClass().getSimpleName(), queueName); running = false; } @Override public boolean isRunning() { return running; } public static class Builder { private String queueName; private int visibilityTimeout = 30; // seconds private int batchSize = 5; private long pollTimeInMS = 100; private AmazonSQS client; private List accountsToAuthorize = new LinkedList<>(); private Scheduler scheduler; public Builder withQueueName(String queueName) { this.queueName = queueName; return this; } /** * @param visibilityTimeout Visibility timeout for the message in SECONDS * @return builder instance */ public Builder withVisibilityTimeout(int visibilityTimeout) { this.visibilityTimeout = visibilityTimeout; return this; } public Builder withBatchSize(int batchSize) { this.batchSize = batchSize; return this; } public Builder withClient(AmazonSQS client) { this.client = client; return this; } public Builder withPollTimeInMS(long pollTimeInMS) { this.pollTimeInMS = pollTimeInMS; return this; } public Builder withAccountsToAuthorize(List accountsToAuthorize) { this.accountsToAuthorize = accountsToAuthorize; return this; } public Builder addAccountToAuthorize(String accountToAuthorize) { this.accountsToAuthorize.add(accountToAuthorize); return this; } public Builder withScheduler(Scheduler scheduler) { this.scheduler = scheduler; return this; } public SQSObservableQueue build() { return new SQSObservableQueue( queueName, client, visibilityTimeout, batchSize, pollTimeInMS, accountsToAuthorize, scheduler); } } // Private methods String getOrCreateQueue() { List queueUrls = listQueues(queueName); if (queueUrls == null || queueUrls.isEmpty()) { CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName(queueName); CreateQueueResult result = client.createQueue(createQueueRequest); return result.getQueueUrl(); } else { return queueUrls.get(0); } } private String getQueueARN() { GetQueueAttributesResult response = client.getQueueAttributes(queueURL, Collections.singletonList("QueueArn")); return response.getAttributes().get("QueueArn"); } private void addPolicy(List accountsToAuthorize) { if (accountsToAuthorize == null || accountsToAuthorize.isEmpty()) { LOGGER.info("No additional security policies attached for the queue " + queueName); return; } LOGGER.info("Authorizing " + accountsToAuthorize + " to the queue " + queueName); Map attributes = new HashMap<>(); attributes.put("Policy", getPolicy(accountsToAuthorize)); SetQueueAttributesResult result = client.setQueueAttributes(queueURL, attributes); LOGGER.info("policy attachment result: " + result); LOGGER.info( "policy attachment result: status=" + result.getSdkHttpMetadata().getHttpStatusCode()); } private String getPolicy(List accountIds) { Policy policy = new Policy("AuthorizedWorkerAccessPolicy"); Statement stmt = new Statement(Effect.Allow); Action action = SQSActions.SendMessage; stmt.getActions().add(action); stmt.setResources(new LinkedList<>()); for (String accountId : accountIds) { Principal principal = new Principal(accountId); stmt.getPrincipals().add(principal); } stmt.getResources().add(new Resource(getQueueARN())); policy.getStatements().add(stmt); return policy.toJson(); } private List listQueues(String queueName) { ListQueuesRequest listQueuesRequest = new ListQueuesRequest().withQueueNamePrefix(queueName); ListQueuesResult resultList = client.listQueues(listQueuesRequest); return resultList.getQueueUrls().stream() .filter(queueUrl -> queueUrl.contains(queueName)) .collect(Collectors.toList()); } private void publishMessages(List messages) { LOGGER.debug("Sending {} messages to the SQS queue: {}", messages.size(), queueName); SendMessageBatchRequest batch = new SendMessageBatchRequest(queueURL); messages.forEach( msg -> { SendMessageBatchRequestEntry sendr = new SendMessageBatchRequestEntry(msg.getId(), msg.getPayload()); batch.getEntries().add(sendr); }); LOGGER.debug("sending {} messages in batch", batch.getEntries().size()); SendMessageBatchResult result = client.sendMessageBatch(batch); LOGGER.debug("send result: {} for SQS queue: {}", result.getFailed().toString(), queueName); } List receiveMessages() { try { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest() .withQueueUrl(queueURL) .withVisibilityTimeout(visibilityTimeoutInSeconds) .withMaxNumberOfMessages(batchSize); ReceiveMessageResult result = client.receiveMessage(receiveMessageRequest); List messages = result.getMessages().stream() .map( msg -> new Message( msg.getMessageId(), msg.getBody(), msg.getReceiptHandle())) .collect(Collectors.toList()); Monitors.recordEventQueueMessagesProcessed(QUEUE_TYPE, this.queueName, messages.size()); return messages; } catch (Exception e) { LOGGER.error("Exception while getting messages from SQS", e); Monitors.recordObservableQMessageReceivedErrors(QUEUE_TYPE); } return new ArrayList<>(); } OnSubscribe getOnSubscribe() { return subscriber -> { Observable interval = Observable.interval(pollTimeInMS, TimeUnit.MILLISECONDS); interval.flatMap( (Long x) -> { if (!isRunning()) { LOGGER.debug( "Component stopped, skip listening for messages from SQS"); return Observable.from(Collections.emptyList()); } List messages = receiveMessages(); return Observable.from(messages); }) .subscribe(subscriber::onNext, subscriber::onError); }; } private List delete(List messages) { if (messages == null || messages.isEmpty()) { return null; } DeleteMessageBatchRequest batch = new DeleteMessageBatchRequest().withQueueUrl(queueURL); List entries = batch.getEntries(); messages.forEach( m -> entries.add( new DeleteMessageBatchRequestEntry() .withId(m.getId()) .withReceiptHandle(m.getReceipt()))); DeleteMessageBatchResult result = client.deleteMessageBatch(batch); List failures = result.getFailed().stream() .map(BatchResultErrorEntry::getId) .collect(Collectors.toList()); LOGGER.debug("Failed to delete messages from queue: {}: {}", queueName, failures); return failures; } } ================================================ FILE: awssqs-event-queue/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.event-queues.sqs.enabled", "type": "java.lang.Boolean", "description": "Enable the use of AWS SQS implementation to provide queues for consuming events.", "sourceType": "com.netflix.conductor.sqs.config.SQSEventQueueConfiguration" }, { "name": "conductor.default-event-queue.type", "type": "java.lang.String", "description": "The default event queue type to listen on for the WAIT task.", "sourceType": "com.netflix.conductor.sqs.config.SQSEventQueueConfiguration" } ], "hints": [ { "name": "conductor.default-event-queue.type", "values": [ { "value": "sqs", "description": "Use AWS SQS as the event queue to listen on for the WAIT task." } ] } ] } ================================================ FILE: awssqs-event-queue/src/test/java/com/netflix/conductor/sqs/eventqueue/DefaultEventQueueProcessorTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.eventqueue; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.core.events.queue.DefaultEventQueueProcessor; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Uninterruptibles; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @SuppressWarnings("unchecked") @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class DefaultEventQueueProcessorTest { private static SQSObservableQueue queue; private static WorkflowExecutor workflowExecutor; private DefaultEventQueueProcessor defaultEventQueueProcessor; @Autowired private ObjectMapper objectMapper; private static final List messages = new LinkedList<>(); private static final List updatedTasks = new LinkedList<>(); private static final List mappedTasks = new LinkedList<>(); @Before public void init() { Map queues = new HashMap<>(); queues.put(Status.COMPLETED, queue); defaultEventQueueProcessor = new DefaultEventQueueProcessor(queues, workflowExecutor, objectMapper); } @BeforeClass public static void setup() { queue = mock(SQSObservableQueue.class); when(queue.getOrCreateQueue()).thenReturn("junit_queue_url"); when(queue.isRunning()).thenReturn(true); Answer answer = (Answer>) invocation -> { List copy = new LinkedList<>(messages); messages.clear(); return copy; }; when(queue.receiveMessages()).thenAnswer(answer); when(queue.getOnSubscribe()).thenCallRealMethod(); when(queue.observe()).thenCallRealMethod(); when(queue.getName()).thenReturn(Status.COMPLETED.name()); TaskModel task0 = new TaskModel(); task0.setStatus(Status.IN_PROGRESS); task0.setTaskId("t0"); task0.setReferenceTaskName("t0"); task0.setTaskType(TASK_TYPE_WAIT); WorkflowModel workflow0 = new WorkflowModel(); workflow0.setWorkflowId("v_0"); workflow0.getTasks().add(task0); TaskModel task2 = new TaskModel(); task2.setStatus(Status.IN_PROGRESS); task2.setTaskId("t2"); task2.setTaskType(TASK_TYPE_WAIT); WorkflowModel workflow2 = new WorkflowModel(); workflow2.setWorkflowId("v_2"); workflow2.getTasks().add(task2); doAnswer( (Answer) invocation -> { List msgs = invocation.getArgument(0, List.class); messages.addAll(msgs); return null; }) .when(queue) .publish(any()); workflowExecutor = mock(WorkflowExecutor.class); assertNotNull(workflowExecutor); doReturn(workflow0).when(workflowExecutor).getWorkflow(eq("v_0"), anyBoolean()); doReturn(workflow2).when(workflowExecutor).getWorkflow(eq("v_2"), anyBoolean()); doAnswer( (Answer) invocation -> { updatedTasks.add(invocation.getArgument(0, TaskResult.class)); return null; }) .when(workflowExecutor) .updateTask(any(TaskResult.class)); } @Test public void test() throws Exception { defaultEventQueueProcessor.updateByTaskRefName( "v_0", "t0", new HashMap<>(), Status.COMPLETED); Uninterruptibles.sleepUninterruptibly(1_000, TimeUnit.MILLISECONDS); assertTrue(updatedTasks.stream().anyMatch(task -> task.getTaskId().equals("t0"))); } @Test(expected = IllegalArgumentException.class) public void testFailure() throws Exception { defaultEventQueueProcessor.updateByTaskRefName( "v_1", "t1", new HashMap<>(), Status.CANCELED); Uninterruptibles.sleepUninterruptibly(1_000, TimeUnit.MILLISECONDS); } @Test public void testWithTaskId() throws Exception { defaultEventQueueProcessor.updateByTaskId("v_2", "t2", new HashMap<>(), Status.COMPLETED); Uninterruptibles.sleepUninterruptibly(1_000, TimeUnit.MILLISECONDS); assertTrue(updatedTasks.stream().anyMatch(task -> task.getTaskId().equals("t2"))); } } ================================================ FILE: awssqs-event-queue/src/test/java/com/netflix/conductor/sqs/eventqueue/SQSObservableQueueTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sqs.eventqueue; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; import org.junit.Test; import org.mockito.stubbing.Answer; import com.netflix.conductor.core.events.queue.Message; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.model.ListQueuesRequest; import com.amazonaws.services.sqs.model.ListQueuesResult; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; import com.amazonaws.services.sqs.model.ReceiveMessageResult; import com.google.common.util.concurrent.Uninterruptibles; import rx.Observable; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SQSObservableQueueTest { @Test public void test() { List messages = new LinkedList<>(); Observable.range(0, 10) .forEach((Integer x) -> messages.add(new Message("" + x, "payload: " + x, null))); assertEquals(10, messages.size()); SQSObservableQueue queue = mock(SQSObservableQueue.class); when(queue.getOrCreateQueue()).thenReturn("junit_queue_url"); Answer answer = (Answer>) invocation -> Collections.emptyList(); when(queue.receiveMessages()).thenReturn(messages).thenAnswer(answer); when(queue.isRunning()).thenReturn(true); when(queue.getOnSubscribe()).thenCallRealMethod(); when(queue.observe()).thenCallRealMethod(); List found = new LinkedList<>(); Observable observable = queue.observe(); assertNotNull(observable); observable.subscribe(found::add); Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); assertEquals(messages.size(), found.size()); assertEquals(messages, found); } @Test public void testException() { com.amazonaws.services.sqs.model.Message message = new com.amazonaws.services.sqs.model.Message() .withMessageId("test") .withBody("") .withReceiptHandle("receiptHandle"); Answer answer = (Answer) invocation -> new ReceiveMessageResult(); AmazonSQS client = mock(AmazonSQS.class); when(client.listQueues(any(ListQueuesRequest.class))) .thenReturn(new ListQueuesResult().withQueueUrls("junit_queue_url")); when(client.receiveMessage(any(ReceiveMessageRequest.class))) .thenThrow(new RuntimeException("Error in SQS communication")) .thenReturn(new ReceiveMessageResult().withMessages(message)) .thenAnswer(answer); SQSObservableQueue queue = new SQSObservableQueue.Builder().withQueueName("junit").withClient(client).build(); queue.start(); List found = new LinkedList<>(); Observable observable = queue.observe(); assertNotNull(observable); observable.subscribe(found::add); Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); assertEquals(1, found.size()); } } ================================================ FILE: build.gradle ================================================ import org.springframework.boot.gradle.plugin.SpringBootPlugin buildscript { repositories { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.7.16' classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.+' } } plugins { id 'io.spring.dependency-management' version '1.1.3' id 'java' id 'application' id 'jacoco' id 'com.netflix.nebula.netflixoss' version '11.3.2' id 'org.sonarqube' version '3.4.0.2513' } /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ // Establish version and status ext.githubProjectName = rootProject.name // Change if github project name is not the same as the root project's name subprojects { tasks.withType(Javadoc).all { enabled = false } } apply from: "$rootDir/dependencies.gradle" apply from: "$rootDir/springboot-bom-overrides.gradle" allprojects { apply plugin: 'com.netflix.nebula.netflixoss' apply plugin: 'io.spring.dependency-management' apply plugin: 'java-library' apply plugin: 'project-report' java { toolchain { languageVersion = JavaLanguageVersion.of(17) } } group = 'com.netflix.conductor' configurations.all { exclude group: 'ch.qos.logback', module: 'logback-classic' exclude group: 'ch.qos.logback', module: 'logback-core' exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' exclude group: 'org.slf4j', module: 'slf4j-log4j12' resolutionStrategy { force 'org.codehaus.jettison:jettison:1.5.4' force "org.apache.commons:commons-compress:${revCommonsCompress}" } } repositories { mavenCentral() // oss-candidate for -rc.* verions: maven { url "https://artifactory-oss.prod.netflix.net/artifactory/maven-oss-candidates" } /** * This repository locates artifacts that don't exist in maven central but we had to backup from jcenter * The exclusiveContent */ exclusiveContent { forRepository { maven { url "https://artifactory-oss.prod.netflix.net/artifactory/required-jcenter-modules-backup" } } filter { includeGroupByRegex "com\\.github\\.vmg.*" } } } dependencyManagement { imports { // dependency versions for the BOM can be found at https://docs.spring.io/spring-boot/docs/2.7.3/reference/htmlsingle/#appendix.dependency-versions mavenBom(SpringBootPlugin.BOM_COORDINATES) } } dependencies { implementation('org.apache.logging.log4j:log4j-core') { version { // this is the preferred version this library will use prefer '2.17.2' // the strict bounds, effectively allowing any 2.x version greater than 2.17.2 // could also remove the upper bound entirely if we wanted too strictly '[2.17.2,3.0)' } } implementation('org.apache.logging.log4j:log4j-api') { version { // this is the preferred version this library will use prefer '2.17.2' // the strict bounds, effectively allowing any 2.x version greater than 2.17.2 // could also remove the upper bound entirely if we wanted too strictly '[2.17.2,3.0)' } } implementation('org.apache.logging.log4j:log4j-slf4j-impl') { version { // this is the preferred version this library will use prefer '2.17.2' // the strict bounds, effectively allowing any 2.x version greater than 2.17.2 // could also remove the upper bound entirely if we wanted too strictly '[2.17.2,3.0)' } } implementation('org.apache.logging.log4j:log4j-jul') { version { // this is the preferred version this library will use prefer '2.17.2' // the strict bounds, effectively allowing any 2.x version greater than 2.17.2 // could also remove the upper bound entirely if we wanted too strictly '[2.17.2,3.0)' } } implementation('org.apache.logging.log4j:log4j-web') { version { // this is the preferred version this library will use prefer '2.17.2' // the strict bounds, effectively allowing any 2.x version greater than 2.17.2 // could also remove the upper bound entirely if we wanted too strictly '[2.17.2,3.0)' } } implementation('org.yaml:snakeyaml') { version { // this is the preferred version this library will use prefer '2.0' // the strict bounds, effectively allowing any 2.x version between 2.0 and 2.1 strictly '[2.0,2.1)' } } implementation('com.fasterxml.jackson.core:jackson-core') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.core:jackson-databind') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-yaml') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.core:jackson-annotations') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-smile') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-cbor') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.datatype:jackson-datatype-jdk8') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.datatype:jackson-datatype-joda') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.datatype:jackson-datatype-jsr310') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('com.fasterxml.jackson.module:jackson-module-afterburner') { version { // this is the preferred version this library will use prefer '2.15.0' // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2 strictly '[2.15.0,2.15.2)' } } implementation('org.apache.logging.log4j:log4j-core') implementation('org.apache.logging.log4j:log4j-api') implementation('org.apache.logging.log4j:log4j-slf4j-impl') implementation('org.apache.logging.log4j:log4j-jul') implementation('org.apache.logging.log4j:log4j-web') annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.yaml', module: 'snakeyaml' } testImplementation('org.springframework.boot:spring-boot-starter-log4j2') testImplementation 'junit:junit' testImplementation "org.junit.vintage:junit-vintage-engine" // Needed for build to work on m1/m2 macs testImplementation 'net.java.dev.jna:jna:5.13.0' } // processes additional configuration metadata json file as described here // https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/appendix-configuration-metadata.html#configuration-metadata-additional-metadata compileJava.inputs.files(processResources) test { useJUnitPlatform() testLogging { events = ["SKIPPED", "FAILED"] exceptionFormat = "full" displayGranularity = 1 showStandardStreams = false } } } // all client and their related modules are published with Java 17 compatibility ["annotations", "common", "client", "client-spring", "grpc", "grpc-client"].each { project(":conductor-$it") { compileJava { options.release = 17 } } } jacocoTestReport { reports { html.required = true xml.required = true csv.required = false } } task server { dependsOn ':conductor-server:bootRun' } sonarqube { properties { property "sonar.projectKey", "com.netflix.conductor:conductor" property "sonar.organization", "netflix" property "sonar.host.url", "https://sonarcloud.io" } } configure(allprojects - project(':conductor-grpc')) { apply plugin: 'com.diffplug.spotless' spotless { java { googleJavaFormat().aosp() removeUnusedImports() importOrder('java', 'javax', 'org', 'com.netflix', '', '\\#com.netflix', '\\#') licenseHeaderFile("$rootDir/licenseheader.txt") } } } ['cassandra-persistence', 'core', 'redis-concurrency-limit', 'test-harness', 'client'].each { configure(project(":conductor-$it")) { spotless { groovy { importOrder('java', 'javax', 'org', 'com.netflix', '', '\\#com.netflix', '\\#') licenseHeaderFile("$rootDir/licenseheader.txt") } } } } ================================================ FILE: cassandra-persistence/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ apply plugin: 'groovy' dependencies { compileOnly 'org.springframework.boot:spring-boot-starter' implementation project(':conductor-common') implementation project(':conductor-core') implementation "com.datastax.cassandra:cassandra-driver-core:${revCassandra}" implementation "org.apache.commons:commons-lang3" testImplementation project(':conductor-core').sourceSets.test.output testImplementation project(':conductor-common').sourceSets.test.output testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" testImplementation "org.testcontainers:spock:${revTestContainer}" testImplementation "org.testcontainers:cassandra:${revTestContainer}" testImplementation "com.google.protobuf:protobuf-java:${revProtoBuf}" } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/CassandraConfiguration.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.cassandra.config.cache.CacheableEventHandlerDAO; import com.netflix.conductor.cassandra.config.cache.CacheableMetadataDAO; import com.netflix.conductor.cassandra.dao.CassandraEventHandlerDAO; import com.netflix.conductor.cassandra.dao.CassandraExecutionDAO; import com.netflix.conductor.cassandra.dao.CassandraMetadataDAO; import com.netflix.conductor.cassandra.dao.CassandraPollDataDAO; import com.netflix.conductor.cassandra.util.Statements; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.MetadataDAO; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Metadata; import com.datastax.driver.core.Session; import com.fasterxml.jackson.databind.ObjectMapper; @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(CassandraProperties.class) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "cassandra") public class CassandraConfiguration { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraConfiguration.class); @Bean public Cluster cluster(CassandraProperties properties) { String host = properties.getHostAddress(); int port = properties.getPort(); LOGGER.info("Connecting to cassandra cluster with host:{}, port:{}", host, port); Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build(); Metadata metadata = cluster.getMetadata(); LOGGER.info("Connected to cluster: {}", metadata.getClusterName()); metadata.getAllHosts() .forEach( h -> LOGGER.info( "Datacenter:{}, host:{}, rack: {}", h.getDatacenter(), h.getEndPoint().resolve().getHostName(), h.getRack())); return cluster; } @Bean public Session session(Cluster cluster) { LOGGER.info("Initializing cassandra session"); return cluster.connect(); } @Bean public MetadataDAO cassandraMetadataDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements, CacheManager cacheManager) { CassandraMetadataDAO cassandraMetadataDAO = new CassandraMetadataDAO(session, objectMapper, properties, statements); return new CacheableMetadataDAO(cassandraMetadataDAO, properties, cacheManager); } @Bean public ExecutionDAO cassandraExecutionDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements) { return new CassandraExecutionDAO(session, objectMapper, properties, statements); } @Bean public EventHandlerDAO cassandraEventHandlerDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements, CacheManager cacheManager) { CassandraEventHandlerDAO cassandraEventHandlerDAO = new CassandraEventHandlerDAO(session, objectMapper, properties, statements); return new CacheableEventHandlerDAO(cassandraEventHandlerDAO, properties, cacheManager); } @Bean public CassandraPollDataDAO cassandraPollDataDAO() { return new CassandraPollDataDAO(); } @Bean public Statements statements(CassandraProperties cassandraProperties) { return new Statements(cassandraProperties.getKeyspace()); } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/CassandraProperties.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.config; import java.time.Duration; import java.time.temporal.ChronoUnit; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; import com.datastax.driver.core.ConsistencyLevel; @ConfigurationProperties("conductor.cassandra") public class CassandraProperties { /** The address for the cassandra database host */ private String hostAddress = "127.0.0.1"; /** The port to be used to connect to the cassandra database instance */ private int port = 9142; /** The name of the cassandra cluster */ private String cluster = ""; /** The keyspace to be used in the cassandra datastore */ private String keyspace = "conductor"; /** * The number of tasks to be stored in a single partition which will be used for sharding * workflows in the datastore */ private int shardSize = 100; /** The replication strategy with which to configure the keyspace */ private String replicationStrategy = "SimpleStrategy"; /** The key to be used while configuring the replication factor */ private String replicationFactorKey = "replication_factor"; /** The replication factor value with which the keyspace is configured */ private int replicationFactorValue = 3; /** The consistency level to be used for read operations */ private ConsistencyLevel readConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM; /** The consistency level to be used for write operations */ private ConsistencyLevel writeConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM; /** The time in seconds after which the in-memory task definitions cache will be refreshed */ @DurationUnit(ChronoUnit.SECONDS) private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60); /** The time in seconds after which the in-memory event handler cache will be refreshed */ @DurationUnit(ChronoUnit.SECONDS) private Duration eventHandlerCacheRefreshInterval = Duration.ofSeconds(60); /** The time to live in seconds for which the event execution will be persisted */ @DurationUnit(ChronoUnit.SECONDS) private Duration eventExecutionPersistenceTtl = Duration.ZERO; public String getHostAddress() { return hostAddress; } public void setHostAddress(String hostAddress) { this.hostAddress = hostAddress; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getCluster() { return cluster; } public void setCluster(String cluster) { this.cluster = cluster; } public String getKeyspace() { return keyspace; } public void setKeyspace(String keyspace) { this.keyspace = keyspace; } public int getShardSize() { return shardSize; } public void setShardSize(int shardSize) { this.shardSize = shardSize; } public String getReplicationStrategy() { return replicationStrategy; } public void setReplicationStrategy(String replicationStrategy) { this.replicationStrategy = replicationStrategy; } public String getReplicationFactorKey() { return replicationFactorKey; } public void setReplicationFactorKey(String replicationFactorKey) { this.replicationFactorKey = replicationFactorKey; } public int getReplicationFactorValue() { return replicationFactorValue; } public void setReplicationFactorValue(int replicationFactorValue) { this.replicationFactorValue = replicationFactorValue; } public ConsistencyLevel getReadConsistencyLevel() { return readConsistencyLevel; } public void setReadConsistencyLevel(ConsistencyLevel readConsistencyLevel) { this.readConsistencyLevel = readConsistencyLevel; } public ConsistencyLevel getWriteConsistencyLevel() { return writeConsistencyLevel; } public void setWriteConsistencyLevel(ConsistencyLevel writeConsistencyLevel) { this.writeConsistencyLevel = writeConsistencyLevel; } public Duration getTaskDefCacheRefreshInterval() { return taskDefCacheRefreshInterval; } public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) { this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval; } public Duration getEventHandlerCacheRefreshInterval() { return eventHandlerCacheRefreshInterval; } public void setEventHandlerCacheRefreshInterval(Duration eventHandlerCacheRefreshInterval) { this.eventHandlerCacheRefreshInterval = eventHandlerCacheRefreshInterval; } public Duration getEventExecutionPersistenceTtl() { return eventExecutionPersistenceTtl; } public void setEventExecutionPersistenceTtl(Duration eventExecutionPersistenceTtl) { this.eventExecutionPersistenceTtl = eventExecutionPersistenceTtl; } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/cache/CacheableEventHandlerDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.config.cache; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.CachePut; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.cassandra.dao.CassandraEventHandlerDAO; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.metrics.Monitors; import static com.netflix.conductor.cassandra.config.cache.CachingConfig.EVENT_HANDLER_CACHE; @Trace public class CacheableEventHandlerDAO implements EventHandlerDAO { private static final Logger LOGGER = LoggerFactory.getLogger(CacheableEventHandlerDAO.class); private static final String CLASS_NAME = CacheableEventHandlerDAO.class.getSimpleName(); private final CassandraEventHandlerDAO cassandraEventHandlerDAO; private final CassandraProperties properties; private final CacheManager cacheManager; public CacheableEventHandlerDAO( CassandraEventHandlerDAO cassandraEventHandlerDAO, CassandraProperties properties, CacheManager cacheManager) { this.cassandraEventHandlerDAO = cassandraEventHandlerDAO; this.properties = properties; this.cacheManager = cacheManager; } @PostConstruct public void scheduleEventHandlerRefresh() { long cacheRefreshTime = properties.getEventHandlerCacheRefreshInterval().getSeconds(); Executors.newSingleThreadScheduledExecutor() .scheduleWithFixedDelay( this::refreshEventHandlersCache, 0, cacheRefreshTime, TimeUnit.SECONDS); } @Override @CachePut(value = EVENT_HANDLER_CACHE, key = "#eventHandler.name") public void addEventHandler(EventHandler eventHandler) { cassandraEventHandlerDAO.addEventHandler(eventHandler); } @Override @CachePut(value = EVENT_HANDLER_CACHE, key = "#eventHandler.name") public void updateEventHandler(EventHandler eventHandler) { cassandraEventHandlerDAO.updateEventHandler(eventHandler); } @Override @CacheEvict(EVENT_HANDLER_CACHE) public void removeEventHandler(String name) { cassandraEventHandlerDAO.removeEventHandler(name); } @Override public List getAllEventHandlers() { Object nativeCache = cacheManager.getCache(EVENT_HANDLER_CACHE).getNativeCache(); if (nativeCache != null && nativeCache instanceof ConcurrentHashMap) { ConcurrentHashMap cacheMap = (ConcurrentHashMap) nativeCache; if (!cacheMap.isEmpty()) { List eventHandlers = new ArrayList<>(); cacheMap.values().stream() .filter(element -> element != null && element instanceof EventHandler) .forEach(element -> eventHandlers.add((EventHandler) element)); return eventHandlers; } } return refreshEventHandlersCache(); } @Override public List getEventHandlersForEvent(String event, boolean activeOnly) { if (activeOnly) { return getAllEventHandlers().stream() .filter(eventHandler -> eventHandler.getEvent().equals(event)) .filter(EventHandler::isActive) .collect(Collectors.toList()); } else { return getAllEventHandlers().stream() .filter(eventHandler -> eventHandler.getEvent().equals(event)) .collect(Collectors.toList()); } } private List refreshEventHandlersCache() { try { Cache eventHandlersCache = cacheManager.getCache(EVENT_HANDLER_CACHE); eventHandlersCache.clear(); List eventHandlers = cassandraEventHandlerDAO.getAllEventHandlers(); eventHandlers.forEach( eventHandler -> eventHandlersCache.put(eventHandler.getName(), eventHandler)); LOGGER.debug("Refreshed event handlers, total num: " + eventHandlers.size()); return eventHandlers; } catch (Exception e) { Monitors.error(CLASS_NAME, "refreshEventHandlersCache"); LOGGER.error("refresh EventHandlers failed", e); } return Collections.emptyList(); } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/cache/CacheableMetadataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.config.cache; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.CachePut; import org.springframework.cache.annotation.Cacheable; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.cassandra.dao.CassandraMetadataDAO; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import static com.netflix.conductor.cassandra.config.cache.CachingConfig.TASK_DEF_CACHE; @Trace public class CacheableMetadataDAO implements MetadataDAO { private static final String CLASS_NAME = CacheableMetadataDAO.class.getSimpleName(); private static final Logger LOGGER = LoggerFactory.getLogger(CacheableMetadataDAO.class); private final CassandraMetadataDAO cassandraMetadataDAO; private final CassandraProperties properties; private final CacheManager cacheManager; public CacheableMetadataDAO( CassandraMetadataDAO cassandraMetadataDAO, CassandraProperties properties, CacheManager cacheManager) { this.cassandraMetadataDAO = cassandraMetadataDAO; this.properties = properties; this.cacheManager = cacheManager; } @PostConstruct public void scheduleCacheRefresh() { long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds(); Executors.newSingleThreadScheduledExecutor() .scheduleWithFixedDelay( this::refreshTaskDefsCache, 0, cacheRefreshTime, TimeUnit.SECONDS); LOGGER.info( "Scheduled cache refresh for Task Definitions, every {} seconds", cacheRefreshTime); } @Override @CachePut(value = TASK_DEF_CACHE, key = "#taskDef.name") public TaskDef createTaskDef(TaskDef taskDef) { cassandraMetadataDAO.createTaskDef(taskDef); return taskDef; } @Override @CachePut(value = TASK_DEF_CACHE, key = "#taskDef.name") public TaskDef updateTaskDef(TaskDef taskDef) { return cassandraMetadataDAO.updateTaskDef(taskDef); } @Override @Cacheable(TASK_DEF_CACHE) public TaskDef getTaskDef(String name) { return cassandraMetadataDAO.getTaskDef(name); } @Override public List getAllTaskDefs() { Object nativeCache = cacheManager.getCache(TASK_DEF_CACHE).getNativeCache(); if (nativeCache != null && nativeCache instanceof ConcurrentHashMap) { ConcurrentHashMap cacheMap = (ConcurrentHashMap) nativeCache; if (!cacheMap.isEmpty()) { List taskDefs = new ArrayList<>(); cacheMap.values().stream() .filter(element -> element != null && element instanceof TaskDef) .forEach(element -> taskDefs.add((TaskDef) element)); return taskDefs; } } return refreshTaskDefsCache(); } @Override @CacheEvict(TASK_DEF_CACHE) public void removeTaskDef(String name) { cassandraMetadataDAO.removeTaskDef(name); } @Override public void createWorkflowDef(WorkflowDef workflowDef) { cassandraMetadataDAO.createWorkflowDef(workflowDef); } @Override public void updateWorkflowDef(WorkflowDef workflowDef) { cassandraMetadataDAO.updateWorkflowDef(workflowDef); } @Override public Optional getLatestWorkflowDef(String name) { return cassandraMetadataDAO.getLatestWorkflowDef(name); } @Override public Optional getWorkflowDef(String name, int version) { return cassandraMetadataDAO.getWorkflowDef(name, version); } @Override public void removeWorkflowDef(String name, Integer version) { cassandraMetadataDAO.removeWorkflowDef(name, version); } @Override public List getAllWorkflowDefs() { return cassandraMetadataDAO.getAllWorkflowDefs(); } @Override public List getAllWorkflowDefsLatestVersions() { return cassandraMetadataDAO.getAllWorkflowDefsLatestVersions(); } private List refreshTaskDefsCache() { try { Cache taskDefsCache = cacheManager.getCache(TASK_DEF_CACHE); taskDefsCache.clear(); List taskDefs = cassandraMetadataDAO.getAllTaskDefs(); taskDefs.forEach(taskDef -> taskDefsCache.put(taskDef.getName(), taskDef)); LOGGER.debug("Refreshed task defs, total num: " + taskDefs.size()); return taskDefs; } catch (Exception e) { Monitors.error(CLASS_NAME, "refreshTaskDefs"); LOGGER.error("refresh TaskDefs failed ", e); } return Collections.emptyList(); } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/cache/CachingConfig.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.config.cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration @EnableCaching public class CachingConfig { public static final String TASK_DEF_CACHE = "taskDefCache"; public static final String EVENT_HANDLER_CACHE = "eventHandlerCache"; @Bean public CacheManager cacheManager() { return new ConcurrentMapCacheManager(TASK_DEF_CACHE, EVENT_HANDLER_CACHE); } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraBaseDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao; import java.io.IOException; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.metrics.Monitors; import com.datastax.driver.core.DataType; import com.datastax.driver.core.Session; import com.datastax.driver.core.schemabuilder.SchemaBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import static com.netflix.conductor.cassandra.util.Constants.DAO_NAME; import static com.netflix.conductor.cassandra.util.Constants.ENTITY_KEY; import static com.netflix.conductor.cassandra.util.Constants.EVENT_EXECUTION_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY; import static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY; import static com.netflix.conductor.cassandra.util.Constants.MESSAGE_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.PAYLOAD_KEY; import static com.netflix.conductor.cassandra.util.Constants.SHARD_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_EXECUTIONS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_HANDLERS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEFS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEF_LIMIT; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_LOOKUP; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOWS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS_INDEX; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEF_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.TOTAL_PARTITIONS_KEY; import static com.netflix.conductor.cassandra.util.Constants.TOTAL_TASKS_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_VALUE; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_VERSION_KEY; /** * Creates the keyspace and tables. * *

CREATE KEYSPACE IF NOT EXISTS conductor WITH replication = { 'class' : * 'NetworkTopologyStrategy', 'us-east': '3'}; * *

CREATE TABLE IF NOT EXISTS conductor.workflows ( workflow_id uuid, shard_id int, task_id text, * entity text, payload text, total_tasks int STATIC, total_partitions int STATIC, PRIMARY * KEY((workflow_id, shard_id), entity, task_id) ); * *

CREATE TABLE IF NOT EXISTS conductor.task_lookup( task_id uuid, workflow_id uuid, PRIMARY KEY * (task_id) ); * *

CREATE TABLE IF NOT EXISTS conductor.task_def_limit( task_def_name text, task_id uuid, * workflow_id uuid, PRIMARY KEY ((task_def_name), task_id_key) ); * *

CREATE TABLE IF NOT EXISTS conductor.workflow_definitions( workflow_def_name text, version * int, workflow_definition text, PRIMARY KEY ((workflow_def_name), version) ); * *

CREATE TABLE IF NOT EXISTS conductor.workflow_defs_index( workflow_def_version_index text, * workflow_def_name_version text, workflow_def_index_value text,PRIMARY KEY * ((workflow_def_version_index), workflow_def_name_version) ); * *

CREATE TABLE IF NOT EXISTS conductor.task_definitions( task_defs text, task_def_name text, * task_definition text, PRIMARY KEY ((task_defs), task_def_name) ); * *

CREATE TABLE IF NOT EXISTS conductor.event_handlers( handlers text, event_handler_name text, * event_handler text, PRIMARY KEY ((handlers), event_handler_name) ); * *

CREATE TABLE IF NOT EXISTS conductor.event_executions( message_id text, event_handler_name * text, event_execution_id text, payload text, PRIMARY KEY ((message_id, event_handler_name), * event_execution_id) ); */ public abstract class CassandraBaseDAO { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraBaseDAO.class); private final ObjectMapper objectMapper; protected final Session session; protected final CassandraProperties properties; private boolean initialized = false; public CassandraBaseDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties) { this.session = session; this.objectMapper = objectMapper; this.properties = properties; init(); } protected static UUID toUUID(String uuidString, String message) { try { return UUID.fromString(uuidString); } catch (IllegalArgumentException iae) { throw new IllegalArgumentException(message + " " + uuidString, iae); } } private void init() { try { if (!initialized) { session.execute(getCreateKeyspaceStatement()); session.execute(getCreateWorkflowsTableStatement()); session.execute(getCreateTaskLookupTableStatement()); session.execute(getCreateTaskDefLimitTableStatement()); session.execute(getCreateWorkflowDefsTableStatement()); session.execute(getCreateWorkflowDefsIndexTableStatement()); session.execute(getCreateTaskDefsTableStatement()); session.execute(getCreateEventHandlersTableStatement()); session.execute(getCreateEventExecutionsTableStatement()); LOGGER.info( "{} initialization complete! Tables created!", getClass().getSimpleName()); initialized = true; } } catch (Exception e) { LOGGER.error("Error initializing and setting up keyspace and table in cassandra", e); throw e; } } private String getCreateKeyspaceStatement() { return SchemaBuilder.createKeyspace(properties.getKeyspace()) .ifNotExists() .with() .replication( ImmutableMap.of( "class", properties.getReplicationStrategy(), properties.getReplicationFactorKey(), properties.getReplicationFactorValue())) .durableWrites(true) .getQueryString(); } private String getCreateWorkflowsTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOWS) .ifNotExists() .addPartitionKey(WORKFLOW_ID_KEY, DataType.uuid()) .addPartitionKey(SHARD_ID_KEY, DataType.cint()) .addClusteringColumn(ENTITY_KEY, DataType.text()) .addClusteringColumn(TASK_ID_KEY, DataType.text()) .addColumn(PAYLOAD_KEY, DataType.text()) .addStaticColumn(TOTAL_TASKS_KEY, DataType.cint()) .addStaticColumn(TOTAL_PARTITIONS_KEY, DataType.cint()) .getQueryString(); } private String getCreateTaskLookupTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_LOOKUP) .ifNotExists() .addPartitionKey(TASK_ID_KEY, DataType.uuid()) .addColumn(WORKFLOW_ID_KEY, DataType.uuid()) .getQueryString(); } private String getCreateTaskDefLimitTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_DEF_LIMIT) .ifNotExists() .addPartitionKey(TASK_DEF_NAME_KEY, DataType.text()) .addClusteringColumn(TASK_ID_KEY, DataType.uuid()) .addColumn(WORKFLOW_ID_KEY, DataType.uuid()) .getQueryString(); } private String getCreateWorkflowDefsTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOW_DEFS) .ifNotExists() .addPartitionKey(WORKFLOW_DEF_NAME_KEY, DataType.text()) .addClusteringColumn(WORKFLOW_VERSION_KEY, DataType.cint()) .addColumn(WORKFLOW_DEFINITION_KEY, DataType.text()) .getQueryString(); } private String getCreateWorkflowDefsIndexTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOW_DEFS_INDEX) .ifNotExists() .addPartitionKey(WORKFLOW_DEF_INDEX_KEY, DataType.text()) .addClusteringColumn(WORKFLOW_DEF_NAME_VERSION_KEY, DataType.text()) .addColumn(WORKFLOW_DEF_INDEX_VALUE, DataType.text()) .getQueryString(); } private String getCreateTaskDefsTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_DEFS) .ifNotExists() .addPartitionKey(TASK_DEFS_KEY, DataType.text()) .addClusteringColumn(TASK_DEF_NAME_KEY, DataType.text()) .addColumn(TASK_DEFINITION_KEY, DataType.text()) .getQueryString(); } private String getCreateEventHandlersTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_EVENT_HANDLERS) .ifNotExists() .addPartitionKey(HANDLERS_KEY, DataType.text()) .addClusteringColumn(EVENT_HANDLER_NAME_KEY, DataType.text()) .addColumn(EVENT_HANDLER_KEY, DataType.text()) .getQueryString(); } private String getCreateEventExecutionsTableStatement() { return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_EVENT_EXECUTIONS) .ifNotExists() .addPartitionKey(MESSAGE_ID_KEY, DataType.text()) .addPartitionKey(EVENT_HANDLER_NAME_KEY, DataType.text()) .addClusteringColumn(EVENT_EXECUTION_ID_KEY, DataType.text()) .addColumn(PAYLOAD_KEY, DataType.text()) .getQueryString(); } String toJson(Object value) { try { return objectMapper.writeValueAsString(value); } catch (JsonProcessingException e) { throw new NonTransientException("Error serializing to json", e); } } T readValue(String json, Class clazz) { try { return objectMapper.readValue(json, clazz); } catch (IOException e) { throw new NonTransientException("Error de-serializing json", e); } } void recordCassandraDaoRequests(String action) { recordCassandraDaoRequests(action, "n/a", "n/a"); } void recordCassandraDaoRequests(String action, String taskType, String workflowType) { Monitors.recordDaoRequests(DAO_NAME, action, taskType, workflowType); } void recordCassandraDaoEventRequests(String action, String event) { Monitors.recordDaoEventRequests(DAO_NAME, action, event); } void recordCassandraDaoPayloadSize( String action, int size, String taskType, String workflowType) { Monitors.recordDaoPayloadSize(DAO_NAME, action, taskType, workflowType, size); } static class WorkflowMetadata { private int totalTasks; private int totalPartitions; public int getTotalTasks() { return totalTasks; } public void setTotalTasks(int totalTasks) { this.totalTasks = totalTasks; } public int getTotalPartitions() { return totalPartitions; } public void setTotalPartitions(int totalPartitions) { this.totalPartitions = totalPartitions; } } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraEventHandlerDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.cassandra.util.Statements; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.metrics.Monitors; import com.datastax.driver.core.PreparedStatement; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.exceptions.DriverException; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY; import static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY; @Trace public class CassandraEventHandlerDAO extends CassandraBaseDAO implements EventHandlerDAO { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraEventHandlerDAO.class); private static final String CLASS_NAME = CassandraEventHandlerDAO.class.getSimpleName(); private final PreparedStatement insertEventHandlerStatement; private final PreparedStatement selectAllEventHandlersStatement; private final PreparedStatement deleteEventHandlerStatement; public CassandraEventHandlerDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements) { super(session, objectMapper, properties); insertEventHandlerStatement = session.prepare(statements.getInsertEventHandlerStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); selectAllEventHandlersStatement = session.prepare(statements.getSelectAllEventHandlersStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); deleteEventHandlerStatement = session.prepare(statements.getDeleteEventHandlerStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); } @Override public void addEventHandler(EventHandler eventHandler) { insertOrUpdateEventHandler(eventHandler); } @Override public void updateEventHandler(EventHandler eventHandler) { insertOrUpdateEventHandler(eventHandler); } @Override public void removeEventHandler(String name) { try { recordCassandraDaoRequests("removeEventHandler"); session.execute(deleteEventHandlerStatement.bind(name)); } catch (Exception e) { Monitors.error(CLASS_NAME, "removeEventHandler"); String errorMsg = String.format("Failed to remove event handler: %s", name); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public List getAllEventHandlers() { return getAllEventHandlersFromDB(); } @Override public List getEventHandlersForEvent(String event, boolean activeOnly) { if (activeOnly) { return getAllEventHandlers().stream() .filter(eventHandler -> eventHandler.getEvent().equals(event)) .filter(EventHandler::isActive) .collect(Collectors.toList()); } else { return getAllEventHandlers().stream() .filter(eventHandler -> eventHandler.getEvent().equals(event)) .collect(Collectors.toList()); } } @SuppressWarnings("unchecked") private List getAllEventHandlersFromDB() { try { ResultSet resultSet = session.execute(selectAllEventHandlersStatement.bind(HANDLERS_KEY)); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("No event handlers were found."); return Collections.EMPTY_LIST; } return rows.stream() .map(row -> readValue(row.getString(EVENT_HANDLER_KEY), EventHandler.class)) .collect(Collectors.toList()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getAllEventHandlersFromDB"); String errorMsg = "Failed to get all event handlers"; LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } private void insertOrUpdateEventHandler(EventHandler eventHandler) { try { String handler = toJson(eventHandler); session.execute(insertEventHandlerStatement.bind(eventHandler.getName(), handler)); recordCassandraDaoRequests("storeEventHandler"); recordCassandraDaoPayloadSize("storeEventHandler", handler.length(), "n/a", "n/a"); } catch (DriverException e) { Monitors.error(CLASS_NAME, "insertOrUpdateEventHandler"); String errorMsg = String.format( "Error creating/updating event handler: %s/%s", eventHandler.getName(), eventHandler.getEvent()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraExecutionDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.cassandra.util.Statements; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.ConcurrentExecutionLimitDAO; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.datastax.driver.core.*; import com.datastax.driver.core.exceptions.DriverException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import static com.netflix.conductor.cassandra.util.Constants.*; @Trace public class CassandraExecutionDAO extends CassandraBaseDAO implements ExecutionDAO, ConcurrentExecutionLimitDAO { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraExecutionDAO.class); private static final String CLASS_NAME = CassandraExecutionDAO.class.getSimpleName(); protected final PreparedStatement insertWorkflowStatement; protected final PreparedStatement insertTaskStatement; protected final PreparedStatement insertEventExecutionStatement; protected final PreparedStatement selectTotalStatement; protected final PreparedStatement selectTaskStatement; protected final PreparedStatement selectWorkflowStatement; protected final PreparedStatement selectWorkflowWithTasksStatement; protected final PreparedStatement selectTaskLookupStatement; protected final PreparedStatement selectTasksFromTaskDefLimitStatement; protected final PreparedStatement selectEventExecutionsStatement; protected final PreparedStatement updateWorkflowStatement; protected final PreparedStatement updateTotalTasksStatement; protected final PreparedStatement updateTotalPartitionsStatement; protected final PreparedStatement updateTaskLookupStatement; protected final PreparedStatement updateTaskDefLimitStatement; protected final PreparedStatement updateEventExecutionStatement; protected final PreparedStatement deleteWorkflowStatement; protected final PreparedStatement deleteTaskStatement; protected final PreparedStatement deleteTaskLookupStatement; protected final PreparedStatement deleteTaskDefLimitStatement; protected final PreparedStatement deleteEventExecutionStatement; protected final int eventExecutionsTTL; public CassandraExecutionDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements) { super(session, objectMapper, properties); eventExecutionsTTL = (int) properties.getEventExecutionPersistenceTtl().getSeconds(); this.insertWorkflowStatement = session.prepare(statements.getInsertWorkflowStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.insertTaskStatement = session.prepare(statements.getInsertTaskStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.insertEventExecutionStatement = session.prepare(statements.getInsertEventExecutionStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.selectTotalStatement = session.prepare(statements.getSelectTotalStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectTaskStatement = session.prepare(statements.getSelectTaskStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectWorkflowStatement = session.prepare(statements.getSelectWorkflowStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectWorkflowWithTasksStatement = session.prepare(statements.getSelectWorkflowWithTasksStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectTaskLookupStatement = session.prepare(statements.getSelectTaskFromLookupTableStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectTasksFromTaskDefLimitStatement = session.prepare(statements.getSelectTasksFromTaskDefLimitStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectEventExecutionsStatement = session.prepare( statements .getSelectAllEventExecutionsForMessageFromEventExecutionsStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.updateWorkflowStatement = session.prepare(statements.getUpdateWorkflowStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.updateTotalTasksStatement = session.prepare(statements.getUpdateTotalTasksStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.updateTotalPartitionsStatement = session.prepare(statements.getUpdateTotalPartitionsStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.updateTaskLookupStatement = session.prepare(statements.getUpdateTaskLookupStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.updateTaskDefLimitStatement = session.prepare(statements.getUpdateTaskDefLimitStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.updateEventExecutionStatement = session.prepare(statements.getUpdateEventExecutionStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteWorkflowStatement = session.prepare(statements.getDeleteWorkflowStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteTaskStatement = session.prepare(statements.getDeleteTaskStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteTaskLookupStatement = session.prepare(statements.getDeleteTaskLookupStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteTaskDefLimitStatement = session.prepare(statements.getDeleteTaskDefLimitStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteEventExecutionStatement = session.prepare(statements.getDeleteEventExecutionsStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); } @Override public List getPendingTasksByWorkflow(String taskName, String workflowId) { List tasks = getTasksForWorkflow(workflowId); return tasks.stream() .filter(task -> taskName.equals(task.getTaskType())) .filter(task -> TaskModel.Status.IN_PROGRESS.equals(task.getStatus())) .collect(Collectors.toList()); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getTasks(String taskType, String startKey, int count) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * Inserts tasks into the Cassandra datastore. Note: Creates the task_id to workflow_id * mapping in the task_lookup table first. Once this succeeds, inserts the tasks into the * workflows table. Tasks belonging to the same shard are created using batch statements. * * @param tasks tasks to be created */ @Override public List createTasks(List tasks) { validateTasks(tasks); String workflowId = tasks.get(0).getWorkflowInstanceId(); UUID workflowUUID = toUUID(workflowId, "Invalid workflow id"); try { WorkflowMetadata workflowMetadata = getWorkflowMetadata(workflowId); int totalTasks = workflowMetadata.getTotalTasks() + tasks.size(); // TODO: write into multiple shards based on number of tasks // update the task_lookup table tasks.forEach( task -> { if (task.getScheduledTime() == 0) { task.setScheduledTime(System.currentTimeMillis()); } session.execute( updateTaskLookupStatement.bind( workflowUUID, toUUID(task.getTaskId(), "Invalid task id"))); }); // update all the tasks in the workflow using batch BatchStatement batchStatement = new BatchStatement(); tasks.forEach( task -> { String taskPayload = toJson(task); batchStatement.add( insertTaskStatement.bind( workflowUUID, DEFAULT_SHARD_ID, task.getTaskId(), taskPayload)); recordCassandraDaoRequests( "createTask", task.getTaskType(), task.getWorkflowType()); recordCassandraDaoPayloadSize( "createTask", taskPayload.length(), task.getTaskType(), task.getWorkflowType()); }); batchStatement.add( updateTotalTasksStatement.bind(totalTasks, workflowUUID, DEFAULT_SHARD_ID)); session.execute(batchStatement); // update the total tasks and partitions for the workflow session.execute( updateTotalPartitionsStatement.bind( DEFAULT_TOTAL_PARTITIONS, totalTasks, workflowUUID)); return tasks; } catch (DriverException e) { Monitors.error(CLASS_NAME, "createTasks"); String errorMsg = String.format( "Error creating %d tasks for workflow: %s", tasks.size(), workflowId); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public void updateTask(TaskModel task) { try { // TODO: calculate the shard number the task belongs to String taskPayload = toJson(task); recordCassandraDaoRequests("updateTask", task.getTaskType(), task.getWorkflowType()); recordCassandraDaoPayloadSize( "updateTask", taskPayload.length(), task.getTaskType(), task.getWorkflowType()); session.execute( insertTaskStatement.bind( UUID.fromString(task.getWorkflowInstanceId()), DEFAULT_SHARD_ID, task.getTaskId(), taskPayload)); if (task.getTaskDefinition().isPresent() && task.getTaskDefinition().get().concurrencyLimit() > 0) { if (task.getStatus().isTerminal()) { removeTaskFromLimit(task); } else if (task.getStatus() == TaskModel.Status.IN_PROGRESS) { addTaskToLimit(task); } } } catch (DriverException e) { Monitors.error(CLASS_NAME, "updateTask"); String errorMsg = String.format( "Error updating task: %s in workflow: %s", task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public boolean exceedsLimit(TaskModel task) { Optional taskDefinition = task.getTaskDefinition(); if (taskDefinition.isEmpty()) { return false; } int limit = taskDefinition.get().concurrencyLimit(); if (limit <= 0) { return false; } try { recordCassandraDaoRequests( "selectTaskDefLimit", task.getTaskType(), task.getWorkflowType()); ResultSet resultSet = session.execute( selectTasksFromTaskDefLimitStatement.bind(task.getTaskDefName())); List taskIds = resultSet.all().stream() .map(row -> row.getUUID(TASK_ID_KEY).toString()) .collect(Collectors.toList()); long current = taskIds.size(); if (!taskIds.contains(task.getTaskId()) && current >= limit) { LOGGER.info( "Task execution count limited. task - {}:{}, limit: {}, current: {}", task.getTaskId(), task.getTaskDefName(), limit, current); Monitors.recordTaskConcurrentExecutionLimited(task.getTaskDefName(), limit); return true; } } catch (DriverException e) { Monitors.error(CLASS_NAME, "exceedsLimit"); String errorMsg = String.format( "Failed to get in progress limit - %s:%s in workflow :%s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } return false; } @Override public boolean removeTask(String taskId) { TaskModel task = getTask(taskId); if (task == null) { LOGGER.warn("No such task found by id {}", taskId); return false; } return removeTask(task); } @Override public TaskModel getTask(String taskId) { try { String workflowId = lookupWorkflowIdFromTaskId(taskId); if (workflowId == null) { return null; } // TODO: implement for query against multiple shards ResultSet resultSet = session.execute( selectTaskStatement.bind( UUID.fromString(workflowId), DEFAULT_SHARD_ID, taskId)); return Optional.ofNullable(resultSet.one()) .map( row -> { String taskRow = row.getString(PAYLOAD_KEY); TaskModel task = readValue(taskRow, TaskModel.class); recordCassandraDaoRequests( "getTask", task.getTaskType(), task.getWorkflowType()); recordCassandraDaoPayloadSize( "getTask", taskRow.length(), task.getTaskType(), task.getWorkflowType()); return task; }) .orElse(null); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getTask"); String errorMsg = String.format("Error getting task by id: %s", taskId); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @Override public List getTasks(List taskIds) { Preconditions.checkNotNull(taskIds); Preconditions.checkArgument(taskIds.size() > 0, "Task ids list cannot be empty"); String workflowId = lookupWorkflowIdFromTaskId(taskIds.get(0)); if (workflowId == null) { return null; } return getWorkflow(workflowId, true).getTasks().stream() .filter(task -> taskIds.contains(task.getTaskId())) .collect(Collectors.toList()); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getPendingTasksForTaskType(String taskType) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } @Override public List getTasksForWorkflow(String workflowId) { return getWorkflow(workflowId, true).getTasks(); } @Override public String createWorkflow(WorkflowModel workflow) { try { List tasks = workflow.getTasks(); workflow.setTasks(new LinkedList<>()); String payload = toJson(workflow); recordCassandraDaoRequests("createWorkflow", "n/a", workflow.getWorkflowName()); recordCassandraDaoPayloadSize( "createWorkflow", payload.length(), "n/a", workflow.getWorkflowName()); session.execute( insertWorkflowStatement.bind( UUID.fromString(workflow.getWorkflowId()), 1, "", payload, 0, 1)); workflow.setTasks(tasks); return workflow.getWorkflowId(); } catch (DriverException e) { Monitors.error(CLASS_NAME, "createWorkflow"); String errorMsg = String.format("Error creating workflow: %s", workflow.getWorkflowId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public String updateWorkflow(WorkflowModel workflow) { try { List tasks = workflow.getTasks(); workflow.setTasks(new LinkedList<>()); String payload = toJson(workflow); recordCassandraDaoRequests("updateWorkflow", "n/a", workflow.getWorkflowName()); recordCassandraDaoPayloadSize( "updateWorkflow", payload.length(), "n/a", workflow.getWorkflowName()); session.execute( updateWorkflowStatement.bind( payload, UUID.fromString(workflow.getWorkflowId()))); workflow.setTasks(tasks); return workflow.getWorkflowId(); } catch (DriverException e) { Monitors.error(CLASS_NAME, "updateWorkflow"); String errorMsg = String.format("Failed to update workflow: %s", workflow.getWorkflowId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @Override public boolean removeWorkflow(String workflowId) { WorkflowModel workflow = getWorkflow(workflowId, true); boolean removed = false; // TODO: calculate number of shards and iterate if (workflow != null) { try { recordCassandraDaoRequests("removeWorkflow", "n/a", workflow.getWorkflowName()); ResultSet resultSet = session.execute( deleteWorkflowStatement.bind( UUID.fromString(workflowId), DEFAULT_SHARD_ID)); removed = resultSet.wasApplied(); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeWorkflow"); String errorMsg = String.format("Failed to remove workflow: %s", workflowId); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } workflow.getTasks().forEach(this::removeTaskLookup); } return removed; } /** * This is a dummy implementation and this feature is not yet implemented for Cassandra backed * Conductor */ @Override public boolean removeWorkflowWithExpiry(String workflowId, int ttlSeconds) { throw new UnsupportedOperationException( "This method is not currently implemented in CassandraExecutionDAO. Please use RedisDAO mode instead now for using TTLs."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public void removeFromPendingWorkflow(String workflowType, String workflowId) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } @Override public WorkflowModel getWorkflow(String workflowId) { return getWorkflow(workflowId, true); } @Override public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) { UUID workflowUUID = toUUID(workflowId, "Invalid workflow id"); try { WorkflowModel workflow = null; ResultSet resultSet; if (includeTasks) { resultSet = session.execute( selectWorkflowWithTasksStatement.bind( workflowUUID, DEFAULT_SHARD_ID)); List tasks = new ArrayList<>(); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("Workflow {} not found in datastore", workflowId); return null; } for (Row row : rows) { String entityKey = row.getString(ENTITY_KEY); if (ENTITY_TYPE_WORKFLOW.equals(entityKey)) { workflow = readValue(row.getString(PAYLOAD_KEY), WorkflowModel.class); } else if (ENTITY_TYPE_TASK.equals(entityKey)) { TaskModel task = readValue(row.getString(PAYLOAD_KEY), TaskModel.class); tasks.add(task); } else { throw new NonTransientException( String.format( "Invalid row with entityKey: %s found in datastore for workflow: %s", entityKey, workflowId)); } } if (workflow != null) { recordCassandraDaoRequests("getWorkflow", "n/a", workflow.getWorkflowName()); tasks.sort(Comparator.comparingInt(TaskModel::getSeq)); workflow.setTasks(tasks); } } else { resultSet = session.execute(selectWorkflowStatement.bind(workflowUUID)); workflow = Optional.ofNullable(resultSet.one()) .map( row -> { WorkflowModel wf = readValue( row.getString(PAYLOAD_KEY), WorkflowModel.class); recordCassandraDaoRequests( "getWorkflow", "n/a", wf.getWorkflowName()); return wf; }) .orElse(null); } return workflow; } catch (DriverException e) { Monitors.error(CLASS_NAME, "getWorkflow"); String errorMsg = String.format("Failed to get workflow: %s", workflowId); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getRunningWorkflowIds(String workflowName, int version) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getPendingWorkflowsByType(String workflowName, int version) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public long getPendingWorkflowCount(String workflowName) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public long getInProgressTaskCount(String taskDefName) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getWorkflowsByType( String workflowName, Long startTime, Long endTime) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor */ @Override public List getWorkflowsByCorrelationId( String workflowName, String correlationId, boolean includeTasks) { throw new UnsupportedOperationException( "This method is not implemented in CassandraExecutionDAO. Please use ExecutionDAOFacade instead."); } @Override public boolean canSearchAcrossWorkflows() { return false; } @Override public boolean addEventExecution(EventExecution eventExecution) { try { String jsonPayload = toJson(eventExecution); recordCassandraDaoEventRequests("addEventExecution", eventExecution.getEvent()); recordCassandraDaoPayloadSize( "addEventExecution", jsonPayload.length(), eventExecution.getEvent(), "n/a"); return session.execute( insertEventExecutionStatement.bind( eventExecution.getMessageId(), eventExecution.getName(), eventExecution.getId(), jsonPayload)) .wasApplied(); } catch (DriverException e) { Monitors.error(CLASS_NAME, "addEventExecution"); String errorMsg = String.format( "Failed to add event execution for event: %s, handler: %s", eventExecution.getEvent(), eventExecution.getName()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @Override public void updateEventExecution(EventExecution eventExecution) { try { String jsonPayload = toJson(eventExecution); recordCassandraDaoEventRequests("updateEventExecution", eventExecution.getEvent()); recordCassandraDaoPayloadSize( "updateEventExecution", jsonPayload.length(), eventExecution.getEvent(), "n/a"); session.execute( updateEventExecutionStatement.bind( eventExecutionsTTL, jsonPayload, eventExecution.getMessageId(), eventExecution.getName(), eventExecution.getId())); } catch (DriverException e) { Monitors.error(CLASS_NAME, "updateEventExecution"); String errorMsg = String.format( "Failed to update event execution for event: %s, handler: %s", eventExecution.getEvent(), eventExecution.getName()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @Override public void removeEventExecution(EventExecution eventExecution) { try { recordCassandraDaoEventRequests("removeEventExecution", eventExecution.getEvent()); session.execute( deleteEventExecutionStatement.bind( eventExecution.getMessageId(), eventExecution.getName(), eventExecution.getId())); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeEventExecution"); String errorMsg = String.format( "Failed to remove event execution for event: %s, handler: %s", eventExecution.getEvent(), eventExecution.getName()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @VisibleForTesting List getEventExecutions( String eventHandlerName, String eventName, String messageId) { try { return session .execute(selectEventExecutionsStatement.bind(messageId, eventHandlerName)) .all() .stream() .filter(row -> !row.isNull(PAYLOAD_KEY)) .map(row -> readValue(row.getString(PAYLOAD_KEY), EventExecution.class)) .collect(Collectors.toList()); } catch (DriverException e) { String errorMsg = String.format( "Failed to fetch event executions for event: %s, handler: %s", eventName, eventHandlerName); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @Override public void addTaskToLimit(TaskModel task) { try { recordCassandraDaoRequests( "addTaskToLimit", task.getTaskType(), task.getWorkflowType()); session.execute( updateTaskDefLimitStatement.bind( UUID.fromString(task.getWorkflowInstanceId()), task.getTaskDefName(), UUID.fromString(task.getTaskId()))); } catch (DriverException e) { Monitors.error(CLASS_NAME, "addTaskToLimit"); String errorMsg = String.format( "Error updating taskDefLimit for task - %s:%s in workflow: %s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public void removeTaskFromLimit(TaskModel task) { try { recordCassandraDaoRequests( "removeTaskFromLimit", task.getTaskType(), task.getWorkflowType()); session.execute( deleteTaskDefLimitStatement.bind( task.getTaskDefName(), UUID.fromString(task.getTaskId()))); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeTaskFromLimit"); String errorMsg = String.format( "Error updating taskDefLimit for task - %s:%s in workflow: %s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } protected boolean removeTask(TaskModel task) { // TODO: calculate shard number based on seq and maxTasksPerShard try { // get total tasks for this workflow WorkflowMetadata workflowMetadata = getWorkflowMetadata(task.getWorkflowInstanceId()); int totalTasks = workflowMetadata.getTotalTasks(); // remove from task_lookup table removeTaskLookup(task); recordCassandraDaoRequests("removeTask", task.getTaskType(), task.getWorkflowType()); // delete task from workflows table and decrement total tasks by 1 BatchStatement batchStatement = new BatchStatement(); batchStatement.add( deleteTaskStatement.bind( UUID.fromString(task.getWorkflowInstanceId()), DEFAULT_SHARD_ID, task.getTaskId())); batchStatement.add( updateTotalTasksStatement.bind( totalTasks - 1, UUID.fromString(task.getWorkflowInstanceId()), DEFAULT_SHARD_ID)); ResultSet resultSet = session.execute(batchStatement); if (task.getTaskDefinition().isPresent() && task.getTaskDefinition().get().concurrencyLimit() > 0) { removeTaskFromLimit(task); } return resultSet.wasApplied(); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeTask"); String errorMsg = String.format("Failed to remove task: %s", task.getTaskId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } protected void removeTaskLookup(TaskModel task) { try { recordCassandraDaoRequests( "removeTaskLookup", task.getTaskType(), task.getWorkflowType()); if (task.getTaskDefinition().isPresent() && task.getTaskDefinition().get().concurrencyLimit() > 0) { removeTaskFromLimit(task); } session.execute(deleteTaskLookupStatement.bind(UUID.fromString(task.getTaskId()))); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeTaskLookup"); String errorMsg = String.format("Failed to remove task lookup: %s", task.getTaskId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } @VisibleForTesting void validateTasks(List tasks) { Preconditions.checkNotNull(tasks, "Tasks object cannot be null"); Preconditions.checkArgument(!tasks.isEmpty(), "Tasks object cannot be empty"); tasks.forEach( task -> { Preconditions.checkNotNull(task, "task object cannot be null"); Preconditions.checkNotNull(task.getTaskId(), "Task id cannot be null"); Preconditions.checkNotNull( task.getWorkflowInstanceId(), "Workflow instance id cannot be null"); Preconditions.checkNotNull( task.getReferenceTaskName(), "Task reference name cannot be null"); }); String workflowId = tasks.get(0).getWorkflowInstanceId(); Optional optionalTask = tasks.stream() .filter(task -> !workflowId.equals(task.getWorkflowInstanceId())) .findAny(); if (optionalTask.isPresent()) { throw new NonTransientException( "Tasks of multiple workflows cannot be created/updated simultaneously"); } } @VisibleForTesting WorkflowMetadata getWorkflowMetadata(String workflowId) { ResultSet resultSet = session.execute(selectTotalStatement.bind(UUID.fromString(workflowId))); recordCassandraDaoRequests("getWorkflowMetadata"); return Optional.ofNullable(resultSet.one()) .map( row -> { WorkflowMetadata workflowMetadata = new WorkflowMetadata(); workflowMetadata.setTotalTasks(row.getInt(TOTAL_TASKS_KEY)); workflowMetadata.setTotalPartitions(row.getInt(TOTAL_PARTITIONS_KEY)); return workflowMetadata; }) .orElseThrow( () -> new NotFoundException( "Workflow with id: %s not found in data store", workflowId)); } @VisibleForTesting String lookupWorkflowIdFromTaskId(String taskId) { UUID taskUUID = toUUID(taskId, "Invalid task id"); try { ResultSet resultSet = session.execute(selectTaskLookupStatement.bind(taskUUID)); return Optional.ofNullable(resultSet.one()) .map(row -> row.getUUID(WORKFLOW_ID_KEY).toString()) .orElse(null); } catch (DriverException e) { Monitors.error(CLASS_NAME, "lookupWorkflowIdFromTaskId"); String errorMsg = String.format("Failed to lookup workflowId from taskId: %s", taskId); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraMetadataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.PriorityQueue; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.cassandra.config.CassandraProperties; import com.netflix.conductor.cassandra.util.Statements; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import com.datastax.driver.core.PreparedStatement; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.exceptions.DriverException; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY; import static com.netflix.conductor.common.metadata.tasks.TaskDef.ONE_HOUR; @Trace public class CassandraMetadataDAO extends CassandraBaseDAO implements MetadataDAO { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraMetadataDAO.class); private static final String CLASS_NAME = CassandraMetadataDAO.class.getSimpleName(); private static final String INDEX_DELIMITER = "/"; private final PreparedStatement insertWorkflowDefStatement; private final PreparedStatement insertWorkflowDefVersionIndexStatement; private final PreparedStatement insertTaskDefStatement; private final PreparedStatement selectWorkflowDefStatement; private final PreparedStatement selectAllWorkflowDefVersionsByNameStatement; private final PreparedStatement selectAllWorkflowDefsStatement; private final PreparedStatement selectAllWorkflowDefsLatestVersionsStatement; private final PreparedStatement selectTaskDefStatement; private final PreparedStatement selectAllTaskDefsStatement; private final PreparedStatement updateWorkflowDefStatement; private final PreparedStatement deleteWorkflowDefStatement; private final PreparedStatement deleteWorkflowDefIndexStatement; private final PreparedStatement deleteTaskDefStatement; public CassandraMetadataDAO( Session session, ObjectMapper objectMapper, CassandraProperties properties, Statements statements) { super(session, objectMapper, properties); this.insertWorkflowDefStatement = session.prepare(statements.getInsertWorkflowDefStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.insertWorkflowDefVersionIndexStatement = session.prepare(statements.getInsertWorkflowDefVersionIndexStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.insertTaskDefStatement = session.prepare(statements.getInsertTaskDefStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.selectWorkflowDefStatement = session.prepare(statements.getSelectWorkflowDefStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectAllWorkflowDefVersionsByNameStatement = session.prepare(statements.getSelectAllWorkflowDefVersionsByNameStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectAllWorkflowDefsStatement = session.prepare(statements.getSelectAllWorkflowDefsStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectAllWorkflowDefsLatestVersionsStatement = session.prepare(statements.getSelectAllWorkflowDefsLatestVersionsStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectTaskDefStatement = session.prepare(statements.getSelectTaskDefStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.selectAllTaskDefsStatement = session.prepare(statements.getSelectAllTaskDefsStatement()) .setConsistencyLevel(properties.getReadConsistencyLevel()); this.updateWorkflowDefStatement = session.prepare(statements.getUpdateWorkflowDefStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteWorkflowDefStatement = session.prepare(statements.getDeleteWorkflowDefStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteWorkflowDefIndexStatement = session.prepare(statements.getDeleteWorkflowDefIndexStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); this.deleteTaskDefStatement = session.prepare(statements.getDeleteTaskDefStatement()) .setConsistencyLevel(properties.getWriteConsistencyLevel()); } @Override public TaskDef createTaskDef(TaskDef taskDef) { return insertOrUpdateTaskDef(taskDef); } @Override public TaskDef updateTaskDef(TaskDef taskDef) { return insertOrUpdateTaskDef(taskDef); } @Override public TaskDef getTaskDef(String name) { return getTaskDefFromDB(name); } @Override public List getAllTaskDefs() { return getAllTaskDefsFromDB(); } @Override public void removeTaskDef(String name) { try { recordCassandraDaoRequests("removeTaskDef"); session.execute(deleteTaskDefStatement.bind(name)); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeTaskDef"); String errorMsg = String.format("Failed to remove task definition: %s", name); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public void createWorkflowDef(WorkflowDef workflowDef) { try { String workflowDefinition = toJson(workflowDef); if (!session.execute( insertWorkflowDefStatement.bind( workflowDef.getName(), workflowDef.getVersion(), workflowDefinition)) .wasApplied()) { throw new ConflictException( "Workflow: %s, version: %s already exists!", workflowDef.getName(), workflowDef.getVersion()); } String workflowDefIndex = getWorkflowDefIndexValue(workflowDef.getName(), workflowDef.getVersion()); session.execute( insertWorkflowDefVersionIndexStatement.bind( workflowDefIndex, workflowDefIndex)); recordCassandraDaoRequests("createWorkflowDef"); recordCassandraDaoPayloadSize( "createWorkflowDef", workflowDefinition.length(), "n/a", workflowDef.getName()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "createWorkflowDef"); String errorMsg = String.format( "Error creating workflow definition: %s/%d", workflowDef.getName(), workflowDef.getVersion()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public void updateWorkflowDef(WorkflowDef workflowDef) { try { String workflowDefinition = toJson(workflowDef); session.execute( updateWorkflowDefStatement.bind( workflowDefinition, workflowDef.getName(), workflowDef.getVersion())); String workflowDefIndex = getWorkflowDefIndexValue(workflowDef.getName(), workflowDef.getVersion()); session.execute( insertWorkflowDefVersionIndexStatement.bind( workflowDefIndex, workflowDefIndex)); recordCassandraDaoRequests("updateWorkflowDef"); recordCassandraDaoPayloadSize( "updateWorkflowDef", workflowDefinition.length(), "n/a", workflowDef.getName()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "updateWorkflowDef"); String errorMsg = String.format( "Error updating workflow definition: %s/%d", workflowDef.getName(), workflowDef.getVersion()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public Optional getLatestWorkflowDef(String name) { List workflowDefList = getAllWorkflowDefVersions(name); if (workflowDefList != null && workflowDefList.size() > 0) { workflowDefList.sort(Comparator.comparingInt(WorkflowDef::getVersion)); return Optional.of(workflowDefList.get(workflowDefList.size() - 1)); } return Optional.empty(); } @Override public Optional getWorkflowDef(String name, int version) { try { recordCassandraDaoRequests("getWorkflowDef"); ResultSet resultSet = session.execute(selectWorkflowDefStatement.bind(name, version)); WorkflowDef workflowDef = Optional.ofNullable(resultSet.one()) .map( row -> readValue( row.getString(WORKFLOW_DEFINITION_KEY), WorkflowDef.class)) .orElse(null); return Optional.ofNullable(workflowDef); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getTaskDef"); String errorMsg = String.format("Error fetching workflow def: %s/%d", name, version); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public void removeWorkflowDef(String name, Integer version) { try { session.execute(deleteWorkflowDefStatement.bind(name, version)); session.execute( deleteWorkflowDefIndexStatement.bind( WORKFLOW_DEF_INDEX_KEY, getWorkflowDefIndexValue(name, version))); } catch (DriverException e) { Monitors.error(CLASS_NAME, "removeWorkflowDef"); String errorMsg = String.format("Failed to remove workflow definition: %s/%d", name, version); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @SuppressWarnings("unchecked") @Override public List getAllWorkflowDefs() { try { ResultSet resultSet = session.execute(selectAllWorkflowDefsStatement.bind(WORKFLOW_DEF_INDEX_KEY)); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("No workflow definitions were found."); return Collections.EMPTY_LIST; } return rows.stream() .map( row -> { String defNameVersion = row.getString(WORKFLOW_DEF_NAME_VERSION_KEY); var nameVersion = getWorkflowNameAndVersion(defNameVersion); return getWorkflowDef(nameVersion.getLeft(), nameVersion.getRight()) .orElse(null); }) .filter(Objects::nonNull) .collect(Collectors.toList()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getAllWorkflowDefs"); String errorMsg = "Error retrieving all workflow defs"; LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @Override public List getAllWorkflowDefsLatestVersions() { try { ResultSet resultSet = session.execute( selectAllWorkflowDefsLatestVersionsStatement.bind( WORKFLOW_DEF_INDEX_KEY)); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("No workflow definitions were found."); return Collections.EMPTY_LIST; } Map> allWorkflowDefs = new HashMap<>(); for (Row row : rows) { String defNameVersion = row.getString(WORKFLOW_DEF_NAME_VERSION_KEY); var nameVersion = getWorkflowNameAndVersion(defNameVersion); WorkflowDef def = getWorkflowDef(nameVersion.getLeft(), nameVersion.getRight()).orElse(null); if (def == null) { continue; } if (allWorkflowDefs.get(def.getName()) == null) { allWorkflowDefs.put( def.getName(), new PriorityQueue<>( (WorkflowDef w1, WorkflowDef w2) -> Integer.compare(w2.getVersion(), w1.getVersion()))); } allWorkflowDefs.get(def.getName()).add(def); } return allWorkflowDefs.values().stream() .map(PriorityQueue::poll) .collect(Collectors.toList()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getAllWorkflowDefsLatestVersions"); String errorMsg = "Error retrieving all workflow defs latest versions"; LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } private TaskDef getTaskDefFromDB(String name) { try { ResultSet resultSet = session.execute(selectTaskDefStatement.bind(name)); recordCassandraDaoRequests("getTaskDef", name, null); return Optional.ofNullable(resultSet.one()).map(this::setDefaults).orElse(null); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getTaskDef"); String errorMsg = String.format("Failed to get task def: %s", name); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } @SuppressWarnings("unchecked") private List getAllTaskDefsFromDB() { try { ResultSet resultSet = session.execute(selectAllTaskDefsStatement.bind(TASK_DEFS_KEY)); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("No task definitions were found."); return Collections.EMPTY_LIST; } return rows.stream().map(this::setDefaults).collect(Collectors.toList()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getAllTaskDefs"); String errorMsg = "Failed to get all task defs"; LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } private List getAllWorkflowDefVersions(String name) { try { ResultSet resultSet = session.execute(selectAllWorkflowDefVersionsByNameStatement.bind(name)); recordCassandraDaoRequests("getAllWorkflowDefVersions", "n/a", name); List rows = resultSet.all(); if (rows.size() == 0) { LOGGER.info("Not workflow definitions were found for : {}", name); return null; } return rows.stream() .map( row -> readValue( row.getString(WORKFLOW_DEFINITION_KEY), WorkflowDef.class)) .collect(Collectors.toList()); } catch (DriverException e) { Monitors.error(CLASS_NAME, "getAllWorkflowDefVersions"); String errorMsg = String.format("Failed to get workflows defs for : %s", name); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } private TaskDef insertOrUpdateTaskDef(TaskDef taskDef) { try { String taskDefinition = toJson(taskDef); session.execute(insertTaskDefStatement.bind(taskDef.getName(), taskDefinition)); recordCassandraDaoRequests("storeTaskDef"); recordCassandraDaoPayloadSize( "storeTaskDef", taskDefinition.length(), taskDef.getName(), "n/a"); } catch (DriverException e) { Monitors.error(CLASS_NAME, "insertOrUpdateTaskDef"); String errorMsg = String.format("Error creating/updating task definition: %s", taskDef.getName()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } return taskDef; } @VisibleForTesting String getWorkflowDefIndexValue(String name, int version) { return name + INDEX_DELIMITER + version; } @VisibleForTesting ImmutablePair getWorkflowNameAndVersion(String nameVersionStr) { int lastIndexOfDelimiter = nameVersionStr.lastIndexOf(INDEX_DELIMITER); if (lastIndexOfDelimiter == -1) { throw new IllegalStateException( nameVersionStr + " is not in the 'workflowName" + INDEX_DELIMITER + "version' pattern."); } String workflowName = nameVersionStr.substring(0, lastIndexOfDelimiter); String versionStr = nameVersionStr.substring(lastIndexOfDelimiter + 1); try { return new ImmutablePair<>(workflowName, Integer.parseInt(versionStr)); } catch (NumberFormatException e) { throw new IllegalStateException( versionStr + " in " + nameVersionStr + " is not a valid number."); } } private TaskDef setDefaults(Row row) { TaskDef taskDef = readValue(row.getString(TASK_DEFINITION_KEY), TaskDef.class); if (taskDef != null && taskDef.getResponseTimeoutSeconds() == 0) { taskDef.setResponseTimeoutSeconds( taskDef.getTimeoutSeconds() == 0 ? ONE_HOUR : taskDef.getTimeoutSeconds() - 1); } return taskDef; } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraPollDataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao; import java.util.List; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.dao.PollDataDAO; /** * This is a dummy implementation and this feature is not implemented for Cassandra backed * Conductor. */ public class CassandraPollDataDAO implements PollDataDAO { @Override public void updateLastPollData(String taskDefName, String domain, String workerId) { throw new UnsupportedOperationException( "This method is not implemented in CassandraPollDataDAO. Please use ExecutionDAOFacade instead."); } @Override public PollData getPollData(String taskDefName, String domain) { throw new UnsupportedOperationException( "This method is not implemented in CassandraPollDataDAO. Please use ExecutionDAOFacade instead."); } @Override public List getPollData(String taskDefName) { throw new UnsupportedOperationException( "This method is not implemented in CassandraPollDataDAO. Please use ExecutionDAOFacade instead."); } } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/util/Constants.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.util; public interface Constants { String DAO_NAME = "cassandra"; String TABLE_WORKFLOWS = "workflows"; String TABLE_TASK_LOOKUP = "task_lookup"; String TABLE_TASK_DEF_LIMIT = "task_def_limit"; String TABLE_WORKFLOW_DEFS = "workflow_definitions"; String TABLE_WORKFLOW_DEFS_INDEX = "workflow_defs_index"; String TABLE_TASK_DEFS = "task_definitions"; String TABLE_EVENT_HANDLERS = "event_handlers"; String TABLE_EVENT_EXECUTIONS = "event_executions"; String WORKFLOW_ID_KEY = "workflow_id"; String SHARD_ID_KEY = "shard_id"; String TASK_ID_KEY = "task_id"; String ENTITY_KEY = "entity"; String PAYLOAD_KEY = "payload"; String TOTAL_TASKS_KEY = "total_tasks"; String TOTAL_PARTITIONS_KEY = "total_partitions"; String TASK_DEF_NAME_KEY = "task_def_name"; String WORKFLOW_DEF_NAME_KEY = "workflow_def_name"; String WORKFLOW_VERSION_KEY = "version"; String WORKFLOW_DEFINITION_KEY = "workflow_definition"; String WORKFLOW_DEF_INDEX_KEY = "workflow_def_version_index"; String WORKFLOW_DEF_INDEX_VALUE = "workflow_def_index_value"; String WORKFLOW_DEF_NAME_VERSION_KEY = "workflow_def_name_version"; String TASK_DEFS_KEY = "task_defs"; String TASK_DEFINITION_KEY = "task_definition"; String HANDLERS_KEY = "handlers"; String EVENT_HANDLER_NAME_KEY = "event_handler_name"; String EVENT_HANDLER_KEY = "event_handler"; String MESSAGE_ID_KEY = "message_id"; String EVENT_EXECUTION_ID_KEY = "event_execution_id"; String ENTITY_TYPE_WORKFLOW = "workflow"; String ENTITY_TYPE_TASK = "task"; int DEFAULT_SHARD_ID = 1; int DEFAULT_TOTAL_PARTITIONS = 1; } ================================================ FILE: cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/util/Statements.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.util; import com.datastax.driver.core.querybuilder.QueryBuilder; import static com.netflix.conductor.cassandra.util.Constants.ENTITY_KEY; import static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_TASK; import static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_WORKFLOW; import static com.netflix.conductor.cassandra.util.Constants.EVENT_EXECUTION_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY; import static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY; import static com.netflix.conductor.cassandra.util.Constants.MESSAGE_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.PAYLOAD_KEY; import static com.netflix.conductor.cassandra.util.Constants.SHARD_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_EXECUTIONS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_HANDLERS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEFS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEF_LIMIT; import static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_LOOKUP; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOWS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS; import static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS_INDEX; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_DEF_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.TASK_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.TOTAL_PARTITIONS_KEY; import static com.netflix.conductor.cassandra.util.Constants.TOTAL_TASKS_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_VALUE; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_ID_KEY; import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_VERSION_KEY; import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker; import static com.datastax.driver.core.querybuilder.QueryBuilder.eq; import static com.datastax.driver.core.querybuilder.QueryBuilder.set; /** * DML statements * *

MetadataDAO * *

* * ExecutionDAO * * * * EventHandlerDAO * * */ public class Statements { private final String keyspace; public Statements(String keyspace) { this.keyspace = keyspace; } // MetadataDAO // Insert Statements /** * @return cql query statement to insert a new workflow definition into the * "workflow_definitions" table */ public String getInsertWorkflowDefStatement() { return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOW_DEFS) .value(WORKFLOW_DEF_NAME_KEY, bindMarker()) .value(WORKFLOW_VERSION_KEY, bindMarker()) .value(WORKFLOW_DEFINITION_KEY, bindMarker()) .ifNotExists() .getQueryString(); } /** * @return cql query statement to insert a workflow def name version index into the * "workflow_defs_index" table */ public String getInsertWorkflowDefVersionIndexStatement() { return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOW_DEFS_INDEX) .value(WORKFLOW_DEF_INDEX_KEY, WORKFLOW_DEF_INDEX_KEY) .value(WORKFLOW_DEF_NAME_VERSION_KEY, bindMarker()) .value(WORKFLOW_DEF_INDEX_VALUE, bindMarker()) .getQueryString(); } /** * @return cql query statement to insert a new task definition into the "task_definitions" table */ public String getInsertTaskDefStatement() { return QueryBuilder.insertInto(keyspace, TABLE_TASK_DEFS) .value(TASK_DEFS_KEY, TASK_DEFS_KEY) .value(TASK_DEF_NAME_KEY, bindMarker()) .value(TASK_DEFINITION_KEY, bindMarker()) .getQueryString(); } // Select Statements /** * @return cql query statement to fetch a workflow definition by name and version from the * "workflow_definitions" table */ public String getSelectWorkflowDefStatement() { return QueryBuilder.select(WORKFLOW_DEFINITION_KEY) .from(keyspace, TABLE_WORKFLOW_DEFS) .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker())) .and(eq(WORKFLOW_VERSION_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve all versions of a workflow definition by name from * the "workflow_definitions" table */ public String getSelectAllWorkflowDefVersionsByNameStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_WORKFLOW_DEFS) .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to fetch all workflow def names and version from the * "workflow_defs_index" table */ public String getSelectAllWorkflowDefsStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX) .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker())) .getQueryString(); } public String getSelectAllWorkflowDefsLatestVersionsStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX) .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to fetch a task definition by name from the "task_definitions" * table */ public String getSelectTaskDefStatement() { return QueryBuilder.select(TASK_DEFINITION_KEY) .from(keyspace, TABLE_TASK_DEFS) .where(eq(TASK_DEFS_KEY, TASK_DEFS_KEY)) .and(eq(TASK_DEF_NAME_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve all task definitions from the "task_definitions" * table */ public String getSelectAllTaskDefsStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_TASK_DEFS) .where(eq(TASK_DEFS_KEY, bindMarker())) .getQueryString(); } // Update Statement /** * @return cql query statement to update a workflow definitinos in the "workflow_definitions" * table */ public String getUpdateWorkflowDefStatement() { return QueryBuilder.update(keyspace, TABLE_WORKFLOW_DEFS) .with(set(WORKFLOW_DEFINITION_KEY, bindMarker())) .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker())) .and(eq(WORKFLOW_VERSION_KEY, bindMarker())) .getQueryString(); } // Delete Statements /** * @return cql query statement to delete a workflow definition by name and version from the * "workflow_definitions" table */ public String getDeleteWorkflowDefStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_WORKFLOW_DEFS) .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker())) .and(eq(WORKFLOW_VERSION_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete a workflow def name/version from the * "workflow_defs_index" table */ public String getDeleteWorkflowDefIndexStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX) .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker())) .and(eq(WORKFLOW_DEF_NAME_VERSION_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete a task definition by name from the "task_definitions" * table */ public String getDeleteTaskDefStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_TASK_DEFS) .where(eq(TASK_DEFS_KEY, TASK_DEFS_KEY)) .and(eq(TASK_DEF_NAME_KEY, bindMarker())) .getQueryString(); } // ExecutionDAO // Insert Statements /** * @return cql query statement to insert a new workflow into the "workflows" table */ public String getInsertWorkflowStatement() { return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOWS) .value(WORKFLOW_ID_KEY, bindMarker()) .value(SHARD_ID_KEY, bindMarker()) .value(TASK_ID_KEY, bindMarker()) .value(ENTITY_KEY, ENTITY_TYPE_WORKFLOW) .value(PAYLOAD_KEY, bindMarker()) .value(TOTAL_TASKS_KEY, bindMarker()) .value(TOTAL_PARTITIONS_KEY, bindMarker()) .getQueryString(); } /** * @return cql query statement to insert a new task into the "workflows" table */ public String getInsertTaskStatement() { return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOWS) .value(WORKFLOW_ID_KEY, bindMarker()) .value(SHARD_ID_KEY, bindMarker()) .value(TASK_ID_KEY, bindMarker()) .value(ENTITY_KEY, ENTITY_TYPE_TASK) .value(PAYLOAD_KEY, bindMarker()) .getQueryString(); } /** * @return cql query statement to insert a new event execution into the "event_executions" table */ public String getInsertEventExecutionStatement() { return QueryBuilder.insertInto(keyspace, TABLE_EVENT_EXECUTIONS) .value(MESSAGE_ID_KEY, bindMarker()) .value(EVENT_HANDLER_NAME_KEY, bindMarker()) .value(EVENT_EXECUTION_ID_KEY, bindMarker()) .value(PAYLOAD_KEY, bindMarker()) .ifNotExists() .getQueryString(); } // Select Statements /** * @return cql query statement to retrieve the total_tasks and total_partitions for a workflow * from the "workflows" table */ public String getSelectTotalStatement() { return QueryBuilder.select(TOTAL_TASKS_KEY, TOTAL_PARTITIONS_KEY) .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, 1)) .getQueryString(); } /** * @return cql query statement to retrieve a task from the "workflows" table */ public String getSelectTaskStatement() { return QueryBuilder.select(PAYLOAD_KEY) .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, bindMarker())) .and(eq(ENTITY_KEY, ENTITY_TYPE_TASK)) .and(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve a workflow (without its tasks) from the "workflows" * table */ public String getSelectWorkflowStatement() { return QueryBuilder.select(PAYLOAD_KEY) .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, 1)) .and(eq(ENTITY_KEY, ENTITY_TYPE_WORKFLOW)) .getQueryString(); } /** * @return cql query statement to retrieve a workflow with its tasks from the "workflows" table */ public String getSelectWorkflowWithTasksStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve the workflow_id for a particular task_id from the * "task_lookup" table */ public String getSelectTaskFromLookupTableStatement() { return QueryBuilder.select(WORKFLOW_ID_KEY) .from(keyspace, TABLE_TASK_LOOKUP) .where(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve all task ids for a given taskDefName with concurrent * execution limit configured from the "task_def_limit" table */ public String getSelectTasksFromTaskDefLimitStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_TASK_DEF_LIMIT) .where(eq(TASK_DEF_NAME_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to retrieve all event executions for a given message and event * handler from the "event_executions" table */ public String getSelectAllEventExecutionsForMessageFromEventExecutionsStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_EVENT_EXECUTIONS) .where(eq(MESSAGE_ID_KEY, bindMarker())) .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker())) .getQueryString(); } // Update Statements /** * @return cql query statement to update a workflow in the "workflows" table */ public String getUpdateWorkflowStatement() { return QueryBuilder.update(keyspace, TABLE_WORKFLOWS) .with(set(PAYLOAD_KEY, bindMarker())) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, 1)) .and(eq(ENTITY_KEY, ENTITY_TYPE_WORKFLOW)) .and(eq(TASK_ID_KEY, "")) .getQueryString(); } /** * @return cql query statement to update the total_tasks in a shard for a workflow in the * "workflows" table */ public String getUpdateTotalTasksStatement() { return QueryBuilder.update(keyspace, TABLE_WORKFLOWS) .with(set(TOTAL_TASKS_KEY, bindMarker())) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to update the total_partitions for a workflow in the "workflows" * table */ public String getUpdateTotalPartitionsStatement() { return QueryBuilder.update(keyspace, TABLE_WORKFLOWS) .with(set(TOTAL_PARTITIONS_KEY, bindMarker())) .and(set(TOTAL_TASKS_KEY, bindMarker())) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, 1)) .getQueryString(); } /** * @return cql query statement to add a new task_id to workflow_id mapping to the "task_lookup" * table */ public String getUpdateTaskLookupStatement() { return QueryBuilder.update(keyspace, TABLE_TASK_LOOKUP) .with(set(WORKFLOW_ID_KEY, bindMarker())) .where(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to add a new task_id to the "task_def_limit" table */ public String getUpdateTaskDefLimitStatement() { return QueryBuilder.update(keyspace, TABLE_TASK_DEF_LIMIT) .with(set(WORKFLOW_ID_KEY, bindMarker())) .where(eq(TASK_DEF_NAME_KEY, bindMarker())) .and(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to update an event execution in the "event_executions" table */ public String getUpdateEventExecutionStatement() { return QueryBuilder.update(keyspace, TABLE_EVENT_EXECUTIONS) .using(QueryBuilder.ttl(bindMarker())) .with(set(PAYLOAD_KEY, bindMarker())) .where(eq(MESSAGE_ID_KEY, bindMarker())) .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker())) .and(eq(EVENT_EXECUTION_ID_KEY, bindMarker())) .getQueryString(); } // Delete statements /** * @return cql query statement to delete a workflow from the "workflows" table */ public String getDeleteWorkflowStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete a task_id to workflow_id mapping from the "task_lookup" * table */ public String getDeleteTaskLookupStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_TASK_LOOKUP) .where(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete a task from the "workflows" table */ public String getDeleteTaskStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_WORKFLOWS) .where(eq(WORKFLOW_ID_KEY, bindMarker())) .and(eq(SHARD_ID_KEY, bindMarker())) .and(eq(ENTITY_KEY, ENTITY_TYPE_TASK)) .and(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete a task_id from the "task_def_limit" table */ public String getDeleteTaskDefLimitStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_TASK_DEF_LIMIT) .where(eq(TASK_DEF_NAME_KEY, bindMarker())) .and(eq(TASK_ID_KEY, bindMarker())) .getQueryString(); } /** * @return cql query statement to delete an event execution from the "event_execution" table */ public String getDeleteEventExecutionsStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_EVENT_EXECUTIONS) .where(eq(MESSAGE_ID_KEY, bindMarker())) .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker())) .and(eq(EVENT_EXECUTION_ID_KEY, bindMarker())) .getQueryString(); } // EventHandlerDAO // Insert Statements /** * @return cql query statement to insert an event handler into the "event_handlers" table */ public String getInsertEventHandlerStatement() { return QueryBuilder.insertInto(keyspace, TABLE_EVENT_HANDLERS) .value(HANDLERS_KEY, HANDLERS_KEY) .value(EVENT_HANDLER_NAME_KEY, bindMarker()) .value(EVENT_HANDLER_KEY, bindMarker()) .getQueryString(); } // Select Statements /** * @return cql query statement to retrieve all event handlers from the "event_handlers" table */ public String getSelectAllEventHandlersStatement() { return QueryBuilder.select() .all() .from(keyspace, TABLE_EVENT_HANDLERS) .where(eq(HANDLERS_KEY, bindMarker())) .getQueryString(); } // Delete Statements /** * @return cql query statement to delete an event handler by name from the "event_handlers" * table */ public String getDeleteEventHandlerStatement() { return QueryBuilder.delete() .from(keyspace, TABLE_EVENT_HANDLERS) .where(eq(HANDLERS_KEY, HANDLERS_KEY)) .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker())) .getQueryString(); } } ================================================ FILE: cassandra-persistence/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.cassandra.write-consistency-level", "defaultValue": "LOCAL_QUORUM" }, { "name": "conductor.cassandra.read-consistency-level", "defaultValue": "LOCAL_QUORUM" } ], "hints": [ { "name": "conductor.cassandra.write-consistency-level", "providers": [ { "name": "handle-as", "parameters": { "target": "java.lang.Enum" } } ] }, { "name": "conductor.cassandra.read-consistency-level", "providers": [ { "name": "handle-as", "parameters": { "target": "java.lang.Enum" } } ] } ] } ================================================ FILE: cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/dao/CassandraEventHandlerDAOSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao import com.netflix.conductor.common.metadata.events.EventExecution import com.netflix.conductor.common.metadata.events.EventHandler import spock.lang.Subject class CassandraEventHandlerDAOSpec extends CassandraSpec { @Subject CassandraEventHandlerDAO eventHandlerDAO CassandraExecutionDAO executionDAO def setup() { eventHandlerDAO = new CassandraEventHandlerDAO(session, objectMapper, cassandraProperties, statements) executionDAO = new CassandraExecutionDAO(session, objectMapper, cassandraProperties, statements) } def testEventHandlerCRUD() { given: String event = "event" String eventHandlerName1 = "event_handler1" String eventHandlerName2 = "event_handler2" EventHandler eventHandler = new EventHandler() eventHandler.setName(eventHandlerName1) eventHandler.setEvent(event) when: // create event handler eventHandlerDAO.addEventHandler(eventHandler) List handlers = eventHandlerDAO.getEventHandlersForEvent(event, false) then: // fetch all event handlers for event handlers != null && handlers.size() == 1 eventHandler.name == handlers[0].name eventHandler.event == handlers[0].event !handlers[0].active and: // add an active event handler for the same event EventHandler eventHandler1 = new EventHandler() eventHandler1.setName(eventHandlerName2) eventHandler1.setEvent(event) eventHandler1.setActive(true) eventHandlerDAO.addEventHandler(eventHandler1) when: // fetch all event handlers handlers = eventHandlerDAO.getAllEventHandlers() then: handlers != null && handlers.size() == 2 when: // fetch all event handlers for event handlers = eventHandlerDAO.getEventHandlersForEvent(event, false) then: handlers != null && handlers.size() == 2 when: // fetch only active handlers for event handlers = eventHandlerDAO.getEventHandlersForEvent(event, true) then: handlers != null && handlers.size() == 1 eventHandler1.name == handlers[0].name eventHandler1.event == handlers[0].event handlers[0].active when: // remove event handler eventHandlerDAO.removeEventHandler(eventHandlerName1) handlers = eventHandlerDAO.getAllEventHandlers() then: handlers != null && handlers.size() == 1 } private static EventExecution getEventExecution(String id, String msgId, String name, String event) { EventExecution eventExecution = new EventExecution(id, msgId); eventExecution.setName(name); eventExecution.setEvent(event); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); return eventExecution; } } ================================================ FILE: cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/dao/CassandraExecutionDAOSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao import com.netflix.conductor.common.metadata.events.EventExecution import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.core.exception.NonTransientException import com.netflix.conductor.core.utils.IDGenerator import com.netflix.conductor.model.TaskModel import com.netflix.conductor.model.WorkflowModel import spock.lang.Subject import static com.netflix.conductor.common.metadata.events.EventExecution.Status.COMPLETED import static com.netflix.conductor.common.metadata.events.EventExecution.Status.IN_PROGRESS class CassandraExecutionDAOSpec extends CassandraSpec { @Subject CassandraExecutionDAO executionDAO def setup() { executionDAO = new CassandraExecutionDAO(session, objectMapper, cassandraProperties, statements) } def "verify if tasks are validated"() { given: def tasks = [] // create tasks for a workflow and add to list TaskModel task1 = new TaskModel(workflowInstanceId: 'uuid', taskId: 'task1id', referenceTaskName: 'task1') TaskModel task2 = new TaskModel(workflowInstanceId: 'uuid', taskId: 'task2id', referenceTaskName: 'task2') tasks << task1 << task2 when: executionDAO.validateTasks(tasks) then: noExceptionThrown() and: // add a task from a different workflow to the list TaskModel task3 = new TaskModel(workflowInstanceId: 'other-uuid', taskId: 'task3id', referenceTaskName: 'task3') tasks << task3 when: executionDAO.validateTasks(tasks) then: def ex = thrown(NonTransientException.class) ex.message == "Tasks of multiple workflows cannot be created/updated simultaneously" } def "workflow CRUD"() { given: String workflowId = new IDGenerator().generate() WorkflowDef workflowDef = new WorkflowDef() workflowDef.name = "def1" workflowDef.setVersion(1) WorkflowModel workflow = new WorkflowModel() workflow.setWorkflowDefinition(workflowDef) workflow.setWorkflowId(workflowId) workflow.setInput(new HashMap<>()) workflow.setStatus(WorkflowModel.Status.RUNNING) workflow.setCreateTime(System.currentTimeMillis()) when: // create a new workflow in the datastore String id = executionDAO.createWorkflow(workflow) then: workflowId == id when: // read the workflow from the datastore WorkflowModel found = executionDAO.getWorkflow(workflowId) then: workflow == found and: // update the workflow workflow.setStatus(WorkflowModel.Status.COMPLETED) executionDAO.updateWorkflow(workflow) when: found = executionDAO.getWorkflow(workflowId) then: workflow == found when: // remove the workflow from datastore boolean removed = executionDAO.removeWorkflow(workflowId) then: removed when: // read workflow again workflow = executionDAO.getWorkflow(workflowId, true) then: workflow == null } def "create tasks and verify methods that read tasks and workflow"() { given: 'we create a workflow' String workflowId = new IDGenerator().generate() WorkflowDef workflowDef = new WorkflowDef(name: 'def1', version: 1) WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, input: new HashMap(), status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis()) executionDAO.createWorkflow(workflow) and: 'create tasks for this workflow' TaskModel task1 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task1', referenceTaskName: 'task1', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task2 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task2', referenceTaskName: 'task2', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task3 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task3', referenceTaskName: 'task3', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) def taskList = [task1, task2, task3] when: 'add the tasks to the datastore' List tasks = executionDAO.createTasks(taskList) then: tasks != null taskList == tasks when: 'read the tasks from the datastore' def retTask1 = executionDAO.getTask(task1.taskId) def retTask2 = executionDAO.getTask(task2.taskId) def retTask3 = executionDAO.getTask(task3.taskId) then: task1 == retTask1 task2 == retTask2 task3 == retTask3 when: 'lookup workflowId for the task' def foundId1 = executionDAO.lookupWorkflowIdFromTaskId(task1.taskId) def foundId2 = executionDAO.lookupWorkflowIdFromTaskId(task2.taskId) def foundId3 = executionDAO.lookupWorkflowIdFromTaskId(task3.taskId) then: foundId1 == workflowId foundId2 == workflowId foundId3 == workflowId when: 'check the metadata' def workflowMetadata = executionDAO.getWorkflowMetadata(workflowId) then: workflowMetadata.totalTasks == 3 workflowMetadata.totalPartitions == 1 when: 'check the getTasks api' def fetchedTasks = executionDAO.getTasks([task1.taskId, task2.taskId, task3.taskId]) then: fetchedTasks != null && fetchedTasks.size() == 3 when: 'get the tasks for the workflow' fetchedTasks = executionDAO.getTasksForWorkflow(workflowId) then: fetchedTasks != null && fetchedTasks.size() == 3 when: 'read workflow with tasks' WorkflowModel found = executionDAO.getWorkflow(workflowId, true) then: found != null workflow.workflowId == found.workflowId found.tasks != null && found.tasks.size() == 3 found.getTaskByRefName('task1') == task1 found.getTaskByRefName('task2') == task2 found.getTaskByRefName('task3') == task3 } def "verify tasks are updated"() { given: 'we create a workflow' String workflowId = new IDGenerator().generate() WorkflowDef workflowDef = new WorkflowDef(name: 'def1', version: 1) WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, input: new HashMap(), status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis()) executionDAO.createWorkflow(workflow) and: 'create tasks for this workflow' TaskModel task1 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task1', referenceTaskName: 'task1', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task2 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task2', referenceTaskName: 'task2', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task3 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task3', referenceTaskName: 'task3', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) and: 'add the tasks to the datastore' executionDAO.createTasks([task1, task2, task3]) and: 'change the status of those tasks' task1.setStatus(TaskModel.Status.IN_PROGRESS) task2.setStatus(TaskModel.Status.COMPLETED) task3.setStatus(TaskModel.Status.FAILED) when: 'update the tasks' executionDAO.updateTask(task1) executionDAO.updateTask(task2) executionDAO.updateTask(task3) then: executionDAO.getTask(task1.taskId).status == TaskModel.Status.IN_PROGRESS executionDAO.getTask(task2.taskId).status == TaskModel.Status.COMPLETED executionDAO.getTask(task3.taskId).status == TaskModel.Status.FAILED when: 'get pending tasks for the workflow' List pendingTasks = executionDAO.getPendingTasksByWorkflow(task1.getTaskType(), workflowId) then: pendingTasks != null && pendingTasks.size() == 1 pendingTasks[0] == task1 } def "verify tasks are removed"() { given: 'we create a workflow' String workflowId = new IDGenerator().generate() WorkflowDef workflowDef = new WorkflowDef(name: 'def1', version: 1) WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, input: new HashMap(), status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis()) executionDAO.createWorkflow(workflow) and: 'create tasks for this workflow' TaskModel task1 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task1', referenceTaskName: 'task1', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task2 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task2', referenceTaskName: 'task2', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) TaskModel task3 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task3', referenceTaskName: 'task3', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate()) and: 'add the tasks to the datastore' executionDAO.createTasks([task1, task2, task3]) when: boolean removed = executionDAO.removeTask(task3.getTaskId()) then: removed def workflowMetadata = executionDAO.getWorkflowMetadata(workflowId) workflowMetadata.totalTasks == 2 workflowMetadata.totalPartitions == 1 when: 'read workflow with tasks again' def found = executionDAO.getWorkflow(workflowId) then: found != null found.workflowId == workflowId found.tasks.size() == 2 found.getTaskByRefName('task1') == task1 found.getTaskByRefName('task2') == task2 and: 'read workflowId for the deleted task id' executionDAO.lookupWorkflowIdFromTaskId(task3.taskId) == null and: 'try to read removed task' executionDAO.getTask(task3.getTaskId()) == null when: 'remove the workflow' removed = executionDAO.removeWorkflow(workflowId) then: 'check task_lookup table' removed executionDAO.lookupWorkflowIdFromTaskId(task1.taskId) == null executionDAO.lookupWorkflowIdFromTaskId(task2.taskId) == null } def "CRUD on task def limit"() { given: String taskDefName = "test_task_def" String taskId = new IDGenerator().generate() TaskDef taskDef = new TaskDef(concurrentExecLimit: 1) WorkflowTask workflowTask = new WorkflowTask(taskDefinition: taskDef) workflowTask.setTaskDefinition(taskDef) TaskModel task = new TaskModel() task.taskDefName = taskDefName task.taskId = taskId task.workflowInstanceId = new IDGenerator().generate() task.setWorkflowTask(workflowTask) task.setTaskType("test_task") task.setWorkflowType("test_workflow") task.setStatus(TaskModel.Status.SCHEDULED) TaskModel newTask = new TaskModel() newTask.setTaskDefName(taskDefName) newTask.setTaskId(new IDGenerator().generate()) newTask.setWorkflowInstanceId(new IDGenerator().generate()) newTask.setWorkflowTask(workflowTask) newTask.setTaskType("test_task") newTask.setWorkflowType("test_workflow") newTask.setStatus(TaskModel.Status.SCHEDULED) when: // no tasks are IN_PROGRESS executionDAO.addTaskToLimit(task) then: !executionDAO.exceedsLimit(task) when: // set a task to IN_PROGRESS task.setStatus(TaskModel.Status.IN_PROGRESS) executionDAO.addTaskToLimit(task) then: // same task is checked !executionDAO.exceedsLimit(task) and: // check if new task can be added executionDAO.exceedsLimit(newTask) when: // set IN_PROGRESS task to COMPLETED task.setStatus(TaskModel.Status.COMPLETED) executionDAO.removeTaskFromLimit(task) then: // check new task again !executionDAO.exceedsLimit(newTask) when: // set new task to IN_PROGRESS newTask.setStatus(TaskModel.Status.IN_PROGRESS) executionDAO.addTaskToLimit(newTask) then: // check new task again !executionDAO.exceedsLimit(newTask) } def "verify if invalid identifiers throw correct exceptions"() { when: 'verify that a non-conforming uuid throws an exception' executionDAO.getTask('invalid_id') then: thrown(IllegalArgumentException.class) when: 'verify that a non-conforming uuid throws an exception' executionDAO.getWorkflow('invalid_id', true) then: thrown(IllegalArgumentException.class) and: 'verify that a non-existing generated id returns null' executionDAO.getTask(new IDGenerator().generate()) == null executionDAO.getWorkflow(new IDGenerator().generate(), true) == null } def "CRUD on event execution"() throws Exception { given: String event = "test-event" String executionId1 = "id_1" String messageId1 = "message1" String eventHandler1 = "test_eh_1" EventExecution eventExecution1 = getEventExecution(executionId1, messageId1, eventHandler1, event) when: // create event execution explicitly executionDAO.addEventExecution(eventExecution1) List eventExecutionList = executionDAO.getEventExecutions(eventHandler1, event, messageId1) then: // fetch executions eventExecutionList != null && eventExecutionList.size() == 1 eventExecutionList[0] == eventExecution1 when: // add a different execution for same message String executionId2 = "id_2" EventExecution eventExecution2 = getEventExecution(executionId2, messageId1, eventHandler1, event) executionDAO.addEventExecution(eventExecution2) eventExecutionList = executionDAO.getEventExecutions(eventHandler1, event, messageId1) then: // fetch executions eventExecutionList != null && eventExecutionList.size() == 2 eventExecutionList[0] == eventExecution1 eventExecutionList[1] == eventExecution2 when: // update the second execution eventExecution2.setStatus(COMPLETED) executionDAO.updateEventExecution(eventExecution2) eventExecutionList = executionDAO.getEventExecutions(eventHandler1, event, messageId1) then: // fetch executions eventExecutionList != null && eventExecutionList.size() == 2 eventExecutionList[0].status == IN_PROGRESS eventExecutionList[1].status == COMPLETED when: // sleep for 5 seconds (TTL) Thread.sleep(5000L) eventExecutionList = executionDAO.getEventExecutions(eventHandler1, event, messageId1) then: eventExecutionList != null && eventExecutionList.size() == 1 when: // delete event execution executionDAO.removeEventExecution(eventExecution1) eventExecutionList = executionDAO.getEventExecutions(eventHandler1, event, messageId1) then: eventExecutionList != null && eventExecutionList.empty } def "verify workflow serialization"() { given: 'define a workflow' String workflowId = new IDGenerator().generate() WorkflowTask workflowTask = new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: 2)) WorkflowDef workflowDef = new WorkflowDef(name: UUID.randomUUID().toString(), version: 1, tasks: [workflowTask]) WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis()) when: 'serialize workflow' def workflowJson = objectMapper.writeValueAsString(workflow) then: !workflowJson.contains('failedReferenceTaskNames') // workflowTask !workflowJson.contains('decisionCases') !workflowJson.contains('defaultCase') !workflowJson.contains('forkTasks') !workflowJson.contains('joinOn') !workflowJson.contains('defaultExclusiveJoinTask') !workflowJson.contains('loopOver') } def "verify task serialization"() { given: 'define a workflow and tasks for this workflow' String workflowId = new IDGenerator().generate() WorkflowTask workflowTask = new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: 2)) TaskModel task = new TaskModel(workflowInstanceId: workflowId, taskType: UUID.randomUUID().toString(), referenceTaskName: UUID.randomUUID().toString(), status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate(), workflowTask: workflowTask) when: 'serialize task' def taskJson = objectMapper.writeValueAsString(task) then: !taskJson.contains('decisionCases') !taskJson.contains('defaultCase') !taskJson.contains('forkTasks') !taskJson.contains('joinOn') !taskJson.contains('defaultExclusiveJoinTask') } def "serde of workflow with large number of tasks"() { given: 'create a workflow and tasks for this workflow' String workflowId = new IDGenerator().generate() def workflowTasks = (0..999) .collect { new WorkflowTask(name: it, taskReferenceName: it, taskDefinition: new TaskDef(name: it)) } WorkflowDef workflowDef = new WorkflowDef(name: UUID.randomUUID().toString(), version: 1, tasks: workflowTasks) def taskList = (0..999) .collect { new TaskModel(workflowInstanceId: workflowId, taskType: it, referenceTaskName: it, status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate(), workflowTask: workflowTasks.get(it)) } WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis()) and: 'create workflow' executionDAO.createWorkflow(workflow) when: 'add the tasks to the datastore' def start_time = System.currentTimeMillis() executionDAO.createTasks(taskList) println("Create 1000 tasks, duration: ${System.currentTimeMillis() - start_time} ms") then: def workflowMetadata = executionDAO.getWorkflowMetadata(workflowId) workflowMetadata.totalTasks == 1000 when: 'read workflow with tasks' start_time = System.currentTimeMillis() WorkflowModel found = executionDAO.getWorkflow(workflowId, true) println("Get workflow with 1000 tasks, duration: ${System.currentTimeMillis() - start_time} ms") then: found != null workflow.workflowId == found.workflowId found.tasks != null && found.tasks.size() == 1000 (0..999).collect {found.getTaskByRefName(""+it) == taskList.get(it)} } private static EventExecution getEventExecution(String id, String msgId, String name, String event) { EventExecution eventExecution = new EventExecution(id, msgId); eventExecution.setName(name); eventExecution.setEvent(event); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); return eventExecution; } } ================================================ FILE: cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/dao/CassandraMetadataDAOSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.WorkflowDef import spock.lang.Subject class CassandraMetadataDAOSpec extends CassandraSpec { @Subject CassandraMetadataDAO metadataDAO def setup() { metadataDAO = new CassandraMetadataDAO(session, objectMapper, cassandraProperties, statements) } def cleanup() { } def "CRUD on WorkflowDef"() throws Exception { given: String name = "workflow_def_1" int version = 1 WorkflowDef workflowDef = new WorkflowDef() workflowDef.setName(name) workflowDef.setVersion(version) workflowDef.setOwnerEmail("test@junit.com") when: 'create workflow definition' metadataDAO.createWorkflowDef(workflowDef) then: // fetch the workflow definition def defOptional = metadataDAO.getWorkflowDef(name, version) defOptional.present defOptional.get() == workflowDef and: // register a higher version int higherVersion = 2 workflowDef.setVersion(higherVersion) workflowDef.setDescription("higher version") when: // register the higher version definition metadataDAO.createWorkflowDef(workflowDef) defOptional = metadataDAO.getWorkflowDef(name, higherVersion) then: // fetch the higher version defOptional.present defOptional.get() == workflowDef when: // fetch latest version defOptional = metadataDAO.getLatestWorkflowDef(name) then: defOptional && defOptional.present defOptional.get() == workflowDef when: // modify the definition workflowDef.setOwnerEmail("test@junit.com") metadataDAO.updateWorkflowDef(workflowDef) defOptional = metadataDAO.getWorkflowDef(name, higherVersion) then: // fetch the workflow definition defOptional.present defOptional.get() == workflowDef when: // delete workflow def metadataDAO.removeWorkflowDef(name, higherVersion) defOptional = metadataDAO.getWorkflowDef(name, higherVersion) then: defOptional.empty } def "CRUD on TaskDef"() { given: String task1Name = "task1" String task2Name = "task2" when: // fetch all task defs def taskDefList = metadataDAO.getAllTaskDefs() then: taskDefList.empty when: // register a task definition TaskDef taskDef = new TaskDef() taskDef.setName(task1Name) metadataDAO.createTaskDef(taskDef) taskDefList = metadataDAO.getAllTaskDefs() then: // fetch all task defs taskDefList && taskDefList.size() == 1 when: // fetch the task def def returnTaskDef = metadataDAO.getTaskDef(task1Name) then: returnTaskDef == taskDef when: // register another task definition TaskDef taskDef1 = new TaskDef() taskDef1.setName(task2Name) metadataDAO.createTaskDef(taskDef1) // fetch all task defs taskDefList = metadataDAO.getAllTaskDefs() then: taskDefList && taskDefList.size() == 2 when: // update task def taskDef.setOwnerEmail("juni@test.com") metadataDAO.updateTaskDef(taskDef) returnTaskDef = metadataDAO.getTaskDef(task1Name) then: returnTaskDef == taskDef when: // delete task def metadataDAO.removeTaskDef(task2Name) taskDefList = metadataDAO.getAllTaskDefs() then: taskDefList && taskDefList.size() == 1 // fetch deleted task def metadataDAO.getTaskDef(task2Name) == null } def "set default response timeout when not set"() { given: String task1Name = "task1" when: // register a task definition TaskDef taskDef = new TaskDef() taskDef.setName(task1Name) taskDef.setResponseTimeoutSeconds(0) metadataDAO.createTaskDef(taskDef) def returnTaskDef = metadataDAO.getTaskDef(task1Name) then: returnTaskDef.getResponseTimeoutSeconds() == 3600 when: // register another task definition taskDef.setTimeoutSeconds(200) taskDef.setResponseTimeoutSeconds(0) metadataDAO.updateTaskDef(taskDef) // fetch all task defs def taskDefList = metadataDAO.getAllTaskDefs() then: taskDefList && taskDefList.size() == 1 taskDefList.get(0).getResponseTimeoutSeconds() == 199 } def "Get All WorkflowDef"() { when: metadataDAO.removeWorkflowDef("workflow_def_1", 1) WorkflowDef workflowDef = new WorkflowDef() workflowDef.setName("workflow_def_1") workflowDef.setVersion(1) workflowDef.setOwnerEmail("test@junit.com") metadataDAO.createWorkflowDef(workflowDef) workflowDef.setName("workflow_def_2") metadataDAO.createWorkflowDef(workflowDef) workflowDef.setVersion(2) metadataDAO.createWorkflowDef(workflowDef) workflowDef.setName("workflow_def_3") workflowDef.setVersion(1) metadataDAO.createWorkflowDef(workflowDef) workflowDef.setVersion(2) metadataDAO.createWorkflowDef(workflowDef) workflowDef.setVersion(3) metadataDAO.createWorkflowDef(workflowDef) then: // fetch the workflow definition def allDefsLatestVersions = metadataDAO.getAllWorkflowDefsLatestVersions() Map allDefsMap = allDefsLatestVersions.collectEntries {wfDef -> [wfDef.getName(), wfDef]} allDefsMap.get("workflow_def_1").getVersion() == 1 allDefsMap.get("workflow_def_2").getVersion() == 2 allDefsMap.get("workflow_def_3").getVersion() == 3 } def "parse index string"() { expect: def pair = metadataDAO.getWorkflowNameAndVersion(nameVersionStr) pair.left == workflowName pair.right == version where: nameVersionStr << ['name/1', 'namespace/name/3', '/namespace/name_with_lodash/2', 'name//4', 'name-with$%/895'] workflowName << ['name', 'namespace/name', '/namespace/name_with_lodash', 'name/', 'name-with$%'] version << [1, 3, 2, 4, 895] } def "parse index string - incorrect values"() { when: metadataDAO.getWorkflowNameAndVersion("name_with_no_version") then: def ex = thrown(IllegalStateException.class) println(ex.message) when: metadataDAO.getWorkflowNameAndVersion("name_with_no_version/") then: ex = thrown(IllegalStateException.class) println(ex.message) when: metadataDAO.getWorkflowNameAndVersion("name/non_number_version") then: ex = thrown(IllegalStateException.class) println(ex.message) } } ================================================ FILE: cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/dao/CassandraSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.dao import java.time.Duration import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.context.ContextConfiguration import org.testcontainers.containers.CassandraContainer import org.testcontainers.spock.Testcontainers import com.netflix.conductor.cassandra.config.CassandraProperties import com.netflix.conductor.cassandra.util.Statements import com.netflix.conductor.common.config.TestObjectMapperConfiguration import com.datastax.driver.core.ConsistencyLevel import com.datastax.driver.core.Session import com.fasterxml.jackson.databind.ObjectMapper import groovy.transform.PackageScope import spock.lang.Shared import spock.lang.Specification @ContextConfiguration(classes = [TestObjectMapperConfiguration.class]) @Testcontainers @PackageScope abstract class CassandraSpec extends Specification { @Shared CassandraContainer cassandra = new CassandraContainer() @Shared Session session @Autowired ObjectMapper objectMapper CassandraProperties cassandraProperties Statements statements def setupSpec() { session = cassandra.cluster.newSession() } def setup() { String keyspaceName = "junit" cassandraProperties = Mock(CassandraProperties.class) { getKeyspace() >> keyspaceName getReplicationStrategy() >> "SimpleStrategy" getReplicationFactorKey() >> "replication_factor" getReplicationFactorValue() >> 1 getReadConsistencyLevel() >> ConsistencyLevel.LOCAL_ONE getWriteConsistencyLevel() >> ConsistencyLevel.LOCAL_ONE getTaskDefCacheRefreshInterval() >> Duration.ofSeconds(60) getEventHandlerCacheRefreshInterval() >> Duration.ofSeconds(60) getEventExecutionPersistenceTtl() >> Duration.ofSeconds(5) } statements = new Statements(keyspaceName) } } ================================================ FILE: cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/util/StatementsSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.cassandra.util import spock.lang.Specification import spock.lang.Subject class StatementsSpec extends Specification { @Subject Statements subject def setup() { subject = new Statements('test') } def "verify statements"() { when: subject then: with(subject) { insertWorkflowDefStatement == "INSERT INTO test.workflow_definitions (workflow_def_name,version,workflow_definition) VALUES (?,?,?) IF NOT EXISTS;" insertTaskDefStatement == "INSERT INTO test.task_definitions (task_defs,task_def_name,task_definition) VALUES ('task_defs',?,?);" selectWorkflowDefStatement == "SELECT workflow_definition FROM test.workflow_definitions WHERE workflow_def_name=? AND version=?;" selectAllWorkflowDefVersionsByNameStatement == "SELECT * FROM test.workflow_definitions WHERE workflow_def_name=?;" selectAllWorkflowDefsStatement == "SELECT * FROM test.workflow_defs_index WHERE workflow_def_version_index=?;" selectTaskDefStatement == "SELECT task_definition FROM test.task_definitions WHERE task_defs='task_defs' AND task_def_name=?;" selectAllTaskDefsStatement == "SELECT * FROM test.task_definitions WHERE task_defs=?;" updateWorkflowDefStatement == "UPDATE test.workflow_definitions SET workflow_definition=? WHERE workflow_def_name=? AND version=?;" deleteWorkflowDefStatement == "DELETE FROM test.workflow_definitions WHERE workflow_def_name=? AND version=?;" deleteWorkflowDefIndexStatement == "DELETE FROM test.workflow_defs_index WHERE workflow_def_version_index=? AND workflow_def_name_version=?;" deleteTaskDefStatement == "DELETE FROM test.task_definitions WHERE task_defs='task_defs' AND task_def_name=?;" insertWorkflowStatement == "INSERT INTO test.workflows (workflow_id,shard_id,task_id,entity,payload,total_tasks,total_partitions) VALUES (?,?,?,'workflow',?,?,?);" insertTaskStatement == "INSERT INTO test.workflows (workflow_id,shard_id,task_id,entity,payload) VALUES (?,?,?,'task',?);" insertEventExecutionStatement == "INSERT INTO test.event_executions (message_id,event_handler_name,event_execution_id,payload) VALUES (?,?,?,?) IF NOT EXISTS;" selectTotalStatement == "SELECT total_tasks,total_partitions FROM test.workflows WHERE workflow_id=? AND shard_id=1;" selectTaskStatement == "SELECT payload FROM test.workflows WHERE workflow_id=? AND shard_id=? AND entity='task' AND task_id=?;" selectWorkflowStatement == "SELECT payload FROM test.workflows WHERE workflow_id=? AND shard_id=1 AND entity='workflow';" selectWorkflowWithTasksStatement == "SELECT * FROM test.workflows WHERE workflow_id=? AND shard_id=?;" selectTaskFromLookupTableStatement == "SELECT workflow_id FROM test.task_lookup WHERE task_id=?;" selectTasksFromTaskDefLimitStatement == "SELECT * FROM test.task_def_limit WHERE task_def_name=?;" selectAllEventExecutionsForMessageFromEventExecutionsStatement == "SELECT * FROM test.event_executions WHERE message_id=? AND event_handler_name=?;" updateWorkflowStatement == "UPDATE test.workflows SET payload=? WHERE workflow_id=? AND shard_id=1 AND entity='workflow' AND task_id='';" updateTotalTasksStatement == "UPDATE test.workflows SET total_tasks=? WHERE workflow_id=? AND shard_id=?;" updateTotalPartitionsStatement == "UPDATE test.workflows SET total_partitions=?,total_tasks=? WHERE workflow_id=? AND shard_id=1;" updateTaskLookupStatement == "UPDATE test.task_lookup SET workflow_id=? WHERE task_id=?;" updateTaskDefLimitStatement == "UPDATE test.task_def_limit SET workflow_id=? WHERE task_def_name=? AND task_id=?;" updateEventExecutionStatement == "UPDATE test.event_executions USING TTL ? SET payload=? WHERE message_id=? AND event_handler_name=? AND event_execution_id=?;" deleteWorkflowStatement == "DELETE FROM test.workflows WHERE workflow_id=? AND shard_id=?;" deleteTaskLookupStatement == "DELETE FROM test.task_lookup WHERE task_id=?;" deleteTaskStatement == "DELETE FROM test.workflows WHERE workflow_id=? AND shard_id=? AND entity='task' AND task_id=?;" deleteTaskDefLimitStatement == "DELETE FROM test.task_def_limit WHERE task_def_name=? AND task_id=?;" deleteEventExecutionsStatement == "DELETE FROM test.event_executions WHERE message_id=? AND event_handler_name=? AND event_execution_id=?;" insertEventHandlerStatement == "INSERT INTO test.event_handlers (handlers,event_handler_name,event_handler) VALUES ('handlers',?,?);" selectAllEventHandlersStatement == "SELECT * FROM test.event_handlers WHERE handlers=?;" deleteEventHandlerStatement == "DELETE FROM test.event_handlers WHERE handlers='handlers' AND event_handler_name=?;" } } } ================================================ FILE: client/build.gradle ================================================ buildscript { repositories { maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath "gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.7.5" } } apply plugin: 'groovy' configurations.all { exclude group: 'amazon', module: 'aws-java-sdk' } dependencies { compileOnly 'org.jetbrains:annotations:23.0.0' implementation project(':conductor-common') implementation "com.sun.jersey:jersey-client:${revJersey}" implementation "javax.ws.rs:javax.ws.rs-api:${revJAXRS}" implementation "org.glassfish.jersey.core:jersey-common:${revJerseyCommon}" implementation "com.netflix.spectator:spectator-api:${revSpectator}" implementation ("com.netflix.eureka:eureka-client:${revEurekaClient}") { exclude group: 'com.google.guava', module: 'guava' } implementation "com.amazonaws:aws-java-sdk-core:${revAwsSdk}" implementation "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider" implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" implementation "org.apache.commons:commons-lang3" implementation "commons-io:commons-io:${revCommonsIo}" implementation "org.slf4j:slf4j-api" testImplementation "org.powermock:powermock-module-junit4:${revPowerMock}" testImplementation "org.powermock:powermock-api-mockito2:${revPowerMock}" testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" } ================================================ FILE: client/spotbugsExclude.xml ================================================ ================================================ FILE: client/src/main/java/com/netflix/conductor/client/automator/PollingSemaphore.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.util.concurrent.Semaphore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A class wrapping a semaphore which holds the number of permits available for polling and * executing tasks. */ class PollingSemaphore { private static final Logger LOGGER = LoggerFactory.getLogger(PollingSemaphore.class); private final Semaphore semaphore; PollingSemaphore(int numSlots) { LOGGER.debug("Polling semaphore initialized with {} permits", numSlots); semaphore = new Semaphore(numSlots); } /** Signals that processing is complete and the specified number of permits can be released. */ void complete(int numSlots) { LOGGER.debug("Completed execution; releasing permit"); semaphore.release(numSlots); } /** * Gets the number of threads available for processing. * * @return number of available permits */ int availableSlots() { int available = semaphore.availablePermits(); LOGGER.debug("Number of available permits: {}", available); return available; } /** * Signals if processing is allowed based on whether specified number of permits can be * acquired. * * @param numSlots the number of permits to acquire * @return {@code true} - if permit is acquired {@code false} - if permit could not be acquired */ public boolean acquireSlots(int numSlots) { boolean acquired = semaphore.tryAcquire(numSlots); LOGGER.debug("Trying to acquire {} permit: {}", numSlots, acquired); return acquired; } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.io.PrintWriter; import java.io.StringWriter; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.*; import java.util.function.Function; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.commons.lang3.time.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.appinfo.InstanceInfo.InstanceStatus; import com.netflix.conductor.client.config.PropertyFactory; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.telemetry.MetricsContainer; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.discovery.EurekaClient; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Spectator; import com.netflix.spectator.api.patterns.ThreadPoolMonitor; /** * Manages the threadpool used by the workers for execution and server communication (polling and * task update). */ class TaskPollExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TaskPollExecutor.class); private static final Registry REGISTRY = Spectator.globalRegistry(); private final EurekaClient eurekaClient; private final TaskClient taskClient; private final int updateRetryCount; private final ExecutorService executorService; private final Map pollingSemaphoreMap; private final Map taskToDomain; private static final String DOMAIN = "domain"; private static final String OVERRIDE_DISCOVERY = "pollOutOfDiscovery"; private static final String ALL_WORKERS = "all"; private static final int LEASE_EXTEND_RETRY_COUNT = 3; private static final double LEASE_EXTEND_DURATION_FACTOR = 0.8; private ScheduledExecutorService leaseExtendExecutorService; Map> leaseExtendMap = new HashMap<>(); TaskPollExecutor( EurekaClient eurekaClient, TaskClient taskClient, int updateRetryCount, Map taskToDomain, String workerNamePrefix, Map taskThreadCount) { this.eurekaClient = eurekaClient; this.taskClient = taskClient; this.updateRetryCount = updateRetryCount; this.taskToDomain = taskToDomain; this.pollingSemaphoreMap = new HashMap<>(); int totalThreadCount = 0; for (Map.Entry entry : taskThreadCount.entrySet()) { String taskType = entry.getKey(); int count = entry.getValue(); totalThreadCount += count; pollingSemaphoreMap.put(taskType, new PollingSemaphore(count)); } LOGGER.info("Initialized the TaskPollExecutor with {} threads", totalThreadCount); this.executorService = Executors.newFixedThreadPool( totalThreadCount, new BasicThreadFactory.Builder() .namingPattern(workerNamePrefix) .uncaughtExceptionHandler(uncaughtExceptionHandler) .build()); ThreadPoolMonitor.attach(REGISTRY, (ThreadPoolExecutor) executorService, workerNamePrefix); LOGGER.info("Initialized the task lease extend executor"); leaseExtendExecutorService = Executors.newSingleThreadScheduledExecutor( new BasicThreadFactory.Builder() .namingPattern("workflow-lease-extend-%d") .daemon(true) .uncaughtExceptionHandler(uncaughtExceptionHandler) .build()); } void pollAndExecute(Worker worker) { Boolean discoveryOverride = Optional.ofNullable( PropertyFactory.getBoolean( worker.getTaskDefName(), OVERRIDE_DISCOVERY, null)) .orElseGet( () -> PropertyFactory.getBoolean( ALL_WORKERS, OVERRIDE_DISCOVERY, false)); if (eurekaClient != null && !eurekaClient.getInstanceRemoteStatus().equals(InstanceStatus.UP) && !discoveryOverride) { LOGGER.debug("Instance is NOT UP in discovery - will not poll"); return; } if (worker.paused()) { MetricsContainer.incrementTaskPausedCount(worker.getTaskDefName()); LOGGER.debug("Worker {} has been paused. Not polling anymore!", worker.getClass()); return; } String taskType = worker.getTaskDefName(); PollingSemaphore pollingSemaphore = getPollingSemaphore(taskType); int slotsToAcquire = pollingSemaphore.availableSlots(); if (slotsToAcquire <= 0 || !pollingSemaphore.acquireSlots(slotsToAcquire)) { return; } int acquiredTasks = 0; try { String domain = Optional.ofNullable(PropertyFactory.getString(taskType, DOMAIN, null)) .orElseGet( () -> Optional.ofNullable( PropertyFactory.getString( ALL_WORKERS, DOMAIN, null)) .orElse(taskToDomain.get(taskType))); LOGGER.debug("Polling task of type: {} in domain: '{}'", taskType, domain); List tasks = MetricsContainer.getPollTimer(taskType) .record( () -> taskClient.batchPollTasksInDomain( taskType, domain, worker.getIdentity(), slotsToAcquire, worker.getBatchPollTimeoutInMS())); acquiredTasks = tasks.size(); for (Task task : tasks) { if (Objects.nonNull(task) && StringUtils.isNotBlank(task.getTaskId())) { MetricsContainer.incrementTaskPollCount(taskType, 1); LOGGER.debug( "Polled task: {} of type: {} in domain: '{}', from worker: {}", task.getTaskId(), taskType, domain, worker.getIdentity()); CompletableFuture taskCompletableFuture = CompletableFuture.supplyAsync( () -> processTask(task, worker, pollingSemaphore), executorService); if (task.getResponseTimeoutSeconds() > 0 && worker.leaseExtendEnabled()) { ScheduledFuture leaseExtendFuture = leaseExtendExecutorService.scheduleWithFixedDelay( extendLease(task, taskCompletableFuture), Math.round( task.getResponseTimeoutSeconds() * LEASE_EXTEND_DURATION_FACTOR), Math.round( task.getResponseTimeoutSeconds() * LEASE_EXTEND_DURATION_FACTOR), TimeUnit.SECONDS); leaseExtendMap.put(task.getTaskId(), leaseExtendFuture); } taskCompletableFuture.whenComplete(this::finalizeTask); } else { // no task was returned in the poll, release the permit pollingSemaphore.complete(1); } } } catch (Exception e) { MetricsContainer.incrementTaskPollErrorCount(worker.getTaskDefName(), e); LOGGER.error("Error when polling for tasks", e); } // immediately release unused permits pollingSemaphore.complete(slotsToAcquire - acquiredTasks); } void shutdown(int timeout) { shutdownAndAwaitTermination(executorService, timeout); shutdownAndAwaitTermination(leaseExtendExecutorService, timeout); leaseExtendMap.clear(); } void shutdownAndAwaitTermination(ExecutorService executorService, int timeout) { try { executorService.shutdown(); if (executorService.awaitTermination(timeout, TimeUnit.SECONDS)) { LOGGER.debug("tasks completed, shutting down"); } else { LOGGER.warn(String.format("forcing shutdown after waiting for %s second", timeout)); executorService.shutdownNow(); } } catch (InterruptedException ie) { LOGGER.warn("shutdown interrupted, invoking shutdownNow"); executorService.shutdownNow(); Thread.currentThread().interrupt(); } } @SuppressWarnings("FieldCanBeLocal") private final Thread.UncaughtExceptionHandler uncaughtExceptionHandler = (thread, error) -> { // JVM may be in unstable state, try to send metrics then exit MetricsContainer.incrementUncaughtExceptionCount(); LOGGER.error("Uncaught exception. Thread {} will exit now", thread, error); }; private Task processTask(Task task, Worker worker, PollingSemaphore pollingSemaphore) { LOGGER.debug( "Executing task: {} of type: {} in worker: {} at {}", task.getTaskId(), task.getTaskDefName(), worker.getClass().getSimpleName(), worker.getIdentity()); try { executeTask(worker, task); } catch (Throwable t) { task.setStatus(Task.Status.FAILED); TaskResult result = new TaskResult(task); handleException(t, result, worker, task); } finally { pollingSemaphore.complete(1); } return task; } private void executeTask(Worker worker, Task task) { StopWatch stopwatch = new StopWatch(); stopwatch.start(); TaskResult result = null; try { LOGGER.debug( "Executing task: {} in worker: {} at {}", task.getTaskId(), worker.getClass().getSimpleName(), worker.getIdentity()); result = worker.execute(task); result.setWorkflowInstanceId(task.getWorkflowInstanceId()); result.setTaskId(task.getTaskId()); result.setWorkerId(worker.getIdentity()); } catch (Exception e) { LOGGER.error( "Unable to execute task: {} of type: {}", task.getTaskId(), task.getTaskDefName(), e); if (result == null) { task.setStatus(Task.Status.FAILED); result = new TaskResult(task); } handleException(e, result, worker, task); } finally { stopwatch.stop(); MetricsContainer.getExecutionTimer(worker.getTaskDefName()) .record(stopwatch.getTime(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); } LOGGER.debug( "Task: {} executed by worker: {} at {} with status: {}", task.getTaskId(), worker.getClass().getSimpleName(), worker.getIdentity(), result.getStatus()); updateTaskResult(updateRetryCount, task, result, worker); } private void finalizeTask(Task task, Throwable throwable) { if (throwable != null) { LOGGER.error( "Error processing task: {} of type: {}", task.getTaskId(), task.getTaskType(), throwable); MetricsContainer.incrementTaskExecutionErrorCount(task.getTaskType(), throwable); } else { LOGGER.debug( "Task:{} of type:{} finished processing with status:{}", task.getTaskId(), task.getTaskDefName(), task.getStatus()); String taskId = task.getTaskId(); ScheduledFuture leaseExtendFuture = leaseExtendMap.get(taskId); if (leaseExtendFuture != null) { leaseExtendFuture.cancel(true); leaseExtendMap.remove(taskId); } } } private void updateTaskResult(int count, Task task, TaskResult result, Worker worker) { try { // upload if necessary Optional optionalExternalStorageLocation = retryOperation( (TaskResult taskResult) -> upload(taskResult, task.getTaskType()), count, result, "evaluateAndUploadLargePayload"); if (optionalExternalStorageLocation.isPresent()) { result.setExternalOutputPayloadStoragePath(optionalExternalStorageLocation.get()); result.setOutputData(null); } retryOperation( (TaskResult taskResult) -> { taskClient.updateTask(taskResult); return null; }, count, result, "updateTask"); } catch (Exception e) { worker.onErrorUpdate(task); MetricsContainer.incrementTaskUpdateErrorCount(worker.getTaskDefName(), e); LOGGER.error( String.format( "Failed to update result: %s for task: %s in worker: %s", result.toString(), task.getTaskDefName(), worker.getIdentity()), e); } } private Optional upload(TaskResult result, String taskType) { try { return taskClient.evaluateAndUploadLargePayload(result.getOutputData(), taskType); } catch (IllegalArgumentException iae) { result.setReasonForIncompletion(iae.getMessage()); result.setOutputData(null); result.setStatus(TaskResult.Status.FAILED_WITH_TERMINAL_ERROR); return Optional.empty(); } } private R retryOperation(Function operation, int count, T input, String opName) { int index = 0; while (index < count) { try { return operation.apply(input); } catch (Exception e) { index++; try { Thread.sleep(500L); } catch (InterruptedException ie) { LOGGER.error("Retry interrupted", ie); } } } throw new RuntimeException("Exhausted retries performing " + opName); } private void handleException(Throwable t, TaskResult result, Worker worker, Task task) { LOGGER.error(String.format("Error while executing task %s", task.toString()), t); MetricsContainer.incrementTaskExecutionErrorCount(worker.getTaskDefName(), t); result.setStatus(TaskResult.Status.FAILED); result.setReasonForIncompletion("Error while executing the task: " + t); StringWriter stringWriter = new StringWriter(); t.printStackTrace(new PrintWriter(stringWriter)); result.log(stringWriter.toString()); updateTaskResult(updateRetryCount, task, result, worker); } private PollingSemaphore getPollingSemaphore(String taskType) { return pollingSemaphoreMap.get(taskType); } private Runnable extendLease(Task task, CompletableFuture taskCompletableFuture) { return () -> { if (taskCompletableFuture.isDone()) { LOGGER.warn( "Task processing for {} completed, but its lease extend was not cancelled", task.getTaskId()); return; } LOGGER.info("Attempting to extend lease for {}", task.getTaskId()); try { TaskResult result = new TaskResult(task); result.setExtendLease(true); retryOperation( (TaskResult taskResult) -> { taskClient.updateTask(taskResult); return null; }, LEASE_EXTEND_RETRY_COUNT, result, "extend lease"); MetricsContainer.incrementTaskLeaseExtendCount(task.getTaskDefName(), 1); } catch (Exception e) { MetricsContainer.incrementTaskLeaseExtendErrorCount(task.getTaskDefName(), e); LOGGER.error("Failed to extend lease for {}", task.getTaskId(), e); } }; } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.discovery.EurekaClient; /** Configures automated polling of tasks and execution via the registered {@link Worker}s. */ public class TaskRunnerConfigurer { private static final Logger LOGGER = LoggerFactory.getLogger(TaskRunnerConfigurer.class); private static final String INVALID_THREAD_COUNT = "Invalid worker thread count specified, use either shared thread pool or config thread count per task"; private static final String MISSING_TASK_THREAD_COUNT = "Missing task thread count config for %s"; private ScheduledExecutorService scheduledExecutorService; private final EurekaClient eurekaClient; private final TaskClient taskClient; private final List workers = new LinkedList<>(); private final int sleepWhenRetry; private final int updateRetryCount; @Deprecated private final int threadCount; private final int shutdownGracePeriodSeconds; private final String workerNamePrefix; private final Map taskToDomain; private final Map taskThreadCount; private TaskPollExecutor taskPollExecutor; /** * @see TaskRunnerConfigurer.Builder * @see TaskRunnerConfigurer#init() */ private TaskRunnerConfigurer(Builder builder) { // only allow either shared thread pool or per task thread pool if (builder.threadCount != -1 && !builder.taskThreadCount.isEmpty()) { LOGGER.error(INVALID_THREAD_COUNT); throw new ConductorClientException(INVALID_THREAD_COUNT); } else if (!builder.taskThreadCount.isEmpty()) { for (Worker worker : builder.workers) { if (!builder.taskThreadCount.containsKey(worker.getTaskDefName())) { LOGGER.info( "No thread count specified for task type {}, default to 1 thread", worker.getTaskDefName()); builder.taskThreadCount.put(worker.getTaskDefName(), 1); } workers.add(worker); } this.taskThreadCount = builder.taskThreadCount; this.threadCount = -1; } else { Set taskTypes = new HashSet<>(); for (Worker worker : builder.workers) { taskTypes.add(worker.getTaskDefName()); workers.add(worker); } this.threadCount = (builder.threadCount == -1) ? workers.size() : builder.threadCount; // shared thread pool will be evenly split between task types int splitThreadCount = threadCount / taskTypes.size(); this.taskThreadCount = taskTypes.stream().collect(Collectors.toMap(v -> v, v -> splitThreadCount)); } this.eurekaClient = builder.eurekaClient; this.taskClient = builder.taskClient; this.sleepWhenRetry = builder.sleepWhenRetry; this.updateRetryCount = builder.updateRetryCount; this.workerNamePrefix = builder.workerNamePrefix; this.taskToDomain = builder.taskToDomain; this.shutdownGracePeriodSeconds = builder.shutdownGracePeriodSeconds; } /** Builder used to create the instances of TaskRunnerConfigurer */ public static class Builder { private String workerNamePrefix = "workflow-worker-%d"; private int sleepWhenRetry = 500; private int updateRetryCount = 3; @Deprecated private int threadCount = -1; private int shutdownGracePeriodSeconds = 10; private final Iterable workers; private EurekaClient eurekaClient; private final TaskClient taskClient; private Map taskToDomain = new HashMap<>(); private Map taskThreadCount = new HashMap<>(); public Builder(TaskClient taskClient, Iterable workers) { Validate.notNull(taskClient, "TaskClient cannot be null"); Validate.notNull(workers, "Workers cannot be null"); this.taskClient = taskClient; this.workers = workers; } /** * @param workerNamePrefix prefix to be used for worker names, defaults to workflow-worker- * if not supplied. * @return Returns the current instance. */ public Builder withWorkerNamePrefix(String workerNamePrefix) { this.workerNamePrefix = workerNamePrefix; return this; } /** * @param sleepWhenRetry time in milliseconds, for which the thread should sleep when task * update call fails, before retrying the operation. * @return Returns the current instance. */ public Builder withSleepWhenRetry(int sleepWhenRetry) { this.sleepWhenRetry = sleepWhenRetry; return this; } /** * @param updateRetryCount number of times to retry the failed updateTask operation * @return Builder instance * @see #withSleepWhenRetry(int) */ public Builder withUpdateRetryCount(int updateRetryCount) { this.updateRetryCount = updateRetryCount; return this; } /** * @param threadCount # of threads assigned to the workers. Should be at-least the size of * taskWorkers to avoid starvation in a busy system. * @return Builder instance * @deprecated Use {@link TaskRunnerConfigurer.Builder#withTaskThreadCount(Map)} instead. */ @Deprecated public Builder withThreadCount(int threadCount) { if (threadCount < 1) { throw new IllegalArgumentException("No. of threads cannot be less than 1"); } this.threadCount = threadCount; return this; } /** * @param shutdownGracePeriodSeconds waiting seconds before forcing shutdown of your worker * @return Builder instance */ public Builder withShutdownGracePeriodSeconds(int shutdownGracePeriodSeconds) { if (shutdownGracePeriodSeconds < 1) { throw new IllegalArgumentException( "Seconds of shutdownGracePeriod cannot be less than 1"); } this.shutdownGracePeriodSeconds = shutdownGracePeriodSeconds; return this; } /** * @param eurekaClient Eureka client - used to identify if the server is in discovery or * not. When the server goes out of discovery, the polling is terminated. If passed * null, discovery check is not done. * @return Builder instance */ public Builder withEurekaClient(EurekaClient eurekaClient) { this.eurekaClient = eurekaClient; return this; } public Builder withTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; return this; } public Builder withTaskThreadCount(Map taskThreadCount) { this.taskThreadCount = taskThreadCount; return this; } /** * Builds an instance of the TaskRunnerConfigurer. * *

Please see {@link TaskRunnerConfigurer#init()} method. The method must be called after * this constructor for the polling to start. */ public TaskRunnerConfigurer build() { return new TaskRunnerConfigurer(this); } } /** * @return Thread Count for the shared executor pool */ @Deprecated public int getThreadCount() { return threadCount; } /** * @return Thread Count for individual task type */ public Map getTaskThreadCount() { return taskThreadCount; } /** * @return seconds before forcing shutdown of worker */ public int getShutdownGracePeriodSeconds() { return shutdownGracePeriodSeconds; } /** * @return sleep time in millisecond before task update retry is done when receiving error from * the Conductor server */ public int getSleepWhenRetry() { return sleepWhenRetry; } /** * @return Number of times updateTask should be retried when receiving error from Conductor * server */ public int getUpdateRetryCount() { return updateRetryCount; } /** * @return prefix used for worker names */ public String getWorkerNamePrefix() { return workerNamePrefix; } /** * Starts the polling. Must be called after {@link TaskRunnerConfigurer.Builder#build()} method. */ public synchronized void init() { this.taskPollExecutor = new TaskPollExecutor( eurekaClient, taskClient, updateRetryCount, taskToDomain, workerNamePrefix, taskThreadCount); this.scheduledExecutorService = Executors.newScheduledThreadPool(workers.size()); workers.forEach( worker -> scheduledExecutorService.scheduleWithFixedDelay( () -> taskPollExecutor.pollAndExecute(worker), worker.getPollingInterval(), worker.getPollingInterval(), TimeUnit.MILLISECONDS)); } /** * Invoke this method within a PreDestroy block within your application to facilitate a graceful * shutdown of your worker, during process termination. */ public void shutdown() { taskPollExecutor.shutdownAndAwaitTermination( scheduledExecutorService, shutdownGracePeriodSeconds); taskPollExecutor.shutdown(shutdownGracePeriodSeconds); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/config/ConductorClientConfiguration.java ================================================ /* * Copyright 2018 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.config; public interface ConductorClientConfiguration { /** * @return the workflow input payload size threshold in KB, beyond which the payload will be * processed based on {@link * ConductorClientConfiguration#isExternalPayloadStorageEnabled()}. */ int getWorkflowInputPayloadThresholdKB(); /** * @return the max value of workflow input payload size threshold in KB, beyond which the * payload will be rejected regardless external payload storage is enabled. */ int getWorkflowInputMaxPayloadThresholdKB(); /** * @return the task output payload size threshold in KB, beyond which the payload will be * processed based on {@link * ConductorClientConfiguration#isExternalPayloadStorageEnabled()}. */ int getTaskOutputPayloadThresholdKB(); /** * @return the max value of task output payload size threshold in KB, beyond which the payload * will be rejected regardless external payload storage is enabled. */ int getTaskOutputMaxPayloadThresholdKB(); /** * @return the flag which controls the use of external storage for storing workflow/task input * and output JSON payloads with size greater than threshold. If it is set to true, the * payload is stored in external location. If it is set to false, the payload is rejected * and the task/workflow execution fails. */ boolean isExternalPayloadStorageEnabled(); } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/config/DefaultConductorClientConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.config; /** * A default implementation of {@link ConductorClientConfiguration} where external payload storage * is disabled. */ public class DefaultConductorClientConfiguration implements ConductorClientConfiguration { @Override public int getWorkflowInputPayloadThresholdKB() { return 5120; } @Override public int getWorkflowInputMaxPayloadThresholdKB() { return 10240; } @Override public int getTaskOutputPayloadThresholdKB() { return 3072; } @Override public int getTaskOutputMaxPayloadThresholdKB() { return 10240; } @Override public boolean isExternalPayloadStorageEnabled() { return false; } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/config/PropertyFactory.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.config; import java.util.concurrent.ConcurrentHashMap; import com.netflix.config.DynamicProperty; /** Used to configure the Conductor workers using properties. */ public class PropertyFactory { private final DynamicProperty global; private final DynamicProperty local; private static final String PROPERTY_PREFIX = "conductor.worker"; private static final ConcurrentHashMap PROPERTY_FACTORY_MAP = new ConcurrentHashMap<>(); private PropertyFactory(String prefix, String propName, String workerName) { this.global = DynamicProperty.getInstance(prefix + "." + propName); this.local = DynamicProperty.getInstance(prefix + "." + workerName + "." + propName); } /** * @param defaultValue Default Value * @return Returns the value as integer. If not value is set (either global or worker specific), * then returns the default value. */ public Integer getInteger(int defaultValue) { Integer value = local.getInteger(); if (value == null) { value = global.getInteger(defaultValue); } return value; } /** * @param defaultValue Default Value * @return Returns the value as String. If not value is set (either global or worker specific), * then returns the default value. */ public String getString(String defaultValue) { String value = local.getString(); if (value == null) { value = global.getString(defaultValue); } return value; } /** * @param defaultValue Default Value * @return Returns the value as Boolean. If not value is set (either global or worker specific), * then returns the default value. */ public Boolean getBoolean(Boolean defaultValue) { Boolean value = local.getBoolean(); if (value == null) { value = global.getBoolean(defaultValue); } return value; } public static Integer getInteger(String workerName, String property, Integer defaultValue) { return getPropertyFactory(workerName, property).getInteger(defaultValue); } public static Boolean getBoolean(String workerName, String property, Boolean defaultValue) { return getPropertyFactory(workerName, property).getBoolean(defaultValue); } public static String getString(String workerName, String property, String defaultValue) { return getPropertyFactory(workerName, property).getString(defaultValue); } private static PropertyFactory getPropertyFactory(String workerName, String property) { String key = property + "." + workerName; return PROPERTY_FACTORY_MAP.computeIfAbsent( key, t -> new PropertyFactory(PROPERTY_PREFIX, property, workerName)); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/exception/ConductorClientException.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.exception; import java.util.List; import com.netflix.conductor.common.validation.ErrorResponse; import com.netflix.conductor.common.validation.ValidationError; /** Client exception thrown from Conductor api clients. */ public class ConductorClientException extends RuntimeException { private int status; private String message; private String instance; private String code; private boolean retryable; public List getValidationErrors() { return validationErrors; } public void setValidationErrors(List validationErrors) { this.validationErrors = validationErrors; } private List validationErrors; public ConductorClientException() { super(); } public ConductorClientException(String message) { super(message); this.message = message; } public ConductorClientException(String message, Throwable cause) { super(message, cause); this.message = message; } public ConductorClientException(int status, String message) { super(message); this.status = status; this.message = message; } public ConductorClientException(int status, ErrorResponse errorResponse) { super(errorResponse.getMessage()); this.status = status; this.retryable = errorResponse.isRetryable(); this.message = errorResponse.getMessage(); this.code = errorResponse.getCode(); this.instance = errorResponse.getInstance(); this.validationErrors = errorResponse.getValidationErrors(); } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append(getClass().getName()).append(": "); if (this.message != null) { builder.append(message); } if (status > 0) { builder.append(" {status=").append(status); if (this.code != null) { builder.append(", code='").append(code).append("'"); } builder.append(", retryable: ").append(retryable); } if (this.instance != null) { builder.append(", instance: ").append(instance); } if (this.validationErrors != null) { builder.append(", validationErrors: ").append(validationErrors.toString()); } builder.append("}"); return builder.toString(); } public String getCode() { return code; } public void setCode(String code) { this.code = code; } public void setStatus(int status) { this.status = status; } public void setMessage(String message) { this.message = message; } public String getInstance() { return instance; } public void setInstance(String instance) { this.instance = instance; } public boolean isRetryable() { return retryable; } public void setRetryable(boolean retryable) { this.retryable = retryable; } @Override public String getMessage() { return this.message; } public int getStatus() { return this.status; } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/ClientBase.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.Collection; import java.util.Map; import java.util.function.Function; import javax.ws.rs.core.UriBuilder; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.config.ConductorClientConfiguration; import com.netflix.conductor.client.config.DefaultConductorClientConfiguration; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.common.validation.ErrorResponse; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource.Builder; /** Abstract client for the REST server */ public abstract class ClientBase { private static final Logger LOGGER = LoggerFactory.getLogger(ClientBase.class); protected ClientRequestHandler requestHandler; protected String root = ""; protected ObjectMapper objectMapper; protected PayloadStorage payloadStorage; protected ConductorClientConfiguration conductorClientConfiguration; protected ClientBase( ClientRequestHandler requestHandler, ConductorClientConfiguration clientConfiguration) { this.objectMapper = new ObjectMapperProvider().getObjectMapper(); // https://github.com/FasterXML/jackson-databind/issues/2683 if (isNewerJacksonVersion()) { objectMapper.registerModule(new JavaTimeModule()); } this.requestHandler = requestHandler; this.conductorClientConfiguration = ObjectUtils.defaultIfNull( clientConfiguration, new DefaultConductorClientConfiguration()); this.payloadStorage = new PayloadStorage(this); } public void setRootURI(String root) { this.root = root; } protected void delete(String url, Object... uriVariables) { deleteWithUriVariables(null, url, uriVariables); } protected void deleteWithUriVariables( Object[] queryParams, String url, Object... uriVariables) { delete(queryParams, url, uriVariables, null); } protected BulkResponse deleteWithRequestBody(Object[] queryParams, String url, Object body) { return delete(queryParams, url, null, body); } private BulkResponse delete( Object[] queryParams, String url, Object[] uriVariables, Object body) { URI uri = null; BulkResponse response = null; try { uri = getURIBuilder(root + url, queryParams).build(uriVariables); response = requestHandler.delete(uri, body); } catch (UniformInterfaceException e) { handleUniformInterfaceException(e, uri); } catch (RuntimeException e) { handleRuntimeException(e, uri); } return response; } protected void put(String url, Object[] queryParams, Object request, Object... uriVariables) { URI uri = null; try { uri = getURIBuilder(root + url, queryParams).build(uriVariables); requestHandler.getWebResourceBuilder(uri, request).put(); } catch (RuntimeException e) { handleException(uri, e); } } protected void postForEntityWithRequestOnly(String url, Object request) { Class type = null; postForEntity(url, request, null, type); } protected void postForEntityWithUriVariablesOnly(String url, Object... uriVariables) { Class type = null; postForEntity(url, null, null, type, uriVariables); } protected T postForEntity( String url, Object request, Object[] queryParams, Class responseType, Object... uriVariables) { return postForEntity( url, request, queryParams, responseType, builder -> builder.post(responseType), uriVariables); } protected T postForEntity( String url, Object request, Object[] queryParams, GenericType responseType, Object... uriVariables) { return postForEntity( url, request, queryParams, responseType, builder -> builder.post(responseType), uriVariables); } private T postForEntity( String url, Object request, Object[] queryParams, Object responseType, Function postWithEntity, Object... uriVariables) { URI uri = null; try { uri = getURIBuilder(root + url, queryParams).build(uriVariables); Builder webResourceBuilder = requestHandler.getWebResourceBuilder(uri, request); if (responseType == null) { webResourceBuilder.post(); return null; } return postWithEntity.apply(webResourceBuilder); } catch (UniformInterfaceException e) { handleUniformInterfaceException(e, uri); } catch (RuntimeException e) { handleRuntimeException(e, uri); } return null; } protected T getForEntity( String url, Object[] queryParams, Class responseType, Object... uriVariables) { return getForEntity( url, queryParams, response -> response.getEntity(responseType), uriVariables); } protected T getForEntity( String url, Object[] queryParams, GenericType responseType, Object... uriVariables) { return getForEntity( url, queryParams, response -> response.getEntity(responseType), uriVariables); } private T getForEntity( String url, Object[] queryParams, Function entityProvider, Object... uriVariables) { URI uri = null; ClientResponse clientResponse; try { uri = getURIBuilder(root + url, queryParams).build(uriVariables); clientResponse = requestHandler.get(uri); if (clientResponse.getStatus() < 300) { return entityProvider.apply(clientResponse); } else { throw new UniformInterfaceException(clientResponse); } } catch (UniformInterfaceException e) { handleUniformInterfaceException(e, uri); } catch (RuntimeException e) { handleRuntimeException(e, uri); } return null; } /** * Uses the {@link PayloadStorage} for storing large payloads. Gets the uri for storing the * payload from the server and then uploads to this location * * @param payloadType the {@link * com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be uploaded * @param payloadBytes the byte array containing the payload * @param payloadSize the size of the payload * @return the path where the payload is stored in external storage */ protected String uploadToExternalPayloadStorage( ExternalPayloadStorage.PayloadType payloadType, byte[] payloadBytes, long payloadSize) { Validate.isTrue( payloadType.equals(ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT) || payloadType.equals(ExternalPayloadStorage.PayloadType.TASK_OUTPUT), "Payload type must be workflow input or task output"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); payloadStorage.upload( externalStorageLocation.getUri(), new ByteArrayInputStream(payloadBytes), payloadSize); return externalStorageLocation.getPath(); } /** * Uses the {@link PayloadStorage} for downloading large payloads to be used by the client. Gets * the uri of the payload fom the server and then downloads from this location. * * @param payloadType the {@link * com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be downloaded * @param path the relative of the payload in external storage * @return the payload object that is stored in external storage */ @SuppressWarnings("unchecked") protected Map downloadFromExternalStorage( ExternalPayloadStorage.PayloadType payloadType, String path) { Validate.notBlank(path, "uri cannot be blank"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation( ExternalPayloadStorage.Operation.READ, payloadType, path); try (InputStream inputStream = payloadStorage.download(externalStorageLocation.getUri())) { return objectMapper.readValue(inputStream, Map.class); } catch (IOException e) { String errorMsg = String.format( "Unable to download payload from external storage location: %s", path); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } } private UriBuilder getURIBuilder(String path, Object[] queryParams) { if (path == null) { path = ""; } UriBuilder builder = UriBuilder.fromPath(path); if (queryParams != null) { for (int i = 0; i < queryParams.length; i += 2) { String param = queryParams[i].toString(); Object value = queryParams[i + 1]; if (value != null) { if (value instanceof Collection) { Object[] values = ((Collection) value).toArray(); builder.queryParam(param, values); } else { builder.queryParam(param, value); } } } } return builder; } protected boolean isNewerJacksonVersion() { Version version = com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION; return version.getMajorVersion() == 2 && version.getMinorVersion() >= 12; } private void handleClientHandlerException(ClientHandlerException exception, URI uri) { String errorMessage = String.format( "Unable to invoke Conductor API with uri: %s, failure to process request or response", uri); LOGGER.error(errorMessage, exception); throw new ConductorClientException(errorMessage, exception); } private void handleRuntimeException(RuntimeException exception, URI uri) { String errorMessage = String.format( "Unable to invoke Conductor API with uri: %s, runtime exception occurred", uri); LOGGER.error(errorMessage, exception); throw new ConductorClientException(errorMessage, exception); } private void handleUniformInterfaceException(UniformInterfaceException exception, URI uri) { ClientResponse clientResponse = exception.getResponse(); if (clientResponse == null) { throw new ConductorClientException( String.format("Unable to invoke Conductor API with uri: %s", uri)); } try { if (clientResponse.getStatus() < 300) { return; } String errorMessage = clientResponse.getEntity(String.class); LOGGER.warn( "Unable to invoke Conductor API with uri: {}, unexpected response from server: statusCode={}, responseBody='{}'.", uri, clientResponse.getStatus(), errorMessage); ErrorResponse errorResponse; try { errorResponse = objectMapper.readValue(errorMessage, ErrorResponse.class); } catch (IOException e) { throw new ConductorClientException(clientResponse.getStatus(), errorMessage); } throw new ConductorClientException(clientResponse.getStatus(), errorResponse); } catch (ConductorClientException e) { throw e; } catch (ClientHandlerException e) { handleClientHandlerException(e, uri); } catch (RuntimeException e) { handleRuntimeException(e, uri); } finally { clientResponse.close(); } } private void handleException(URI uri, RuntimeException e) { if (e instanceof UniformInterfaceException) { handleUniformInterfaceException(((UniformInterfaceException) e), uri); } else if (e instanceof ClientHandlerException) { handleClientHandlerException((ClientHandlerException) e, uri); } else { handleRuntimeException(e, uri); } } /** * Converts ClientResponse object to string with detailed debug information including status * code, media type, response headers, and response body if exists. */ private String clientResponseToString(ClientResponse response) { if (response == null) { return null; } StringBuilder builder = new StringBuilder(); builder.append("[status: ").append(response.getStatus()); builder.append(", media type: ").append(response.getType()); if (response.getStatus() != 404) { try { String responseBody = response.getEntity(String.class); if (responseBody != null) { builder.append(", response body: ").append(responseBody); } } catch (RuntimeException ignore) { // Ignore if there is no response body, or IO error - it may have already been read // in certain scenario. } } builder.append(", response headers: ").append(response.getHeaders()); builder.append("]"); return builder.toString(); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/ClientRequestHandler.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.net.URI; import javax.ws.rs.core.MediaType; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.common.model.BulkResponse; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; public class ClientRequestHandler { private final Client client; public ClientRequestHandler( ClientConfig config, ClientHandler handler, ClientFilter... filters) { ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); // https://github.com/FasterXML/jackson-databind/issues/2683 if (isNewerJacksonVersion()) { objectMapper.registerModule(new JavaTimeModule()); } JacksonJsonProvider provider = new JacksonJsonProvider(objectMapper); config.getSingletons().add(provider); if (handler == null) { this.client = Client.create(config); } else { this.client = new Client(handler, config); } for (ClientFilter filter : filters) { this.client.addFilter(filter); } } public BulkResponse delete(URI uri, Object body) { if (body != null) { return client.resource(uri) .type(MediaType.APPLICATION_JSON_TYPE) .delete(BulkResponse.class, body); } else { client.resource(uri).delete(); } return null; } public ClientResponse get(URI uri) { return client.resource(uri) .accept(MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN) .get(ClientResponse.class); } public WebResource.Builder getWebResourceBuilder(URI URI, Object entity) { return client.resource(URI) .type(MediaType.APPLICATION_JSON) .entity(entity) .accept(MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON); } private boolean isNewerJacksonVersion() { Version version = com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION; return version.getMajorVersion() == 2 && version.getMinorVersion() >= 12; } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/EventClient.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.util.List; import org.apache.commons.lang3.Validate; import com.netflix.conductor.client.config.ConductorClientConfiguration; import com.netflix.conductor.client.config.DefaultConductorClientConfiguration; import com.netflix.conductor.common.metadata.events.EventHandler; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; // Client class for all Event Handler operations public class EventClient extends ClientBase { private static final GenericType> eventHandlerList = new GenericType>() {}; /** Creates a default metadata client */ public EventClient() { this(new DefaultClientConfig(), new DefaultConductorClientConfiguration(), null); } /** * @param clientConfig REST Client configuration */ public EventClient(ClientConfig clientConfig) { this(clientConfig, new DefaultConductorClientConfiguration(), null); } /** * @param clientConfig REST Client configuration * @param clientHandler Jersey client handler. Useful when plugging in various http client * interaction modules (e.g. ribbon) */ public EventClient(ClientConfig clientConfig, ClientHandler clientHandler) { this(clientConfig, new DefaultConductorClientConfiguration(), clientHandler); } /** * @param config config REST Client configuration * @param handler handler Jersey client handler. Useful when plugging in various http client * interaction modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public EventClient(ClientConfig config, ClientHandler handler, ClientFilter... filters) { this(config, new DefaultConductorClientConfiguration(), handler, filters); } /** * @param config REST Client configuration * @param clientConfiguration Specific properties configured for the client, see {@link * ConductorClientConfiguration} * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public EventClient( ClientConfig config, ConductorClientConfiguration clientConfiguration, ClientHandler handler, ClientFilter... filters) { super(new ClientRequestHandler(config, handler, filters), clientConfiguration); } EventClient(ClientRequestHandler requestHandler) { super(requestHandler, null); } /** * Register an event handler with the server * * @param eventHandler the eventHandler definition */ public void registerEventHandler(EventHandler eventHandler) { Validate.notNull(eventHandler, "Event Handler definition cannot be null"); postForEntityWithRequestOnly("event", eventHandler); } /** * Updates an event handler with the server * * @param eventHandler the eventHandler definition */ public void updateEventHandler(EventHandler eventHandler) { Validate.notNull(eventHandler, "Event Handler definition cannot be null"); put("event", null, eventHandler); } /** * @param event name of the event * @param activeOnly if true, returns only the active handlers * @return Returns the list of all the event handlers for a given event */ public List getEventHandlers(String event, boolean activeOnly) { Validate.notBlank(event, "Event cannot be blank"); return getForEntity( "event/{event}", new Object[] {"activeOnly", activeOnly}, eventHandlerList, event); } /** * Removes the event handler definition from the conductor server * * @param name the name of the event handler to be unregistered */ public void unregisterEventHandler(String name) { Validate.notBlank(name, "Event handler name cannot be blank"); delete("event/{name}", name); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/MetadataClient.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.util.List; import org.apache.commons.lang3.Validate; import com.netflix.conductor.client.config.ConductorClientConfiguration; import com.netflix.conductor.client.config.DefaultConductorClientConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; public class MetadataClient extends ClientBase { private static final GenericType> workflowDefList = new GenericType>() {}; /** Creates a default metadata client */ public MetadataClient() { this(new DefaultClientConfig(), new DefaultConductorClientConfiguration(), null); } /** * @param clientConfig REST Client configuration */ public MetadataClient(ClientConfig clientConfig) { this(clientConfig, new DefaultConductorClientConfiguration(), null); } /** * @param clientConfig REST Client configuration * @param clientHandler Jersey client handler. Useful when plugging in various http client * interaction modules (e.g. ribbon) */ public MetadataClient(ClientConfig clientConfig, ClientHandler clientHandler) { this(clientConfig, new DefaultConductorClientConfiguration(), clientHandler); } /** * @param config config REST Client configuration * @param handler handler Jersey client handler. Useful when plugging in various http client * interaction modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public MetadataClient(ClientConfig config, ClientHandler handler, ClientFilter... filters) { this(config, new DefaultConductorClientConfiguration(), handler, filters); } /** * @param config REST Client configuration * @param clientConfiguration Specific properties configured for the client, see {@link * ConductorClientConfiguration} * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public MetadataClient( ClientConfig config, ConductorClientConfiguration clientConfiguration, ClientHandler handler, ClientFilter... filters) { super(new ClientRequestHandler(config, handler, filters), clientConfiguration); } MetadataClient(ClientRequestHandler requestHandler) { super(requestHandler, null); } // Workflow Metadata Operations /** * Register a workflow definition with the server * * @param workflowDef the workflow definition */ public void registerWorkflowDef(WorkflowDef workflowDef) { Validate.notNull(workflowDef, "Workflow definition cannot be null"); postForEntityWithRequestOnly("metadata/workflow", workflowDef); } public void validateWorkflowDef(WorkflowDef workflowDef) { Validate.notNull(workflowDef, "Workflow definition cannot be null"); postForEntityWithRequestOnly("metadata/workflow/validate", workflowDef); } /** * Updates a list of existing workflow definitions * * @param workflowDefs List of workflow definitions to be updated */ public void updateWorkflowDefs(List workflowDefs) { Validate.notNull(workflowDefs, "Workflow defs list cannot be null"); put("metadata/workflow", null, workflowDefs); } /** * Retrieve the workflow definition * * @param name the name of the workflow * @param version the version of the workflow def * @return Workflow definition for the given workflow and version */ public WorkflowDef getWorkflowDef(String name, Integer version) { Validate.notBlank(name, "name cannot be blank"); return getForEntity( "metadata/workflow/{name}", new Object[] {"version", version}, WorkflowDef.class, name); } /** */ public List getAllWorkflowsWithLatestVersions() { return getForEntity( "metadata/workflow/latest-versions", null, workflowDefList, (Object) null); } /** * Removes the workflow definition of a workflow from the conductor server. It does not remove * associated workflows. Use with caution. * * @param name Name of the workflow to be unregistered. * @param version Version of the workflow definition to be unregistered. */ public void unregisterWorkflowDef(String name, Integer version) { Validate.notBlank(name, "Workflow name cannot be blank"); Validate.notNull(version, "Version cannot be null"); delete("metadata/workflow/{name}/{version}", name, version); } // Task Metadata Operations /** * Registers a list of task types with the conductor server * * @param taskDefs List of task types to be registered. */ public void registerTaskDefs(List taskDefs) { Validate.notNull(taskDefs, "Task defs list cannot be null"); postForEntityWithRequestOnly("metadata/taskdefs", taskDefs); } /** * Updates an existing task definition * * @param taskDef the task definition to be updated */ public void updateTaskDef(TaskDef taskDef) { Validate.notNull(taskDef, "Task definition cannot be null"); put("metadata/taskdefs", null, taskDef); } /** * Retrieve the task definition of a given task type * * @param taskType type of task for which to retrieve the definition * @return Task Definition for the given task type */ public TaskDef getTaskDef(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); return getForEntity("metadata/taskdefs/{tasktype}", null, TaskDef.class, taskType); } /** * Removes the task definition of a task type from the conductor server. Use with caution. * * @param taskType Task type to be unregistered. */ public void unregisterTaskDef(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); delete("metadata/taskdefs/{tasktype}", taskType); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/PayloadStorage.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import javax.ws.rs.core.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.amazonaws.util.IOUtils; /** An implementation of {@link ExternalPayloadStorage} for storing large JSON payload data. */ class PayloadStorage implements ExternalPayloadStorage { private static final Logger LOGGER = LoggerFactory.getLogger(PayloadStorage.class); private final ClientBase clientBase; PayloadStorage(ClientBase clientBase) { this.clientBase = clientBase; } /** * This method is not intended to be used in the client. The client makes a request to the * server to get the {@link ExternalStorageLocation} */ @Override public ExternalStorageLocation getLocation( Operation operation, PayloadType payloadType, String path) { String uri; switch (payloadType) { case WORKFLOW_INPUT: case WORKFLOW_OUTPUT: uri = "workflow"; break; case TASK_INPUT: case TASK_OUTPUT: uri = "tasks"; break; default: throw new ConductorClientException( String.format( "Invalid payload type: %s for operation: %s", payloadType.toString(), operation.toString())); } return clientBase.getForEntity( String.format("%s/externalstoragelocation", uri), new Object[] { "path", path, "operation", operation.toString(), "payloadType", payloadType.toString() }, ExternalStorageLocation.class); } /** * Uploads the payload to the uri specified. * * @param uri the location to which the object is to be uploaded * @param payload an {@link InputStream} containing the json payload which is to be uploaded * @param payloadSize the size of the json payload in bytes * @throws ConductorClientException if the upload fails due to an invalid path or an error from * external storage */ @Override public void upload(String uri, InputStream payload, long payloadSize) { HttpURLConnection connection = null; try { URL url = new URI(uri).toURL(); connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(true); connection.setRequestMethod("PUT"); try (BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(connection.getOutputStream())) { long count = IOUtils.copy(payload, bufferedOutputStream); bufferedOutputStream.flush(); // Check the HTTP response code int responseCode = connection.getResponseCode(); if (Response.Status.fromStatusCode(responseCode).getFamily() != Response.Status.Family.SUCCESSFUL) { String errorMsg = String.format("Unable to upload. Response code: %d", responseCode); LOGGER.error(errorMsg); throw new ConductorClientException(errorMsg); } LOGGER.debug( "Uploaded {} bytes to uri: {}, with HTTP response code: {}", count, uri, responseCode); } } catch (URISyntaxException | MalformedURLException e) { String errorMsg = String.format("Invalid path specified: %s", uri); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } catch (IOException e) { String errorMsg = String.format("Error uploading to path: %s", uri); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } finally { if (connection != null) { connection.disconnect(); } try { if (payload != null) { payload.close(); } } catch (IOException e) { LOGGER.warn("Unable to close inputstream when uploading to uri: {}", uri); } } } /** * Downloads the payload from the given uri. * * @param uri the location from where the object is to be downloaded * @return an inputstream of the payload in the external storage * @throws ConductorClientException if the download fails due to an invalid path or an error * from external storage */ @Override public InputStream download(String uri) { HttpURLConnection connection = null; String errorMsg; try { URL url = new URI(uri).toURL(); connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(false); // Check the HTTP response code int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { LOGGER.debug( "Download completed with HTTP response code: {}", connection.getResponseCode()); return org.apache.commons.io.IOUtils.toBufferedInputStream( connection.getInputStream()); } errorMsg = String.format("Unable to download. Response code: %d", responseCode); LOGGER.error(errorMsg); throw new ConductorClientException(errorMsg); } catch (URISyntaxException | MalformedURLException e) { errorMsg = String.format("Invalid uri specified: %s", uri); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } catch (IOException e) { errorMsg = String.format("Error downloading from uri: %s", uri); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } finally { if (connection != null) { connection.disconnect(); } } } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/TaskClient.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.config.ConductorClientConfiguration; import com.netflix.conductor.client.config.DefaultConductorClientConfiguration; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.telemetry.MetricsContainer; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; /** Client for conductor task management including polling for task, updating task status etc. */ public class TaskClient extends ClientBase { private static final GenericType> taskList = new GenericType>() {}; private static final GenericType> taskExecLogList = new GenericType>() {}; private static final GenericType> pollDataList = new GenericType>() {}; private static final GenericType> searchResultTaskSummary = new GenericType>() {}; private static final GenericType> searchResultTask = new GenericType>() {}; private static final GenericType> queueSizeMap = new GenericType>() {}; private static final Logger LOGGER = LoggerFactory.getLogger(TaskClient.class); /** Creates a default task client */ public TaskClient() { this(new DefaultClientConfig(), new DefaultConductorClientConfiguration(), null); } /** * @param config REST Client configuration */ public TaskClient(ClientConfig config) { this(config, new DefaultConductorClientConfiguration(), null); } /** * @param config REST Client configuration * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) */ public TaskClient(ClientConfig config, ClientHandler handler) { this(config, new DefaultConductorClientConfiguration(), handler); } /** * @param config REST Client configuration * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public TaskClient(ClientConfig config, ClientHandler handler, ClientFilter... filters) { this(config, new DefaultConductorClientConfiguration(), handler, filters); } /** * @param config REST Client configuration * @param clientConfiguration Specific properties configured for the client, see {@link * ConductorClientConfiguration} * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public TaskClient( ClientConfig config, ConductorClientConfiguration clientConfiguration, ClientHandler handler, ClientFilter... filters) { super(new ClientRequestHandler(config, handler, filters), clientConfiguration); } TaskClient(ClientRequestHandler requestHandler) { super(requestHandler, null); } /** * Perform a poll for a task of a specific task type. * * @param taskType The taskType to poll for * @param domain The domain of the task type * @param workerId Name of the client worker. Used for logging. * @return Task waiting to be executed. */ public Task pollTask(String taskType, String workerId, String domain) { Validate.notBlank(taskType, "Task type cannot be blank"); Validate.notBlank(workerId, "Worker id cannot be blank"); Object[] params = new Object[] {"workerid", workerId, "domain", domain}; Task task = ObjectUtils.defaultIfNull( getForEntity("tasks/poll/{taskType}", params, Task.class, taskType), new Task()); populateTaskPayloads(task); return task; } /** * Perform a batch poll for tasks by task type. Batch size is configurable by count. * * @param taskType Type of task to poll for * @param workerId Name of the client worker. Used for logging. * @param count Maximum number of tasks to be returned. Actual number of tasks returned can be * less than this number. * @param timeoutInMillisecond Long poll wait timeout. * @return List of tasks awaiting to be executed. */ public List batchPollTasksByTaskType( String taskType, String workerId, int count, int timeoutInMillisecond) { Validate.notBlank(taskType, "Task type cannot be blank"); Validate.notBlank(workerId, "Worker id cannot be blank"); Validate.isTrue(count > 0, "Count must be greater than 0"); Object[] params = new Object[] { "workerid", workerId, "count", count, "timeout", timeoutInMillisecond }; List tasks = getForEntity("tasks/poll/batch/{taskType}", params, taskList, taskType); tasks.forEach(this::populateTaskPayloads); return tasks; } /** * Batch poll for tasks in a domain. Batch size is configurable by count. * * @param taskType Type of task to poll for * @param domain The domain of the task type * @param workerId Name of the client worker. Used for logging. * @param count Maximum number of tasks to be returned. Actual number of tasks returned can be * less than this number. * @param timeoutInMillisecond Long poll wait timeout. * @return List of tasks awaiting to be executed. */ public List batchPollTasksInDomain( String taskType, String domain, String workerId, int count, int timeoutInMillisecond) { Validate.notBlank(taskType, "Task type cannot be blank"); Validate.notBlank(workerId, "Worker id cannot be blank"); Validate.isTrue(count > 0, "Count must be greater than 0"); Object[] params = new Object[] { "workerid", workerId, "count", count, "timeout", timeoutInMillisecond, "domain", domain }; List tasks = getForEntity("tasks/poll/batch/{taskType}", params, taskList, taskType); tasks.forEach(this::populateTaskPayloads); return tasks; } /** * Populates the task input/output from external payload storage if the external storage path is * specified. * * @param task the task for which the input is to be populated. */ private void populateTaskPayloads(Task task) { if (StringUtils.isNotBlank(task.getExternalInputPayloadStoragePath())) { MetricsContainer.incrementExternalPayloadUsedCount( task.getTaskDefName(), ExternalPayloadStorage.Operation.READ.name(), ExternalPayloadStorage.PayloadType.TASK_INPUT.name()); task.setInputData( downloadFromExternalStorage( ExternalPayloadStorage.PayloadType.TASK_INPUT, task.getExternalInputPayloadStoragePath())); task.setExternalInputPayloadStoragePath(null); } if (StringUtils.isNotBlank(task.getExternalOutputPayloadStoragePath())) { MetricsContainer.incrementExternalPayloadUsedCount( task.getTaskDefName(), ExternalPayloadStorage.Operation.READ.name(), PayloadType.TASK_OUTPUT.name()); task.setOutputData( downloadFromExternalStorage( ExternalPayloadStorage.PayloadType.TASK_OUTPUT, task.getExternalOutputPayloadStoragePath())); task.setExternalOutputPayloadStoragePath(null); } } /** * Updates the result of a task execution. If the size of the task output payload is bigger than * {@link ConductorClientConfiguration#getTaskOutputPayloadThresholdKB()}, it is uploaded to * {@link ExternalPayloadStorage}, if enabled, else the task is marked as * FAILED_WITH_TERMINAL_ERROR. * * @param taskResult the {@link TaskResult} of the executed task to be updated. */ public void updateTask(TaskResult taskResult) { Validate.notNull(taskResult, "Task result cannot be null"); postForEntityWithRequestOnly("tasks", taskResult); } public Optional evaluateAndUploadLargePayload( Map taskOutputData, String taskType) { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { objectMapper.writeValue(byteArrayOutputStream, taskOutputData); byte[] taskOutputBytes = byteArrayOutputStream.toByteArray(); long taskResultSize = taskOutputBytes.length; MetricsContainer.recordTaskResultPayloadSize(taskType, taskResultSize); long payloadSizeThreshold = conductorClientConfiguration.getTaskOutputPayloadThresholdKB() * 1024L; if (taskResultSize > payloadSizeThreshold) { if (!conductorClientConfiguration.isExternalPayloadStorageEnabled() || taskResultSize > conductorClientConfiguration.getTaskOutputMaxPayloadThresholdKB() * 1024L) { throw new IllegalArgumentException( String.format( "The TaskResult payload size: %d is greater than the permissible %d bytes", taskResultSize, payloadSizeThreshold)); } MetricsContainer.incrementExternalPayloadUsedCount( taskType, ExternalPayloadStorage.Operation.WRITE.name(), ExternalPayloadStorage.PayloadType.TASK_OUTPUT.name()); return Optional.of( uploadToExternalPayloadStorage( PayloadType.TASK_OUTPUT, taskOutputBytes, taskResultSize)); } return Optional.empty(); } catch (IOException e) { String errorMsg = String.format("Unable to update task: %s with task result", taskType); LOGGER.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } } /** * Ack for the task poll. * * @param taskId Id of the task to be polled * @param workerId user identified worker. * @return true if the task was found with the given ID and acknowledged. False otherwise. If * the server returns false, the client should NOT attempt to ack again. */ public Boolean ack(String taskId, String workerId) { Validate.notBlank(taskId, "Task id cannot be blank"); String response = postForEntity( "tasks/{taskId}/ack", null, new Object[] {"workerid", workerId}, String.class, taskId); return Boolean.valueOf(response); } /** * Log execution messages for a task. * * @param taskId id of the task * @param logMessage the message to be logged */ public void logMessageForTask(String taskId, String logMessage) { Validate.notBlank(taskId, "Task id cannot be blank"); postForEntityWithRequestOnly("tasks/" + taskId + "/log", logMessage); } /** * Fetch execution logs for a task. * * @param taskId id of the task. */ public List getTaskLogs(String taskId) { Validate.notBlank(taskId, "Task id cannot be blank"); return getForEntity("tasks/{taskId}/log", null, taskExecLogList, taskId); } /** * Retrieve information about the task * * @param taskId ID of the task * @return Task details */ public Task getTaskDetails(String taskId) { Validate.notBlank(taskId, "Task id cannot be blank"); return getForEntity("tasks/{taskId}", null, Task.class, taskId); } /** * Removes a task from a taskType queue * * @param taskType the taskType to identify the queue * @param taskId the id of the task to be removed */ public void removeTaskFromQueue(String taskType, String taskId) { Validate.notBlank(taskType, "Task type cannot be blank"); Validate.notBlank(taskId, "Task id cannot be blank"); delete("tasks/queue/{taskType}/{taskId}", taskType, taskId); } public int getQueueSizeForTask(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); Integer queueSize = getForEntity( "tasks/queue/size", new Object[] {"taskType", taskType}, new GenericType() {}); return queueSize != null ? queueSize : 0; } public int getQueueSizeForTask( String taskType, String domain, String isolationGroupId, String executionNamespace) { Validate.notBlank(taskType, "Task type cannot be blank"); List params = new LinkedList<>(); params.add("taskType"); params.add(taskType); if (StringUtils.isNotBlank(domain)) { params.add("domain"); params.add(domain); } if (StringUtils.isNotBlank(isolationGroupId)) { params.add("isolationGroupId"); params.add(isolationGroupId); } if (StringUtils.isNotBlank(executionNamespace)) { params.add("executionNamespace"); params.add(executionNamespace); } Integer queueSize = getForEntity( "tasks/queue/size", params.toArray(new Object[0]), new GenericType() {}); return queueSize != null ? queueSize : 0; } /** * Get last poll data for a given task type * * @param taskType the task type for which poll data is to be fetched * @return returns the list of poll data for the task type */ public List getPollData(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); Object[] params = new Object[] {"taskType", taskType}; return getForEntity("tasks/queue/polldata", params, pollDataList); } /** * Get the last poll data for all task types * * @return returns a list of poll data for all task types */ public List getAllPollData() { return getForEntity("tasks/queue/polldata/all", null, pollDataList); } /** * Requeue pending tasks for all running workflows * * @return returns the number of tasks that have been requeued */ public String requeueAllPendingTasks() { return postForEntity("tasks/queue/requeue", null, null, String.class); } /** * Requeue pending tasks of a specific task type * * @return returns the number of tasks that have been requeued */ public String requeuePendingTasksByTaskType(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); return postForEntity("tasks/queue/requeue/{taskType}", null, null, String.class, taskType); } /** * Search for tasks based on payload * * @param query the search string * @return returns the {@link SearchResult} containing the {@link TaskSummary} matching the * query */ public SearchResult search(String query) { return getForEntity("tasks/search", new Object[] {"query", query}, searchResultTaskSummary); } /** * Search for tasks based on payload * * @param query the search string * @return returns the {@link SearchResult} containing the {@link Task} matching the query */ public SearchResult searchV2(String query) { return getForEntity("tasks/search-v2", new Object[] {"query", query}, searchResultTask); } /** * Paginated search for tasks based on payload * * @param start start value of page * @param size number of tasks to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link TaskSummary} that match the query */ public SearchResult search( Integer start, Integer size, String sort, String freeText, String query) { Object[] params = new Object[] { "start", start, "size", size, "sort", sort, "freeText", freeText, "query", query }; return getForEntity("tasks/search", params, searchResultTaskSummary); } /** * Paginated search for tasks based on payload * * @param start start value of page * @param size number of tasks to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link Task} that match the query */ public SearchResult searchV2( Integer start, Integer size, String sort, String freeText, String query) { Object[] params = new Object[] { "start", start, "size", size, "sort", sort, "freeText", freeText, "query", query }; return getForEntity("tasks/search-v2", params, searchResultTask); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/http/WorkflowClient.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.config.ConductorClientConfiguration; import com.netflix.conductor.client.config.DefaultConductorClientConfiguration; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.telemetry.MetricsContainer; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.common.run.WorkflowTestRequest; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; public class WorkflowClient extends ClientBase { private static final GenericType> searchResultWorkflowSummary = new GenericType>() {}; private static final GenericType> searchResultWorkflow = new GenericType>() {}; private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowClient.class); /** Creates a default workflow client */ public WorkflowClient() { this(new DefaultClientConfig(), new DefaultConductorClientConfiguration(), null); } /** * @param config REST Client configuration */ public WorkflowClient(ClientConfig config) { this(config, new DefaultConductorClientConfiguration(), null); } /** * @param config REST Client configuration * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) */ public WorkflowClient(ClientConfig config, ClientHandler handler) { this(config, new DefaultConductorClientConfiguration(), handler); } /** * @param config REST Client configuration * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public WorkflowClient(ClientConfig config, ClientHandler handler, ClientFilter... filters) { this(config, new DefaultConductorClientConfiguration(), handler, filters); } /** * @param config REST Client configuration * @param clientConfiguration Specific properties configured for the client, see {@link * ConductorClientConfiguration} * @param handler Jersey client handler. Useful when plugging in various http client interaction * modules (e.g. ribbon) * @param filters Chain of client side filters to be applied per request */ public WorkflowClient( ClientConfig config, ConductorClientConfiguration clientConfiguration, ClientHandler handler, ClientFilter... filters) { super(new ClientRequestHandler(config, handler, filters), clientConfiguration); } WorkflowClient(ClientRequestHandler requestHandler) { super(requestHandler, null); } /** * Starts a workflow. If the size of the workflow input payload is bigger than {@link * ConductorClientConfiguration#getWorkflowInputPayloadThresholdKB()}, it is uploaded to {@link * ExternalPayloadStorage}, if enabled, else the workflow is rejected. * * @param startWorkflowRequest the {@link StartWorkflowRequest} object to start the workflow. * @return the id of the workflow instance that can be used for tracking. * @throws ConductorClientException if {@link ExternalPayloadStorage} is disabled or if the * payload size is greater than {@link * ConductorClientConfiguration#getWorkflowInputMaxPayloadThresholdKB()}. * @throws NullPointerException if {@link StartWorkflowRequest} is null or {@link * StartWorkflowRequest#getName()} is null. * @throws IllegalArgumentException if {@link StartWorkflowRequest#getName()} is empty. */ public String startWorkflow(StartWorkflowRequest startWorkflowRequest) { Validate.notNull(startWorkflowRequest, "StartWorkflowRequest cannot be null"); Validate.notBlank(startWorkflowRequest.getName(), "Workflow name cannot be null or empty"); Validate.isTrue( StringUtils.isBlank(startWorkflowRequest.getExternalInputPayloadStoragePath()), "External Storage Path must not be set"); String version = startWorkflowRequest.getVersion() != null ? startWorkflowRequest.getVersion().toString() : "latest"; try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { objectMapper.writeValue(byteArrayOutputStream, startWorkflowRequest.getInput()); byte[] workflowInputBytes = byteArrayOutputStream.toByteArray(); long workflowInputSize = workflowInputBytes.length; MetricsContainer.recordWorkflowInputPayloadSize( startWorkflowRequest.getName(), version, workflowInputSize); if (workflowInputSize > conductorClientConfiguration.getWorkflowInputPayloadThresholdKB() * 1024L) { if (!conductorClientConfiguration.isExternalPayloadStorageEnabled() || (workflowInputSize > conductorClientConfiguration .getWorkflowInputMaxPayloadThresholdKB() * 1024L)) { String errorMsg = String.format( "Input payload larger than the allowed threshold of: %d KB", conductorClientConfiguration .getWorkflowInputPayloadThresholdKB()); throw new ConductorClientException(errorMsg); } else { MetricsContainer.incrementExternalPayloadUsedCount( startWorkflowRequest.getName(), ExternalPayloadStorage.Operation.WRITE.name(), ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT.name()); String externalStoragePath = uploadToExternalPayloadStorage( ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT, workflowInputBytes, workflowInputSize); startWorkflowRequest.setExternalInputPayloadStoragePath(externalStoragePath); startWorkflowRequest.setInput(null); } } } catch (IOException e) { String errorMsg = String.format( "Unable to start workflow:%s, version:%s", startWorkflowRequest.getName(), version); LOGGER.error(errorMsg, e); MetricsContainer.incrementWorkflowStartErrorCount(startWorkflowRequest.getName(), e); throw new ConductorClientException(errorMsg, e); } try { return postForEntity( "workflow", startWorkflowRequest, null, String.class, startWorkflowRequest.getName()); } catch (ConductorClientException e) { String errorMsg = String.format( "Unable to send start workflow request:%s, version:%s", startWorkflowRequest.getName(), version); LOGGER.error(errorMsg, e); MetricsContainer.incrementWorkflowStartErrorCount(startWorkflowRequest.getName(), e); throw e; } } /** * Retrieve a workflow by workflow id * * @param workflowId the id of the workflow * @param includeTasks specify if the tasks in the workflow need to be returned * @return the requested workflow */ public Workflow getWorkflow(String workflowId, boolean includeTasks) { Validate.notBlank(workflowId, "workflow id cannot be blank"); Workflow workflow = getForEntity( "workflow/{workflowId}", new Object[] {"includeTasks", includeTasks}, Workflow.class, workflowId); populateWorkflowOutput(workflow); return workflow; } /** * Retrieve all workflows for a given correlation id and name * * @param name the name of the workflow * @param correlationId the correlation id * @param includeClosed specify if all workflows are to be returned or only running workflows * @param includeTasks specify if the tasks in the workflow need to be returned * @return list of workflows for the given correlation id and name */ public List getWorkflows( String name, String correlationId, boolean includeClosed, boolean includeTasks) { Validate.notBlank(name, "name cannot be blank"); Validate.notBlank(correlationId, "correlationId cannot be blank"); Object[] params = new Object[] {"includeClosed", includeClosed, "includeTasks", includeTasks}; List workflows = getForEntity( "workflow/{name}/correlated/{correlationId}", params, new GenericType>() {}, name, correlationId); workflows.forEach(this::populateWorkflowOutput); return workflows; } /** * Populates the workflow output from external payload storage if the external storage path is * specified. * * @param workflow the workflow for which the output is to be populated. */ private void populateWorkflowOutput(Workflow workflow) { if (StringUtils.isNotBlank(workflow.getExternalOutputPayloadStoragePath())) { MetricsContainer.incrementExternalPayloadUsedCount( workflow.getWorkflowName(), ExternalPayloadStorage.Operation.READ.name(), ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT.name()); workflow.setOutput( downloadFromExternalStorage( ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT, workflow.getExternalOutputPayloadStoragePath())); } } /** * Removes a workflow from the system * * @param workflowId the id of the workflow to be deleted * @param archiveWorkflow flag to indicate if the workflow and associated tasks should be * archived before deletion */ public void deleteWorkflow(String workflowId, boolean archiveWorkflow) { Validate.notBlank(workflowId, "Workflow id cannot be blank"); Object[] params = new Object[] {"archiveWorkflow", archiveWorkflow}; deleteWithUriVariables(params, "workflow/{workflowId}/remove", workflowId); } /** * Terminates the execution of all given workflows instances * * @param workflowIds the ids of the workflows to be terminated * @param reason the reason to be logged and displayed * @return the {@link BulkResponse} contains bulkErrorResults and bulkSuccessfulResults */ public BulkResponse terminateWorkflows(List workflowIds, String reason) { Validate.isTrue(!workflowIds.isEmpty(), "workflow id cannot be blank"); return postForEntity( "workflow/bulk/terminate", workflowIds, new Object[] {"reason", reason}, BulkResponse.class); } /** * Retrieve all running workflow instances for a given name and version * * @param workflowName the name of the workflow * @param version the version of the wokflow definition. Defaults to 1. * @return the list of running workflow instances */ public List getRunningWorkflow(String workflowName, Integer version) { Validate.notBlank(workflowName, "Workflow name cannot be blank"); return getForEntity( "workflow/running/{name}", new Object[] {"version", version}, new GenericType>() {}, workflowName); } /** * Retrieve all workflow instances for a given workflow name between a specific time period * * @param workflowName the name of the workflow * @param version the version of the workflow definition. Defaults to 1. * @param startTime the start time of the period * @param endTime the end time of the period * @return returns a list of workflows created during the specified during the time period */ public List getWorkflowsByTimePeriod( String workflowName, int version, Long startTime, Long endTime) { Validate.notBlank(workflowName, "Workflow name cannot be blank"); Validate.notNull(startTime, "Start time cannot be null"); Validate.notNull(endTime, "End time cannot be null"); Object[] params = new Object[] {"version", version, "startTime", startTime, "endTime", endTime}; return getForEntity( "workflow/running/{name}", params, new GenericType>() {}, workflowName); } /** * Starts the decision task for the given workflow instance * * @param workflowId the id of the workflow instance */ public void runDecider(String workflowId) { Validate.notBlank(workflowId, "workflow id cannot be blank"); put("workflow/decide/{workflowId}", null, null, workflowId); } /** * Pause a workflow by workflow id * * @param workflowId the workflow id of the workflow to be paused */ public void pauseWorkflow(String workflowId) { Validate.notBlank(workflowId, "workflow id cannot be blank"); put("workflow/{workflowId}/pause", null, null, workflowId); } /** * Resume a paused workflow by workflow id * * @param workflowId the workflow id of the paused workflow */ public void resumeWorkflow(String workflowId) { Validate.notBlank(workflowId, "workflow id cannot be blank"); put("workflow/{workflowId}/resume", null, null, workflowId); } /** * Skips a given task from a current RUNNING workflow * * @param workflowId the id of the workflow instance * @param taskReferenceName the reference name of the task to be skipped */ public void skipTaskFromWorkflow(String workflowId, String taskReferenceName) { Validate.notBlank(workflowId, "workflow id cannot be blank"); Validate.notBlank(taskReferenceName, "Task reference name cannot be blank"); put( "workflow/{workflowId}/skiptask/{taskReferenceName}", null, null, workflowId, taskReferenceName); } /** * Reruns the workflow from a specific task * * @param workflowId the id of the workflow * @param rerunWorkflowRequest the request containing the task to rerun from * @return the id of the workflow */ public String rerunWorkflow(String workflowId, RerunWorkflowRequest rerunWorkflowRequest) { Validate.notBlank(workflowId, "workflow id cannot be blank"); Validate.notNull(rerunWorkflowRequest, "RerunWorkflowRequest cannot be null"); return postForEntity( "workflow/{workflowId}/rerun", rerunWorkflowRequest, null, String.class, workflowId); } /** * Restart a completed workflow * * @param workflowId the workflow id of the workflow to be restarted * @param useLatestDefinitions if true, use the latest workflow and task definitions when * restarting the workflow if false, use the workflow and task definitions embedded in the * workflow execution when restarting the workflow */ public void restart(String workflowId, boolean useLatestDefinitions) { Validate.notBlank(workflowId, "workflow id cannot be blank"); Object[] params = new Object[] {"useLatestDefinitions", useLatestDefinitions}; postForEntity("workflow/{workflowId}/restart", null, params, Void.TYPE, workflowId); } /** * Retries the last failed task in a workflow * * @param workflowId the workflow id of the workflow with the failed task */ public void retryLastFailedTask(String workflowId) { Validate.notBlank(workflowId, "workflow id cannot be blank"); postForEntityWithUriVariablesOnly("workflow/{workflowId}/retry", workflowId); } /** * Resets the callback times of all IN PROGRESS tasks to 0 for the given workflow * * @param workflowId the id of the workflow */ public void resetCallbacksForInProgressTasks(String workflowId) { Validate.notBlank(workflowId, "workflow id cannot be blank"); postForEntityWithUriVariablesOnly("workflow/{workflowId}/resetcallbacks", workflowId); } /** * Terminates the execution of the given workflow instance * * @param workflowId the id of the workflow to be terminated * @param reason the reason to be logged and displayed */ public void terminateWorkflow(String workflowId, String reason) { Validate.notBlank(workflowId, "workflow id cannot be blank"); deleteWithUriVariables( new Object[] {"reason", reason}, "workflow/{workflowId}", workflowId); } /** * Search for workflows based on payload * * @param query the search query * @return the {@link SearchResult} containing the {@link WorkflowSummary} that match the query */ public SearchResult search(String query) { return getForEntity( "workflow/search", new Object[] {"query", query}, searchResultWorkflowSummary); } /** * Search for workflows based on payload * * @param query the search query * @return the {@link SearchResult} containing the {@link Workflow} that match the query */ public SearchResult searchV2(String query) { return getForEntity( "workflow/search-v2", new Object[] {"query", query}, searchResultWorkflow); } /** * Paginated search for workflows based on payload * * @param start start value of page * @param size number of workflows to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link WorkflowSummary} that match the query */ public SearchResult search( Integer start, Integer size, String sort, String freeText, String query) { Object[] params = new Object[] { "start", start, "size", size, "sort", sort, "freeText", freeText, "query", query }; return getForEntity("workflow/search", params, searchResultWorkflowSummary); } /** * Paginated search for workflows based on payload * * @param start start value of page * @param size number of workflows to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link Workflow} that match the query */ public SearchResult searchV2( Integer start, Integer size, String sort, String freeText, String query) { Object[] params = new Object[] { "start", start, "size", size, "sort", sort, "freeText", freeText, "query", query }; return getForEntity("workflow/search-v2", params, searchResultWorkflow); } public Workflow testWorkflow(WorkflowTestRequest testRequest) { Validate.notNull(testRequest, "testRequest cannot be null"); if (testRequest.getWorkflowDef() != null) { testRequest.setName(testRequest.getWorkflowDef().getName()); testRequest.setVersion(testRequest.getWorkflowDef().getVersion()); } return postForEntity("workflow/test", testRequest, null, Workflow.class); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/telemetry/MetricsContainer.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.telemetry; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import com.netflix.spectator.api.BasicTag; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.Id; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Spectator; import com.netflix.spectator.api.Tag; import com.netflix.spectator.api.Timer; import com.netflix.spectator.api.patterns.PolledMeter; public class MetricsContainer { private static final String TASK_TYPE = "taskType"; private static final String WORKFLOW_TYPE = "workflowType"; private static final String WORKFLOW_VERSION = "version"; private static final String EXCEPTION = "exception"; private static final String ENTITY_NAME = "entityName"; private static final String OPERATION = "operation"; private static final String PAYLOAD_TYPE = "payload_type"; private static final String TASK_EXECUTION_QUEUE_FULL = "task_execution_queue_full"; private static final String TASK_POLL_ERROR = "task_poll_error"; private static final String TASK_PAUSED = "task_paused"; private static final String TASK_EXECUTE_ERROR = "task_execute_error"; private static final String TASK_ACK_FAILED = "task_ack_failed"; private static final String TASK_ACK_ERROR = "task_ack_error"; private static final String TASK_UPDATE_ERROR = "task_update_error"; private static final String TASK_LEASE_EXTEND_ERROR = "task_lease_extend_error"; private static final String TASK_LEASE_EXTEND_COUNTER = "task_lease_extend_counter"; private static final String TASK_POLL_COUNTER = "task_poll_counter"; private static final String TASK_EXECUTE_TIME = "task_execute_time"; private static final String TASK_POLL_TIME = "task_poll_time"; private static final String TASK_RESULT_SIZE = "task_result_size"; private static final String WORKFLOW_INPUT_SIZE = "workflow_input_size"; private static final String EXTERNAL_PAYLOAD_USED = "external_payload_used"; private static final String WORKFLOW_START_ERROR = "workflow_start_error"; private static final String THREAD_UNCAUGHT_EXCEPTION = "thread_uncaught_exceptions"; private static final Registry REGISTRY = Spectator.globalRegistry(); private static final Map TIMERS = new ConcurrentHashMap<>(); private static final Map COUNTERS = new ConcurrentHashMap<>(); private static final Map GAUGES = new ConcurrentHashMap<>(); private static final String CLASS_NAME = MetricsContainer.class.getSimpleName(); private MetricsContainer() {} public static Timer getPollTimer(String taskType) { return getTimer(TASK_POLL_TIME, TASK_TYPE, taskType); } public static Timer getExecutionTimer(String taskType) { return getTimer(TASK_EXECUTE_TIME, TASK_TYPE, taskType); } private static Timer getTimer(String name, String... additionalTags) { String key = CLASS_NAME + "." + name + "." + String.join(",", additionalTags); return TIMERS.computeIfAbsent( key, k -> { List tagList = getTags(additionalTags); tagList.add(new BasicTag("unit", TimeUnit.MILLISECONDS.name())); return REGISTRY.timer(name, tagList); }); } @SuppressWarnings({"rawtypes", "unchecked"}) private static List getTags(String[] additionalTags) { List tagList = new ArrayList(); tagList.add(new BasicTag("class", CLASS_NAME)); for (int j = 0; j < additionalTags.length - 1; j++) { tagList.add(new BasicTag(additionalTags[j], additionalTags[j + 1])); j++; } return tagList; } private static void incrementCount(String name, String... additionalTags) { getCounter(name, additionalTags).increment(); } private static Counter getCounter(String name, String... additionalTags) { String key = CLASS_NAME + "." + name + "." + String.join(",", additionalTags); return COUNTERS.computeIfAbsent( key, k -> { List tags = getTags(additionalTags); return REGISTRY.counter(name, tags); }); } private static AtomicLong getGauge(String name, String... additionalTags) { String key = CLASS_NAME + "." + name + "." + String.join(",", additionalTags); return GAUGES.computeIfAbsent( key, pollTimer -> { Id id = REGISTRY.createId(name, getTags(additionalTags)); return PolledMeter.using(REGISTRY).withId(id).monitorValue(new AtomicLong(0)); }); } public static void incrementTaskExecutionQueueFullCount(String taskType) { incrementCount(TASK_EXECUTION_QUEUE_FULL, TASK_TYPE, taskType); } public static void incrementUncaughtExceptionCount() { incrementCount(THREAD_UNCAUGHT_EXCEPTION); } public static void incrementTaskPollErrorCount(String taskType, Exception e) { incrementCount( TASK_POLL_ERROR, TASK_TYPE, taskType, EXCEPTION, e.getClass().getSimpleName()); } public static void incrementTaskPausedCount(String taskType) { incrementCount(TASK_PAUSED, TASK_TYPE, taskType); } public static void incrementTaskExecutionErrorCount(String taskType, Throwable e) { incrementCount( TASK_EXECUTE_ERROR, TASK_TYPE, taskType, EXCEPTION, e.getClass().getSimpleName()); } public static void incrementTaskAckFailedCount(String taskType) { incrementCount(TASK_ACK_FAILED, TASK_TYPE, taskType); } public static void incrementTaskAckErrorCount(String taskType, Exception e) { incrementCount( TASK_ACK_ERROR, TASK_TYPE, taskType, EXCEPTION, e.getClass().getSimpleName()); } public static void recordTaskResultPayloadSize(String taskType, long payloadSize) { getGauge(TASK_RESULT_SIZE, TASK_TYPE, taskType).getAndSet(payloadSize); } public static void incrementTaskUpdateErrorCount(String taskType, Throwable t) { incrementCount( TASK_UPDATE_ERROR, TASK_TYPE, taskType, EXCEPTION, t.getClass().getSimpleName()); } public static void incrementTaskLeaseExtendErrorCount(String taskType, Throwable t) { incrementCount( TASK_LEASE_EXTEND_ERROR, TASK_TYPE, taskType, EXCEPTION, t.getClass().getSimpleName()); } public static void incrementTaskLeaseExtendCount(String taskType, int taskCount) { getCounter(TASK_LEASE_EXTEND_COUNTER, TASK_TYPE, taskType).increment(taskCount); } public static void incrementTaskPollCount(String taskType, int taskCount) { getCounter(TASK_POLL_COUNTER, TASK_TYPE, taskType).increment(taskCount); } public static void recordWorkflowInputPayloadSize( String workflowType, String version, long payloadSize) { getGauge(WORKFLOW_INPUT_SIZE, WORKFLOW_TYPE, workflowType, WORKFLOW_VERSION, version) .getAndSet(payloadSize); } public static void incrementExternalPayloadUsedCount( String name, String operation, String payloadType) { incrementCount( EXTERNAL_PAYLOAD_USED, ENTITY_NAME, name, OPERATION, operation, PAYLOAD_TYPE, payloadType); } public static void incrementWorkflowStartErrorCount(String workflowType, Throwable t) { incrementCount( WORKFLOW_START_ERROR, WORKFLOW_TYPE, workflowType, EXCEPTION, t.getClass().getSimpleName()); } } ================================================ FILE: client/src/main/java/com/netflix/conductor/client/worker/Worker.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.worker; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.config.PropertyFactory; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.amazonaws.util.EC2MetadataUtils; public interface Worker { /** * Retrieve the name of the task definition the worker is currently working on. * * @return the name of the task definition. */ String getTaskDefName(); /** * Executes a task and returns the updated task. * * @param task Task to be executed. * @return the {@link TaskResult} object If the task is not completed yet, return with the * status as IN_PROGRESS. */ TaskResult execute(Task task); /** * Called when the task coordinator fails to update the task to the server. Client should store * the task id (in a database) and retry the update later * * @param task Task which cannot be updated back to the server. */ default void onErrorUpdate(Task task) {} /** * Override this method to pause the worker from polling. * * @return true if the worker is paused and no more tasks should be polled from server. */ default boolean paused() { return PropertyFactory.getBoolean(getTaskDefName(), "paused", false); } /** * Override this method to app specific rules. * * @return returns the serverId as the id of the instance that the worker is running. */ default String getIdentity() { String serverId; try { serverId = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { serverId = System.getenv("HOSTNAME"); } if (serverId == null) { serverId = (EC2MetadataUtils.getInstanceId() == null) ? System.getProperty("user.name") : EC2MetadataUtils.getInstanceId(); } LoggerHolder.logger.debug("Setting worker id to {}", serverId); return serverId; } /** * Override this method to change the interval between polls. * * @return interval in millisecond at which the server should be polled for worker tasks. */ default int getPollingInterval() { return PropertyFactory.getInteger(getTaskDefName(), "pollInterval", 1000); } default boolean leaseExtendEnabled() { return PropertyFactory.getBoolean(getTaskDefName(), "leaseExtendEnabled", false); } default int getBatchPollTimeoutInMS() { return PropertyFactory.getInteger(getTaskDefName(), "batchPollTimeoutInMS", 1000); } static Worker create(String taskType, Function executor) { return new Worker() { @Override public String getTaskDefName() { return taskType; } @Override public TaskResult execute(Task task) { return executor.apply(task); } @Override public boolean paused() { return Worker.super.paused(); } }; } } final class LoggerHolder { static final Logger logger = LoggerFactory.getLogger(Worker.class); } ================================================ FILE: client/src/test/groovy/com/netflix/conductor/client/http/ClientSpecification.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http import com.netflix.conductor.common.config.ObjectMapperProvider import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification abstract class ClientSpecification extends Specification { protected static final String ROOT_URL = "dummyroot/" protected static URI createURI(String path) { URI.create(ROOT_URL + path) } protected ClientRequestHandler requestHandler protected ObjectMapper objectMapper def setup() { requestHandler = Mock(ClientRequestHandler.class) objectMapper = new ObjectMapperProvider().getObjectMapper() } } ================================================ FILE: client/src/test/groovy/com/netflix/conductor/client/http/EventClientSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http import com.netflix.conductor.common.metadata.events.EventHandler import com.sun.jersey.api.client.ClientResponse import com.sun.jersey.api.client.WebResource import spock.lang.Subject import spock.lang.Unroll class EventClientSpec extends ClientSpecification { @Subject EventClient eventClient def setup() { eventClient = new EventClient(requestHandler) eventClient.setRootURI(ROOT_URL) } def "register event handler"() { given: EventHandler handler = new EventHandler() URI uri = createURI("event") when: eventClient.registerEventHandler(handler) then: 1 * requestHandler.getWebResourceBuilder(uri, handler) >> Mock(WebResource.Builder.class) } def "update event handler"() { given: EventHandler handler = new EventHandler() URI uri = createURI("event") when: eventClient.updateEventHandler(handler) then: 1 * requestHandler.getWebResourceBuilder(uri, handler) >> Mock(WebResource.Builder.class) } def "unregister event handler"() { given: String eventName = "test" URI uri = createURI("event/$eventName") when: eventClient.unregisterEventHandler(eventName) then: 1 * requestHandler.delete(uri, null) } @Unroll def "get event handlers activeOnly=#activeOnly"() { given: def handlers = [new EventHandler(), new EventHandler()] String eventName = "test" URI uri = createURI("event/$eventName?activeOnly=$activeOnly") when: def eventHandlers = eventClient.getEventHandlers(eventName, activeOnly) then: eventHandlers && eventHandlers.size() == 2 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> handlers } where: activeOnly << [true, false] } } ================================================ FILE: client/src/test/groovy/com/netflix/conductor/client/http/MetadataClientSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http import com.netflix.conductor.client.exception.ConductorClientException import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.sun.jersey.api.client.ClientResponse import spock.lang.Subject class MetadataClientSpec extends ClientSpecification { @Subject MetadataClient metadataClient def setup() { metadataClient = new MetadataClient(requestHandler) metadataClient.setRootURI(ROOT_URL) } def "workflow delete"() { given: String workflowName = 'test' int version = 1 URI uri = createURI("metadata/workflow/$workflowName/$version") when: metadataClient.unregisterWorkflowDef(workflowName, version) then: 1 * requestHandler.delete(uri, null) } def "workflow delete throws exception"() { given: String workflowName = 'test' int version = 1 URI uri = createURI("metadata/workflow/$workflowName/$version") when: metadataClient.unregisterWorkflowDef(workflowName, version) then: 1 * requestHandler.delete(uri, null) >> { throw new RuntimeException(clientResponse) } def ex = thrown(ConductorClientException.class) ex.message == "Unable to invoke Conductor API with uri: $uri, runtime exception occurred" } def "workflow delete version missing"() { when: metadataClient.unregisterWorkflowDef("some name", null) then: thrown(NullPointerException.class) } def "workflow delete name missing"() { when: metadataClient.unregisterWorkflowDef(null, 1) then: thrown(NullPointerException.class) when: metadataClient.unregisterWorkflowDef(" ", 1) then: thrown(IllegalArgumentException.class) } def "workflow get all definitions latest version"() { given: List result = new ArrayList() URI uri = createURI("metadata/workflow/latest-versions") when: metadataClient.getAllWorkflowsWithLatestVersions() then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } } } ================================================ FILE: client/src/test/groovy/com/netflix/conductor/client/http/TaskClientSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.SearchResult import com.netflix.conductor.common.run.TaskSummary import com.sun.jersey.api.client.ClientResponse import spock.lang.Subject class TaskClientSpec extends ClientSpecification { @Subject TaskClient taskClient def setup() { taskClient = new TaskClient(requestHandler) taskClient.setRootURI(ROOT_URL) } def "search"() { given: String query = 'my_complex_query' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new TaskSummary()] URI uri = createURI("tasks/search?query=$query") when: SearchResult searchResult = taskClient.search(query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof TaskSummary } def "searchV2"() { given: String query = 'my_complex_query' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new Task()] URI uri = createURI("tasks/search-v2?query=$query") when: SearchResult searchResult = taskClient.searchV2('my_complex_query') then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof Task } def "search with params"() { given: String query = 'my_complex_query' int start = 0 int size = 10 String sort = 'sort' String freeText = 'text' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new TaskSummary()] URI uri = createURI("tasks/search?start=$start&size=$size&sort=$sort&freeText=$freeText&query=$query") when: SearchResult searchResult = taskClient.search(start, size, sort, freeText, query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof TaskSummary } def "searchV2 with params"() { given: String query = 'my_complex_query' int start = 0 int size = 10 String sort = 'sort' String freeText = 'text' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new Task()] URI uri = createURI("tasks/search-v2?start=$start&size=$size&sort=$sort&freeText=$freeText&query=$query") when: SearchResult searchResult = taskClient.searchV2(start, size, sort, freeText, query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof Task } } ================================================ FILE: client/src/test/groovy/com/netflix/conductor/client/http/WorkflowClientSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.http import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.run.SearchResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.common.run.WorkflowSummary import com.sun.jersey.api.client.ClientResponse import spock.lang.Subject class WorkflowClientSpec extends ClientSpecification { @Subject WorkflowClient workflowClient def setup() { workflowClient = new WorkflowClient(requestHandler) workflowClient.setRootURI(ROOT_URL) } def "search"() { given: String query = 'my_complex_query' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new WorkflowSummary()] URI uri = createURI("workflow/search?query=$query") when: SearchResult searchResult = workflowClient.search(query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof WorkflowSummary } def "searchV2"() { given: String query = 'my_complex_query' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new Workflow(workflowDefinition: new WorkflowDef(), createTime: System.currentTimeMillis() )] URI uri = createURI("workflow/search-v2?query=$query") when: SearchResult searchResult = workflowClient.searchV2('my_complex_query') then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof Workflow } def "search with params"() { given: String query = 'my_complex_query' int start = 0 int size = 10 String sort = 'sort' String freeText = 'text' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new WorkflowSummary()] URI uri = createURI("workflow/search?start=$start&size=$size&sort=$sort&freeText=$freeText&query=$query") when: SearchResult searchResult = workflowClient.search(start, size, sort, freeText, query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof WorkflowSummary } def "searchV2 with params"() { given: String query = 'my_complex_query' int start = 0 int size = 10 String sort = 'sort' String freeText = 'text' SearchResult result = new SearchResult<>() result.totalHits = 1 result.results = [new Workflow(workflowDefinition: new WorkflowDef(), createTime: System.currentTimeMillis() )] URI uri = createURI("workflow/search-v2?start=$start&size=$size&sort=$sort&freeText=$freeText&query=$query") when: SearchResult searchResult = workflowClient.searchV2(start, size, sort, freeText, query) then: 1 * requestHandler.get(uri) >> Mock(ClientResponse.class) { getEntity(_) >> result } searchResult.totalHits == result.totalHits searchResult.results && searchResult.results.size() == 1 searchResult.results[0] instanceof Workflow } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/automator/PollingSemaphoreTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.IntStream; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class PollingSemaphoreTest { @Test public void testBlockAfterAvailablePermitsExhausted() throws Exception { int threads = 5; ExecutorService executorService = Executors.newFixedThreadPool(threads); PollingSemaphore pollingSemaphore = new PollingSemaphore(threads); List> futuresList = new ArrayList<>(); IntStream.range(0, threads) .forEach( t -> futuresList.add( CompletableFuture.runAsync( () -> pollingSemaphore.acquireSlots(1), executorService))); CompletableFuture allFutures = CompletableFuture.allOf( futuresList.toArray(new CompletableFuture[futuresList.size()])); allFutures.get(); assertEquals(0, pollingSemaphore.availableSlots()); assertFalse(pollingSemaphore.acquireSlots(1)); executorService.shutdown(); } @Test public void testAllowsPollingWhenPermitBecomesAvailable() throws Exception { int threads = 5; ExecutorService executorService = Executors.newFixedThreadPool(threads); PollingSemaphore pollingSemaphore = new PollingSemaphore(threads); List> futuresList = new ArrayList<>(); IntStream.range(0, threads) .forEach( t -> futuresList.add( CompletableFuture.runAsync( () -> pollingSemaphore.acquireSlots(1), executorService))); CompletableFuture allFutures = CompletableFuture.allOf( futuresList.toArray(new CompletableFuture[futuresList.size()])); allFutures.get(); assertEquals(0, pollingSemaphore.availableSlots()); pollingSemaphore.complete(1); assertTrue(pollingSemaphore.availableSlots() > 0); assertTrue(pollingSemaphore.acquireSlots(1)); executorService.shutdown(); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/automator/TaskPollExecutorTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.*; import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import com.netflix.appinfo.InstanceInfo; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.discovery.EurekaClient; import static com.netflix.conductor.common.metadata.tasks.TaskResult.Status.COMPLETED; import static com.netflix.conductor.common.metadata.tasks.TaskResult.Status.IN_PROGRESS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; public class TaskPollExecutorTest { private static final String TEST_TASK_DEF_NAME = "test"; private static final Map TASK_THREAD_MAP = Collections.singletonMap(TEST_TASK_DEF_NAME, 1); @Test public void testTaskExecutionException() throws InterruptedException { Worker worker = Worker.create( TEST_TASK_DEF_NAME, task -> { throw new NoSuchMethodError(); }); TaskClient taskClient = Mockito.mock(TaskClient.class); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-%d", TASK_THREAD_MAP); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(testTask())); when(taskClient.ack(any(), any())).thenReturn(true); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { assertEquals("test-worker-1", Thread.currentThread().getName()); Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(TaskResult.Status.FAILED, result.getStatus()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).updateTask(any()); } @SuppressWarnings("rawtypes") @Test public void testMultipleTasksExecution() throws InterruptedException { String outputKey = "KEY"; Task task = testTask(); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.execute(any())) .thenAnswer( new Answer() { private int count = 0; Map outputMap = new HashMap<>(); public TaskResult answer(InvocationOnMock invocation) throws InterruptedException { // Sleep for 2 seconds to simulate task execution Thread.sleep(2000L); TaskResult taskResult = new TaskResult(task); outputMap.put(outputKey, count++); taskResult.setOutputData(outputMap); return taskResult; } }); TaskClient taskClient = Mockito.mock(TaskClient.class); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); when(taskClient.ack(any(), any())).thenReturn(true); CountDownLatch latch = new CountDownLatch(3); doAnswer( new Answer() { private int count = 0; public TaskResult answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(count, result.getOutputData().get(outputKey)); count++; latch.countDown(); return null; } }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); // execute() is called 3 times on the worker (once for each task) verify(worker, times(3)).execute(any()); verify(taskClient, times(3)).updateTask(any()); } @SuppressWarnings("unchecked") @Test public void testLargePayloadCanFailUpdateWithRetry() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); when(taskClient.ack(any(), any())).thenReturn(true); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertNull(result.getReasonForIncompletion()); result.setReasonForIncompletion("some_reason_1"); throw new ConductorClientException(); }) .when(taskClient) .evaluateAndUploadLargePayload(any(Map.class), any()); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(worker) .onErrorUpdate(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); // When evaluateAndUploadLargePayload fails indefinitely, task update shouldn't be called. verify(taskClient, times(0)).updateTask(any()); } @Test public void testLargePayloadLocationUpdate() throws InterruptedException { Task task = testTask(); String largePayloadLocation = "large_payload_location"; Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); when(taskClient.ack(any(), any())).thenReturn(true); //noinspection unchecked when(taskClient.evaluateAndUploadLargePayload(any(Map.class), any())) .thenReturn(Optional.of(largePayloadLocation)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertNull(result.getOutputData()); assertEquals( largePayloadLocation, result.getExternalOutputPayloadStoragePath()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient, times(1)).updateTask(any()); } @Test public void testTaskPollException() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenThrow(ConductorClientException.class) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).updateTask(any()); } @Test public void testTaskPoll() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("test"); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).updateTask(any()); } @Test public void testTaskPollDomain() throws InterruptedException { TaskClient taskClient = Mockito.mock(TaskClient.class); String testDomain = "foo"; Map taskToDomain = new HashMap<>(); taskToDomain.put(TEST_TASK_DEF_NAME, testDomain); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, taskToDomain, "test-worker-", TASK_THREAD_MAP); String workerName = "test-worker"; Worker worker = mock(Worker.class); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.getIdentity()).thenReturn(workerName); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(taskClient) .batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt()); } @Test public void testPollOutOfDiscoveryForTask() throws InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); when(client.getInstanceRemoteStatus()).thenReturn(InstanceInfo.InstanceStatus.UNKNOWN); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("task_run_always"); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(new Task())) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( client, taskClient, 1, new HashMap<>(), "test-worker-", Collections.singletonMap("task_run_always", 1)); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).updateTask(any()); } @Test public void testPollOutOfDiscoveryAsDefaultFalseForTask() throws ExecutionException, InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); when(client.getInstanceRemoteStatus()).thenReturn(InstanceInfo.InstanceStatus.UNKNOWN); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("task_do_not_run_always"); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( client, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); ScheduledFuture f = Executors.newSingleThreadScheduledExecutor() .schedule( () -> taskPollExecutor.pollAndExecute(worker), 0, TimeUnit.SECONDS); f.get(); verify(taskClient, times(0)).updateTask(any()); } @Test public void testPollOutOfDiscoveryAsExplicitFalseForTask() throws ExecutionException, InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); when(client.getInstanceRemoteStatus()).thenReturn(InstanceInfo.InstanceStatus.UNKNOWN); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("task_explicit_do_not_run_always"); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( client, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); ScheduledFuture f = Executors.newSingleThreadScheduledExecutor() .schedule( () -> taskPollExecutor.pollAndExecute(worker), 0, TimeUnit.SECONDS); f.get(); verify(taskClient, times(0)).updateTask(any()); } @Test public void testPollOutOfDiscoveryIsIgnoredWhenDiscoveryIsUp() throws InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); when(client.getInstanceRemoteStatus()).thenReturn(InstanceInfo.InstanceStatus.UP); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("task_ignore_override"); when(worker.execute(any())).thenReturn(new TaskResult(task)); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( client, taskClient, 1, new HashMap<>(), "test-worker-", Collections.singletonMap("task_ignore_override", 1)); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(IN_PROGRESS, result.getStatus()); assertEquals(task.getTaskId(), result.getTaskId()); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).updateTask(any()); } @Test public void testTaskThreadCount() throws InterruptedException { TaskClient taskClient = Mockito.mock(TaskClient.class); Map taskThreadCount = new HashMap<>(); taskThreadCount.put(TEST_TASK_DEF_NAME, 1); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, -1, new HashMap<>(), "test-worker-", taskThreadCount); String workerName = "test-worker"; Worker worker = mock(Worker.class); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.getIdentity()).thenReturn(workerName); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(taskClient) .batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); verify(taskClient).batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt()); } @Test public void testTaskLeaseExtend() throws InterruptedException { Task task = testTask(); task.setResponseTimeoutSeconds(1); Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getTaskDefName()).thenReturn("test"); when(worker.execute(any())).thenReturn(new TaskResult(task)); when(worker.leaseExtendEnabled()).thenReturn(true); TaskClient taskClient = Mockito.mock(TaskClient.class); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenReturn(Arrays.asList(task)); TaskResult result = new TaskResult(task); result.getLogs().add(new TaskExecLog("lease extend")); result.setExtendLease(true); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", TASK_THREAD_MAP); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { assertTrue( taskPollExecutor.leaseExtendMap.containsKey(task.getTaskId())); latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 5, TimeUnit.SECONDS); latch.await(); } @Test public void testBatchTasksExecution() throws InterruptedException { int threadCount = 10; TaskClient taskClient = Mockito.mock(TaskClient.class); Map taskThreadCount = new HashMap<>(); taskThreadCount.put(TEST_TASK_DEF_NAME, threadCount); String workerName = "test-worker"; Worker worker = mock(Worker.class); when(worker.getPollingInterval()).thenReturn(3000); when(worker.getBatchPollTimeoutInMS()).thenReturn(1000); when(worker.getTaskDefName()).thenReturn(TEST_TASK_DEF_NAME); when(worker.getIdentity()).thenReturn(workerName); List tasks = new ArrayList<>(); for (int i = 0; i < threadCount; i++) { Task task = testTask(); tasks.add(task); when(worker.execute(task)) .thenAnswer( new Answer() { Map outputMap = new HashMap<>(); public TaskResult answer(InvocationOnMock invocation) throws InterruptedException { // Sleep for 1 seconds to simulate task execution Thread.sleep(1000L); TaskResult taskResult = new TaskResult(task); outputMap.put("key", "value"); taskResult.setOutputData(outputMap); taskResult.setStatus(COMPLETED); return taskResult; } }); } when(taskClient.batchPollTasksInDomain( TEST_TASK_DEF_NAME, null, workerName, threadCount, 1000)) .thenReturn(tasks); when(taskClient.ack(any(), any())).thenReturn(true); TaskPollExecutor taskPollExecutor = new TaskPollExecutor( null, taskClient, 1, new HashMap<>(), "test-worker-", taskThreadCount); CountDownLatch latch = new CountDownLatch(threadCount); doAnswer( new Answer() { public TaskResult answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(COMPLETED, result.getStatus()); assertEquals("value", result.getOutputData().get("key")); latch.countDown(); return null; } }) .when(taskClient) .updateTask(any()); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); latch.await(); // execute() is called 10 times on the worker (once for each task) verify(worker, times(threadCount)).execute(any()); verify(taskClient, times(threadCount)).updateTask(any()); } private Task testTask() { Task task = new Task(); task.setTaskId(UUID.randomUUID().toString()); task.setStatus(Task.Status.IN_PROGRESS); task.setTaskDefName(TEST_TASK_DEF_NAME); return task; } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerConfigurerTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.automator; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import static com.netflix.conductor.common.metadata.tasks.TaskResult.Status.COMPLETED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TaskRunnerConfigurerTest { private static final String TEST_TASK_DEF_NAME = "test"; private TaskClient client; @Before public void setup() { client = Mockito.mock(TaskClient.class); } @Test(expected = NullPointerException.class) public void testNoWorkersException() { new TaskRunnerConfigurer.Builder(null, null).build(); } @Test(expected = ConductorClientException.class) public void testInvalidThreadConfig() { Worker worker1 = Worker.create("task1", TaskResult::new); Worker worker2 = Worker.create("task2", TaskResult::new); Map taskThreadCount = new HashMap<>(); taskThreadCount.put(worker1.getTaskDefName(), 2); taskThreadCount.put(worker2.getTaskDefName(), 3); new TaskRunnerConfigurer.Builder(client, Arrays.asList(worker1, worker2)) .withThreadCount(10) .withTaskThreadCount(taskThreadCount) .build(); } @Test public void testMissingTaskThreadConfig() { Worker worker1 = Worker.create("task1", TaskResult::new); Worker worker2 = Worker.create("task2", TaskResult::new); Map taskThreadCount = new HashMap<>(); taskThreadCount.put(worker1.getTaskDefName(), 2); TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(client, Arrays.asList(worker1, worker2)) .withTaskThreadCount(taskThreadCount) .build(); assertFalse(configurer.getTaskThreadCount().isEmpty()); assertEquals(2, configurer.getTaskThreadCount().size()); assertEquals(2, configurer.getTaskThreadCount().get("task1").intValue()); assertEquals(1, configurer.getTaskThreadCount().get("task2").intValue()); } @Test public void testPerTaskThreadPool() { Worker worker1 = Worker.create("task1", TaskResult::new); Worker worker2 = Worker.create("task2", TaskResult::new); Map taskThreadCount = new HashMap<>(); taskThreadCount.put(worker1.getTaskDefName(), 2); taskThreadCount.put(worker2.getTaskDefName(), 3); TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(client, Arrays.asList(worker1, worker2)) .withTaskThreadCount(taskThreadCount) .build(); configurer.init(); assertEquals(-1, configurer.getThreadCount()); assertEquals(2, configurer.getTaskThreadCount().get("task1").intValue()); assertEquals(3, configurer.getTaskThreadCount().get("task2").intValue()); } @Test public void testSharedThreadPool() { Worker worker = Worker.create(TEST_TASK_DEF_NAME, TaskResult::new); TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(client, Arrays.asList(worker, worker, worker)) .build(); configurer.init(); assertEquals(3, configurer.getThreadCount()); assertEquals(500, configurer.getSleepWhenRetry()); assertEquals(3, configurer.getUpdateRetryCount()); assertEquals(10, configurer.getShutdownGracePeriodSeconds()); assertFalse(configurer.getTaskThreadCount().isEmpty()); assertEquals(1, configurer.getTaskThreadCount().size()); assertEquals(3, configurer.getTaskThreadCount().get(TEST_TASK_DEF_NAME).intValue()); configurer = new TaskRunnerConfigurer.Builder(client, Collections.singletonList(worker)) .withThreadCount(100) .withSleepWhenRetry(100) .withUpdateRetryCount(10) .withShutdownGracePeriodSeconds(15) .withWorkerNamePrefix("test-worker-") .build(); assertEquals(100, configurer.getThreadCount()); configurer.init(); assertEquals(100, configurer.getThreadCount()); assertEquals(100, configurer.getSleepWhenRetry()); assertEquals(10, configurer.getUpdateRetryCount()); assertEquals(15, configurer.getShutdownGracePeriodSeconds()); assertEquals("test-worker-", configurer.getWorkerNamePrefix()); assertFalse(configurer.getTaskThreadCount().isEmpty()); assertEquals(1, configurer.getTaskThreadCount().size()); assertEquals(100, configurer.getTaskThreadCount().get(TEST_TASK_DEF_NAME).intValue()); } @Test public void testMultipleWorkersExecution() throws Exception { String task1Name = "task1"; Worker worker1 = mock(Worker.class); when(worker1.getPollingInterval()).thenReturn(3000); when(worker1.getTaskDefName()).thenReturn(task1Name); when(worker1.getIdentity()).thenReturn("worker1"); when(worker1.execute(any())) .thenAnswer( invocation -> { // Sleep for 2 seconds to simulate task execution Thread.sleep(2000); TaskResult taskResult = new TaskResult(); taskResult.setStatus(COMPLETED); return taskResult; }); String task2Name = "task2"; Worker worker2 = mock(Worker.class); when(worker2.getPollingInterval()).thenReturn(3000); when(worker2.getTaskDefName()).thenReturn(task2Name); when(worker2.getIdentity()).thenReturn("worker2"); when(worker2.execute(any())) .thenAnswer( invocation -> { // Sleep for 2 seconds to simulate task execution Thread.sleep(2000); TaskResult taskResult = new TaskResult(); taskResult.setStatus(COMPLETED); return taskResult; }); Task task1 = testTask(task1Name); Task task2 = testTask(task2Name); TaskClient taskClient = Mockito.mock(TaskClient.class); TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(taskClient, Arrays.asList(worker1, worker2)) .withThreadCount(2) .withSleepWhenRetry(100000) .withUpdateRetryCount(1) .withWorkerNamePrefix("test-worker-") .build(); when(taskClient.batchPollTasksInDomain(any(), any(), any(), anyInt(), anyInt())) .thenAnswer( invocation -> { Object[] args = invocation.getArguments(); String taskName = args[0].toString(); if (taskName.equals(task1Name)) { return Arrays.asList(task1); } else if (taskName.equals(task2Name)) { return Arrays.asList(task2); } else { return Collections.emptyList(); } }); when(taskClient.ack(any(), any())).thenReturn(true); AtomicInteger task1Counter = new AtomicInteger(0); AtomicInteger task2Counter = new AtomicInteger(0); CountDownLatch latch = new CountDownLatch(2); doAnswer( invocation -> { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertEquals(COMPLETED, result.getStatus()); if (result.getWorkerId().equals("worker1")) { task1Counter.incrementAndGet(); } else if (result.getWorkerId().equals("worker2")) { task2Counter.incrementAndGet(); } latch.countDown(); return null; }) .when(taskClient) .updateTask(any()); configurer.init(); latch.await(); assertEquals(1, task1Counter.get()); assertEquals(1, task2Counter.get()); } private Task testTask(String taskDefName) { Task task = new Task(); task.setTaskId(UUID.randomUUID().toString()); task.setStatus(Task.Status.IN_PROGRESS); task.setTaskDefName(taskDefName); return task; } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/config/TestPropertyFactory.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.config; import org.junit.Test; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.TaskResult; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class TestPropertyFactory { @Test public void testIdentity() { Worker worker = Worker.create("Test2", TaskResult::new); assertNotNull(worker.getIdentity()); boolean paused = worker.paused(); assertFalse("Paused? " + paused, paused); } @Test public void test() { int val = PropertyFactory.getInteger("workerB", "pollingInterval", 100); assertEquals("got: " + val, 2, val); assertEquals( 100, PropertyFactory.getInteger("workerB", "propWithoutValue", 100).intValue()); assertFalse( PropertyFactory.getBoolean( "workerB", "paused", true)); // Global value set to 'false' assertTrue( PropertyFactory.getBoolean( "workerA", "paused", false)); // WorkerA value set to 'true' assertEquals( 42, PropertyFactory.getInteger("workerA", "batchSize", 42) .intValue()); // No global value set, so will return the default value // supplied assertEquals( 84, PropertyFactory.getInteger("workerB", "batchSize", 42) .intValue()); // WorkerB's value set to 84 assertEquals("domainA", PropertyFactory.getString("workerA", "domain", null)); assertEquals("domainB", PropertyFactory.getString("workerB", "domain", null)); assertNull(PropertyFactory.getString("workerC", "domain", null)); // Non Existent } @Test public void testProperty() { Worker worker = Worker.create("Test", TaskResult::new); boolean paused = worker.paused(); assertTrue("Paused? " + paused, paused); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/sample/Main.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.sample; import java.util.Arrays; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; public class Main { public static void main(String[] args) { TaskClient taskClient = new TaskClient(); taskClient.setRootURI("http://localhost:8080/api/"); // Point this to the server API int threadCount = 2; // number of threads used to execute workers. To avoid starvation, should be // same or more than number of workers Worker worker1 = new SampleWorker("task_1"); Worker worker2 = new SampleWorker("task_5"); // Create TaskRunnerConfigurer TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(taskClient, Arrays.asList(worker1, worker2)) .withThreadCount(threadCount) .build(); // Start the polling and execution of tasks configurer.init(); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/sample/SampleWorker.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.sample; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskResult.Status; public class SampleWorker implements Worker { private final String taskDefName; public SampleWorker(String taskDefName) { this.taskDefName = taskDefName; } @Override public String getTaskDefName() { return taskDefName; } @Override public TaskResult execute(Task task) { TaskResult result = new TaskResult(task); result.setStatus(Status.COMPLETED); // Register the output of the task result.getOutputData().put("outputKey1", "value"); result.getOutputData().put("oddEven", 1); result.getOutputData().put("mod", 4); return result; } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/testing/AbstractWorkflowTests.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.testing; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; import com.netflix.conductor.client.http.MetadataClient; import com.netflix.conductor.client.http.WorkflowClient; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowTestRequest; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.jupiter.api.Assertions.assertNotNull; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class AbstractWorkflowTests { protected static ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); protected static TypeReference>> mockType = new TypeReference>>() {}; protected MetadataClient metadataClient; protected WorkflowClient workflowClient; @BeforeAll public void setup() { String baseURL = "http://localhost:8080/api/"; metadataClient = new MetadataClient(); metadataClient.setRootURI(baseURL); workflowClient = new WorkflowClient(); workflowClient.setRootURI(baseURL); } protected WorkflowTestRequest getWorkflowTestRequest(WorkflowDef def) throws IOException { WorkflowTestRequest testRequest = new WorkflowTestRequest(); testRequest.setInput(new HashMap<>()); testRequest.setName(def.getName()); testRequest.setVersion(def.getVersion()); testRequest.setWorkflowDef(def); Map> taskRefToMockOutput = new HashMap<>(); for (WorkflowTask task : def.collectTasks()) { List taskRuns = new LinkedList<>(); WorkflowTestRequest.TaskMock mock = new WorkflowTestRequest.TaskMock(); mock.setStatus(TaskResult.Status.COMPLETED); Map output = new HashMap<>(); output.put("response", Map.of()); mock.setOutput(output); taskRuns.add(mock); taskRefToMockOutput.put(task.getTaskReferenceName(), taskRuns); if (task.getType().equals(TaskType.SUB_WORKFLOW.name())) { Object inlineSubWorkflowDefObj = task.getSubWorkflowParam().getWorkflowDefinition(); if (inlineSubWorkflowDefObj != null) { // If not null, it represents WorkflowDef object WorkflowDef inlineSubWorkflowDef = (WorkflowDef) inlineSubWorkflowDefObj; WorkflowTestRequest subWorkflowTestRequest = getWorkflowTestRequest(inlineSubWorkflowDef); testRequest .getSubWorkflowTestRequest() .put(task.getTaskReferenceName(), subWorkflowTestRequest); } else { // Inline definition is null String subWorkflowName = task.getSubWorkflowParam().getName(); // Load up the sub workflow from the JSON WorkflowDef subWorkflowDef = getWorkflowDef("/workflows/" + subWorkflowName + ".json"); assertNotNull(subWorkflowDef); WorkflowTestRequest subWorkflowTestRequest = getWorkflowTestRequest(subWorkflowDef); testRequest .getSubWorkflowTestRequest() .put(task.getTaskReferenceName(), subWorkflowTestRequest); } } } testRequest.setTaskRefToMockOutput(taskRefToMockOutput); return testRequest; } protected WorkflowDef getWorkflowDef(String path) throws IOException { InputStream inputStream = AbstractWorkflowTests.class.getResourceAsStream(path); if (inputStream == null) { throw new IOException("No file found at " + path); } return objectMapper.readValue(new InputStreamReader(inputStream), WorkflowDef.class); } protected Workflow getWorkflow(String path) throws IOException { InputStream inputStream = AbstractWorkflowTests.class.getResourceAsStream(path); if (inputStream == null) { throw new IOException("No file found at " + path); } return objectMapper.readValue(new InputStreamReader(inputStream), Workflow.class); } protected Map> getTestInputs(String path) throws IOException { InputStream inputStream = AbstractWorkflowTests.class.getResourceAsStream(path); if (inputStream == null) { throw new IOException("No file found at " + path); } return objectMapper.readValue(new InputStreamReader(inputStream), mockType); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/testing/LoanWorkflowInput.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.testing; import java.math.BigDecimal; public class LoanWorkflowInput { private String userEmail; private BigDecimal loanAmount; public String getUserEmail() { return userEmail; } public void setUserEmail(String userEmail) { this.userEmail = userEmail; } public BigDecimal getLoanAmount() { return loanAmount; } public void setLoanAmount(BigDecimal loanAmount) { this.loanAmount = loanAmount; } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/testing/LoanWorkflowTest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.testing; import java.io.IOException; import java.math.BigDecimal; import java.util.List; import java.util.Map; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowTestRequest; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; /** Unit test a workflow with inputs read from a file. */ public class LoanWorkflowTest extends AbstractWorkflowTests { /** Uses mock inputs to verify the workflow execution and input/outputs of the tasks */ // Tests are commented out since it requires a running server // @Test public void verifyWorkflowExecutionWithMockInputs() throws IOException { WorkflowDef def = getWorkflowDef("/workflows/calculate_loan_workflow.json"); assertNotNull(def); Map> testInputs = getTestInputs("/test_data/loan_workflow_input.json"); assertNotNull(testInputs); WorkflowTestRequest testRequest = new WorkflowTestRequest(); testRequest.setWorkflowDef(def); LoanWorkflowInput workflowInput = new LoanWorkflowInput(); workflowInput.setUserEmail("user@example.com"); workflowInput.setLoanAmount(new BigDecimal(11_000)); testRequest.setInput(objectMapper.convertValue(workflowInput, Map.class)); testRequest.setTaskRefToMockOutput(testInputs); testRequest.setName(def.getName()); testRequest.setVersion(def.getVersion()); Workflow execution = workflowClient.testWorkflow(testRequest); assertNotNull(execution); // Assert that the workflow completed successfully assertEquals(Workflow.WorkflowStatus.COMPLETED, execution.getStatus()); // Ensure the inputs were captured correctly assertEquals( workflowInput.getLoanAmount().toString(), String.valueOf(execution.getInput().get("loanAmount"))); assertEquals(workflowInput.getUserEmail(), execution.getInput().get("userEmail")); // A total of 3 tasks were executed assertEquals(3, execution.getTasks().size()); Task fetchUserDetails = execution.getTasks().get(0); Task getCreditScore = execution.getTasks().get(1); Task calculateLoanAmount = execution.getTasks().get(2); // fetch user details received the correct input from the workflow assertEquals( workflowInput.getUserEmail(), fetchUserDetails.getInputData().get("userEmail")); // And that the task produced the right output int userAccountNo = 12345; assertEquals(userAccountNo, fetchUserDetails.getOutputData().get("userAccount")); // get credit score received the right account number from the output of the fetch user // details assertEquals(userAccountNo, getCreditScore.getInputData().get("userAccountNumber")); int expectedCreditRating = 750; // The task produced the right output assertEquals(expectedCreditRating, getCreditScore.getOutputData().get("creditRating")); // Calculate loan amount gets the right loan amount from workflow input assertEquals( workflowInput.getLoanAmount().toString(), String.valueOf(calculateLoanAmount.getInputData().get("loanAmount"))); // Calculate loan amount gets the right credit rating from the previous task assertEquals(expectedCreditRating, calculateLoanAmount.getInputData().get("creditRating")); int authorizedLoanAmount = 10_000; assertEquals( authorizedLoanAmount, calculateLoanAmount.getOutputData().get("authorizedLoanAmount")); // Finally, lets verify the workflow outputs assertEquals(userAccountNo, execution.getOutput().get("accountNumber")); assertEquals(expectedCreditRating, execution.getOutput().get("creditRating")); assertEquals(authorizedLoanAmount, execution.getOutput().get("authorizedLoanAmount")); System.out.println(execution); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/testing/RegressionTest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.testing; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowTestRequest; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; /** * This test demonstrates how to use execution data from the previous executed workflows as golden * input and output and use them to regression test the workflow definition. * *

Regression tests are useful ensuring any changes to the workflow definition does not change * the behavior. */ public class RegressionTest extends AbstractWorkflowTests { // @Test // Tests are commented out since it requires a running server // Uses a previously executed successful run to verify the workflow execution, and it's output. public void verifyWorkflowOutput() throws IOException, ExecutionException, InterruptedException, TimeoutException { // Workflow Definition WorkflowDef def = getWorkflowDef("/workflows/workflow1.json"); // Golden output to verify against Workflow workflow = getWorkflow("/test_data/workflow1_run.json"); WorkflowTestRequest testRequest = new WorkflowTestRequest(); testRequest.setInput(new HashMap<>()); testRequest.setName(def.getName()); testRequest.setVersion(def.getVersion()); testRequest.setWorkflowDef(def); Map> taskRefToMockOutput = new HashMap<>(); for (Task task : workflow.getTasks()) { List taskRuns = new ArrayList<>(); WorkflowTestRequest.TaskMock mock = new WorkflowTestRequest.TaskMock(); mock.setStatus(TaskResult.Status.valueOf(task.getStatus().name())); mock.setOutput(task.getOutputData()); taskRuns.add(mock); taskRefToMockOutput.put(def.getTasks().get(0).getTaskReferenceName(), taskRuns); } testRequest.setTaskRefToMockOutput(taskRefToMockOutput); Workflow execution = workflowClient.testWorkflow(testRequest); assertNotNull(execution); assertEquals(workflow.getTasks().size(), execution.getTasks().size()); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/testing/SubWorkflowTest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.testing; import java.io.IOException; import java.util.List; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowTestRequest; import static org.junit.jupiter.api.Assertions.*; /** Demonstrates how to test workflows that contain sub-workflows */ public class SubWorkflowTest extends AbstractWorkflowTests { // @Test // Tests are commented out since it requires a running server public void verifySubWorkflowExecutions() throws IOException { WorkflowDef def = getWorkflowDef("/workflows/kitchensink.json"); assertNotNull(def); WorkflowDef subWorkflowDef = getWorkflowDef("/workflows/PopulationMinMax.json"); metadataClient.registerWorkflowDef(subWorkflowDef); WorkflowTestRequest testRequest = getWorkflowTestRequest(def); // The following are the dynamic tasks which are not present in the workflow definition but // are created by dynamic fork testRequest .getTaskRefToMockOutput() .put("_x_test_worker_0_0", List.of(new WorkflowTestRequest.TaskMock())); testRequest .getTaskRefToMockOutput() .put("_x_test_worker_0_1", List.of(new WorkflowTestRequest.TaskMock())); testRequest .getTaskRefToMockOutput() .put("_x_test_worker_0_2", List.of(new WorkflowTestRequest.TaskMock())); testRequest .getTaskRefToMockOutput() .put("simple_task_1__1", List.of(new WorkflowTestRequest.TaskMock())); testRequest .getTaskRefToMockOutput() .put("simple_task_5", List.of(new WorkflowTestRequest.TaskMock())); Workflow execution = workflowClient.testWorkflow(testRequest); assertNotNull(execution); // Verfiy that the workflow COMPLETES assertEquals(Workflow.WorkflowStatus.COMPLETED, execution.getStatus()); // That the workflow executes a wait task assertTrue( execution.getTasks().stream() .anyMatch(t -> t.getReferenceTaskName().equals("wait"))); // That the call_made variable was set to True assertEquals(true, execution.getVariables().get("call_made")); // Total number of tasks executed are 17 assertEquals(17, execution.getTasks().size()); } } ================================================ FILE: client/src/test/java/com/netflix/conductor/client/worker/TestWorkflowTask.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.worker; import java.io.InputStream; import java.util.List; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class TestWorkflowTask { private ObjectMapper objectMapper; @Before public void setup() { objectMapper = new ObjectMapperProvider().getObjectMapper(); } @Test public void test() throws Exception { WorkflowTask task = new WorkflowTask(); task.setType("Hello"); task.setName("name"); String json = objectMapper.writeValueAsString(task); WorkflowTask read = objectMapper.readValue(json, WorkflowTask.class); assertNotNull(read); assertEquals(task.getName(), read.getName()); assertEquals(task.getType(), read.getType()); task = new WorkflowTask(); task.setWorkflowTaskType(TaskType.SUB_WORKFLOW); task.setName("name"); json = objectMapper.writeValueAsString(task); read = objectMapper.readValue(json, WorkflowTask.class); assertNotNull(read); assertEquals(task.getName(), read.getName()); assertEquals(task.getType(), read.getType()); assertEquals(TaskType.SUB_WORKFLOW.name(), read.getType()); } @SuppressWarnings("unchecked") @Test public void testObjectMapper() throws Exception { try (InputStream stream = TestWorkflowTask.class.getResourceAsStream("/tasks.json")) { List tasks = objectMapper.readValue(stream, List.class); assertNotNull(tasks); assertEquals(1, tasks.size()); } } } ================================================ FILE: client/src/test/resources/config.properties ================================================ conductor.worker.pollingInterval=2 conductor.worker.paused=false conductor.worker.workerA.paused=true conductor.worker.workerA.domain=domainA conductor.worker.workerB.batchSize=84 conductor.worker.workerB.domain=domainB conductor.worker.Test.paused=true conductor.worker.domainTestTask2.domain=visinghDomain conductor.worker.task_run_always.pollOutOfDiscovery=true conductor.worker.task_explicit_do_not_run_always.pollOutOfDiscovery=false conductor.worker.task_ignore_override.pollOutOfDiscovery=true ================================================ FILE: client/src/test/resources/tasks.json ================================================ [ { "taskType": "task_1", "status": "IN_PROGRESS", "inputData": { "mod": null, "oddEven": null }, "referenceTaskName": "task_1", "retryCount": 0, "seq": 1, "pollCount": 1, "taskDefName": "task_1", "scheduledTime": 1539623183131, "startTime": 1539623436841, "endTime": 0, "updateTime": 1539623436841, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "2d525ed8-d0e5-44c8-a2df-a110b25c09ac", "workflowType": "kitchensink", "taskId": "bc5d9deb-cf86-443d-a1f6-59c36d2464f7", "callbackAfterSeconds": 0, "workerId": "test", "workflowTask": { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "ownerApp": "falguni-test", "createTime": 1534274994644, "createdBy": "CPEWORKFLOW", "name": "task_1", "description": "Test Task 01", "retryCount": 0, "timeoutSeconds": 5, "inputKeys": [ "mod", "oddEven" ], "outputKeys": [ "someOutput" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 0, "responseTimeoutSeconds": 0, "concurrentExecLimit": 0, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 } }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "taskDefinition": { "present": true }, "queueWaitTime": 253710, "taskStatus": "IN_PROGRESS" } ] ================================================ FILE: client/src/test/resources/test_data/loan_workflow_input.json ================================================ { "fetch_user_details": [{ "status": "COMPLETED", "output": { "userAccount": 12345 } }], "get_credit_score": [{ "status": "COMPLETED", "output": { "creditRating": 750 } }], "calculate_loan_amount": [{ "status": "COMPLETED", "output": { "authorizedLoanAmount": 10000 } }] } ================================================ FILE: client/src/test/resources/test_data/workflow1_run.json ================================================ { "createTime": 1675903039613, "updateTime": 1675903040396, "createdBy": "", "updatedBy": "", "status": "COMPLETED", "endTime": 1675903040396, "workflowId": "e90bd2d6-a811-11ed-bd43-f84d89b1eac3", "parentWorkflowId": "", "parentWorkflowTaskId": "", "tasks": [ { "taskType": "HTTP", "status": "COMPLETED", "inputData": { "asyncComplete": false, "http_request": { "method": "GET", "readTimeOut": 3000, "uri": "https://catfact.ninja/fact", "connectionTimeOut": 3000 } }, "referenceTaskName": "get_random_fact", "retryCount": 0, "seq": 1, "pollCount": 1, "taskDefName": "get_random_fact", "scheduledTime": 1675903039616, "startTime": 1675903039623, "endTime": 1675903040391, "updateTime": 1675903039623, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e90bd2d6-a811-11ed-bd43-f84d89b1eac3", "workflowType": "test_http", "taskId": "e90c4807-a811-11ed-bd43-f84d89b1eac3", "callbackAfterSeconds": 0, "workerId": "n33l3", "outputData": { "response": { "headers": { "Transfer-Encoding": [ "chunked" ], "Server": [ "nginx" ], "X-Ratelimit-Remaining": [ "99" ], "Access-Control-Allow-Origin": [ "*" ], "X-Content-Type-Options": [ "nosniff" ], "Connection": [ "keep-alive" ], "Date": [ "Thu, 09 Feb 2023 00:37:20 GMT" ], "X-Frame-Options": [ "SAMEORIGIN" ], "X-Ratelimit-Limit": [ "100" ], "Cache-Control": [ "no-cache, private" ], "Vary": [ "Accept-Encoding" ], "Set-Cookie": [ "XSRF-TOKEN=eyJpdiI6IjNNRUJ4ellWaU1HSWdGMzNWWlEzdXc9PSIsInZhbHVlIjoiYUozemozNWJJUERFVzZ2QU5TTFdGdS9oY3krclg2dWlKa0oza3gwUFlrNTd4L2YydWFSYjFKTjNzdFArRmlIL2lHbGEvU2tnbm0vWjZGZDVuMWx6UEVaT1AwNlM5REp1b0dMaWFTZldYN1FSblJQSFZSalREWXhiVUVrNUpMYXAiLCJtYWMiOiI1MjFjOWY2MDFhNWZkMDFlOGNjYjE0MmU1YmU1MGEwODQ3ZTBjNTdkMzRiZWMzYWQyMjk3NzFkNGYwYTU5NWVlIiwidGFnIjoiIn0%3D; expires=Thu, 09-Feb-2023 02:37:20 GMT; path=/; samesite=lax", "catfacts_session=eyJpdiI6IjRkMThJVWZFRnREdWExWHZ5Q0k0cEE9PSIsInZhbHVlIjoiOXZFYzJsb3IvUGZFV2tQSUVNTEVzMDJjTGRzczl2bXhtRW1PUytxTERITHp3b3dNQlBtdXlwdENMcThXTU82S1JBOHlJMU01ZlBoYUVPeG1ETmhRZEFaUDFOU0pxdHFXQ0xEWUhTQXFpcSt0SC82dmNsSDFWdmxpUFFyUjM1c3EiLCJtYWMiOiIwNzRhYTRiZjA5Nzg5M2NmMGE1NjIxMDk0NGYwNjE3MDYyZmJmMTRmYzExZGMzYWI1MTQwOWYyZjMzZGFjOGZiIiwidGFnIjoiIn0%3D; expires=Thu, 09-Feb-2023 02:37:20 GMT; path=/; httponly; samesite=lax" ], "X-XSS-Protection": [ "1; mode=block" ], "Content-Type": [ "application/json" ] }, "reasonPhrase": "OK", "body": { "fact": "Cats' hearing stops at 65 khz (kilohertz); humans' hearing stops at 20 khz.", "length": 75 }, "statusCode": 200 } }, "workflowTask": { "name": "get_random_fact", "taskReferenceName": "get_random_fact", "inputParameters": { "asyncComplete": false, "http_request": { "method": "GET", "readTimeOut": 3000, "uri": "https://catfact.ninja/fact", "connectionTimeOut": 3000 } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "rateLimited": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "loopOverTask": false, "taskDefinition": null, "queueWaitTime": 7 } ], "input": { "_X-Request-Id": "9aceedeb-3b3c-4074-8cad-79edaac3809b", "_X-Host-Id": "localhost" }, "output": { "data": "Cats' hearing stops at 65 khz (kilohertz); humans' hearing stops at 20 khz." }, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "name": "test_http", "description": "v1", "version": 1, "tasks": [ { "name": "get_random_fact", "taskReferenceName": "get_random_fact", "inputParameters": { "asyncComplete": false, "http_request": { "method": "GET", "readTimeOut": 3000, "uri": "https://catfact.ninja/fact", "connectionTimeOut": 3000 } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "rateLimited": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "data": "${get_random_fact.output.response.body.fact}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "user@orkes.io", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1675903039613, "workflowVersion": 1, "workflowName": "test_http" } ================================================ FILE: client/src/test/resources/workflows/PopulationMinMax.json ================================================ { "createTime": 1670136356629, "updateTime": 1670136356636, "name": "PopulationMinMax", "description": "Edit or extend this sample workflow. Set the workflow name to get started", "version": 1, "tasks": [ { "name": "set_variable_task_jqc56h_ref", "taskReferenceName": "set_variable_task_jqc56h_ref", "inputParameters": { "name": "Orkes" }, "type": "SET_VARIABLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "data": "${get_random_fact.output.response.body.fact}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "user@orkes.io", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ================================================ FILE: client/src/test/resources/workflows/calculate_loan_workflow.json ================================================ { "name": "test_workflow", "description": "Edit or extend this sample workflow. Set the workflow name to get started", "version": 1, "tasks": [ { "name": "fetch_user_details", "taskReferenceName": "fetch_user_details", "type": "SIMPLE", "inputParameters": { "userEmail": "${workflow.input.userEmail}" } }, { "name": "get_credit_score", "taskReferenceName": "get_credit_score", "type": "SIMPLE", "inputParameters": { "userAccountNumber": "${fetch_user_details.output.userAccount}" } }, { "name": "calculate_loan_amount", "taskReferenceName": "calculate_loan_amount", "type": "SIMPLE", "inputParameters": { "creditRating": "${get_credit_score.output.creditRating}", "loanAmount": "${workflow.input.loanAmount}" } } ], "inputParameters": [ "userEmail" ], "outputParameters": { "accountNumber": "${fetch_user_details.output.userAccount}", "creditRating": "${get_credit_score.output.creditRating}", "authorizedLoanAmount": "${calculate_loan_amount.output.authorizedLoanAmount}" }, "schemaVersion": 2 } ================================================ FILE: client/src/test/resources/workflows/kitchensink.json ================================================ { "createTime": 1670136330055, "updateTime": 1670176591044, "name": "kitchensink", "version": 1, "tasks": [ { "name": "x_test_worker_2", "taskReferenceName": "simple_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "jq", "taskReferenceName": "jq", "inputParameters": { "key1": { "value1": [ "a", "b" ] }, "queryExpression": "{ key3: (.key1.value1 + .key2.value2) }", "value2": [ "d", "e" ] }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "wait", "taskReferenceName": "wait", "inputParameters": { "duration": "1 s" }, "type": "WAIT", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "set_state", "taskReferenceName": "set_state", "inputParameters": { "call_made": true, "number": "${simple_task_0.output.number}" }, "type": "SET_VARIABLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sub_flow", "taskReferenceName": "sub_flow", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "PopulationMinMax" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "dynamic_fork", "taskReferenceName": "dynamic_fork", "inputParameters": { "forkTaskName": "x_test_worker_0", "forkTaskInputs": [ 1, 2, 3 ] }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "forkedTasks", "dynamicForkTasksInputParamName": "forkedTasksInputs", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "dynamic_fork_join", "taskReferenceName": "dynamic_fork_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork", "taskReferenceName": "fork", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "loop_until_success", "taskReferenceName": "loop_until_success", "inputParameters": { "loop_count": 2 }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": true, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ( $.loop_count['iteration'] < $.loop_until_success ) { true; } else { false; }", "loopOver": [ { "name": "fact_length", "taskReferenceName": "fact_length", "description": "Fail if the fact is too short", "inputParameters": { "number": "${get_data.output.number}" }, "type": "SWITCH", "decisionCases": { "LONG": [ { "name": "x_test_worker_1", "taskReferenceName": "simple_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "SHORT": [ { "name": "too_short", "taskReferenceName": "too_short", "inputParameters": { "terminationReason": "value too short", "terminationStatus": "FAILED" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "evaluatorType": "javascript", "expression": "$.number < 15 ? 'LONG':'LONG'" } ] }, { "name": "sub_flow_inline", "taskReferenceName": "sub_flow_inline", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "inline_sub", "version": 1, "workflowDefinition": { "name": "inline_sub", "version": 1, "tasks": [ { "name": "x_test_worker_2", "taskReferenceName": "simple_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fact_length2", "taskReferenceName": "fact_length2", "description": "Fail if the fact is too short", "inputParameters": { "number": "${get_data.output.number}" }, "type": "SWITCH", "decisionCases": { "LONG": [ { "name": "x_test_worker_1", "taskReferenceName": "simple_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "SHORT": [ { "name": "too_short", "taskReferenceName": "too_short", "inputParameters": { "terminationReason": "value too short", "terminationStatus": "FAILED" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "evaluatorType": "javascript", "expression": "$.number < 15 ? 'LONG':'LONG'" }, { "name": "sub_flow_inline_lvl2", "taskReferenceName": "sub_flow_inline_lvl2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "inline_sub", "version": 1, "workflowDefinition": { "name": "inline_sub", "version": 1, "tasks": [ { "name": "x_test_worker_2", "taskReferenceName": "simple_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "taskDefinition": { "name": "sub_flow_inline", "description": "sub_flow_inline", "retryCount": 0, "timeoutSeconds": 3000, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 20, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "pollTimeoutSeconds": 3600, "backoffScaleFactor": 1 } } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "taskDefinition": { "name": "sub_flow_inline", "description": "sub_flow_inline", "retryCount": 0, "timeoutSeconds": 3000, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 20, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "pollTimeoutSeconds": 3600, "backoffScaleFactor": 1 } } ], [ { "name": "x_test_worker_1", "taskReferenceName": "simple_task_5", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": ["sub_flow_inline","simple_task_5"], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork_join", "taskReferenceName": "fork_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": ["simple_task_5","sub_flow_inline"], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "user@orkes.io", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ================================================ FILE: client/src/test/resources/workflows/workflow1.json ================================================ { "createTime": 1674453020104, "updateTime": 1674453020105, "name": "test_http", "description": "v1", "version": 1, "tasks": [ { "name": "get_random_fact", "taskReferenceName": "get_random_fact", "inputParameters": { "http_request": { "uri": "https://catfact.ninja/fact", "method": "GET", "connectionTimeOut": 3000, "readTimeOut": 3000 } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "data": "${get_random_fact.output.response.body.fact}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "user@orkes.io", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ================================================ FILE: client-spring/build.gradle ================================================ dependencies { implementation project(':conductor-common') api project(':conductor-client') api project(':conductor-java-sdk') implementation "com.netflix.eureka:eureka-client:${revEurekaClient}" implementation 'org.springframework.boot:spring-boot-starter' } ================================================ FILE: client-spring/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import java.time.Duration; import java.util.HashMap; import java.util.Map; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("conductor.client") public class ClientProperties { private String rootUri; private String workerNamePrefix = "workflow-worker-%d"; private int threadCount = 1; private Duration sleepWhenRetryDuration = Duration.ofMillis(500); private int updateRetryCount = 3; private Map taskToDomain = new HashMap<>(); private Map taskThreadCount = new HashMap<>(); private int shutdownGracePeriodSeconds = 10; public String getRootUri() { return rootUri; } public void setRootUri(String rootUri) { this.rootUri = rootUri; } public String getWorkerNamePrefix() { return workerNamePrefix; } public void setWorkerNamePrefix(String workerNamePrefix) { this.workerNamePrefix = workerNamePrefix; } public int getThreadCount() { return threadCount; } public void setThreadCount(int threadCount) { this.threadCount = threadCount; } public Duration getSleepWhenRetryDuration() { return sleepWhenRetryDuration; } public void setSleepWhenRetryDuration(Duration sleepWhenRetryDuration) { this.sleepWhenRetryDuration = sleepWhenRetryDuration; } public int getUpdateRetryCount() { return updateRetryCount; } public void setUpdateRetryCount(int updateRetryCount) { this.updateRetryCount = updateRetryCount; } public Map getTaskToDomain() { return taskToDomain; } public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } public int getShutdownGracePeriodSeconds() { return shutdownGracePeriodSeconds; } public void setShutdownGracePeriodSeconds(int shutdownGracePeriodSeconds) { this.shutdownGracePeriodSeconds = shutdownGracePeriodSeconds; } public Map getTaskThreadCount() { return taskThreadCount; } public void setTaskThreadCount(Map taskThreadCount) { this.taskThreadCount = taskThreadCount; } } ================================================ FILE: client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import java.util.ArrayList; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; import com.netflix.discovery.EurekaClient; @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(ClientProperties.class) public class ConductorClientAutoConfiguration { @Autowired(required = false) private EurekaClient eurekaClient; @Autowired(required = false) private List workers = new ArrayList<>(); @ConditionalOnMissingBean @Bean public TaskClient taskClient(ClientProperties clientProperties) { TaskClient taskClient = new TaskClient(); taskClient.setRootURI(clientProperties.getRootUri()); return taskClient; } @ConditionalOnMissingBean @Bean public AnnotatedWorkerExecutor annotatedWorkerExecutor(TaskClient taskClient) { return new AnnotatedWorkerExecutor(taskClient); } @ConditionalOnMissingBean @Bean(initMethod = "init", destroyMethod = "shutdown") public TaskRunnerConfigurer taskRunnerConfigurer( TaskClient taskClient, ClientProperties clientProperties) { return new TaskRunnerConfigurer.Builder(taskClient, workers) .withTaskThreadCount(clientProperties.getTaskThreadCount()) .withThreadCount(clientProperties.getThreadCount()) .withSleepWhenRetry((int) clientProperties.getSleepWhenRetryDuration().toMillis()) .withUpdateRetryCount(clientProperties.getUpdateRetryCount()) .withTaskToDomain(clientProperties.getTaskToDomain()) .withShutdownGracePeriodSeconds(clientProperties.getShutdownGracePeriodSeconds()) .withEurekaClient(eurekaClient) .build(); } } ================================================ FILE: client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; import com.netflix.conductor.sdk.workflow.executor.task.WorkerConfiguration; @Component public class ConductorWorkerAutoConfiguration implements ApplicationListener { @Autowired private TaskClient taskClient; @Override public void onApplicationEvent(ContextRefreshedEvent refreshedEvent) { ApplicationContext applicationContext = refreshedEvent.getApplicationContext(); Environment environment = applicationContext.getEnvironment(); WorkerConfiguration configuration = new SpringWorkerConfiguration(environment); AnnotatedWorkerExecutor annotatedWorkerExecutor = new AnnotatedWorkerExecutor(taskClient, configuration); Map beans = applicationContext.getBeansWithAnnotation(Component.class); beans.values() .forEach( bean -> { annotatedWorkerExecutor.addBean(bean); }); annotatedWorkerExecutor.startPolling(); } } ================================================ FILE: client-spring/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import org.springframework.core.env.Environment; import com.netflix.conductor.sdk.workflow.executor.task.WorkerConfiguration; public class SpringWorkerConfiguration extends WorkerConfiguration { private final Environment environment; public SpringWorkerConfiguration(Environment environment) { this.environment = environment; } @Override public int getPollingInterval(String taskName) { String key = "conductor.worker." + taskName + ".pollingInterval"; return environment.getProperty(key, Integer.class, 0); } @Override public int getThreadCount(String taskName) { String key = "conductor.worker." + taskName + ".threadCount"; return environment.getProperty(key, Integer.class, 0); } @Override public String getDomain(String taskName) { String key = "conductor.worker." + taskName + ".domain"; return environment.getProperty(key, String.class, null); } } ================================================ FILE: client-spring/src/main/resources/META-INF/spring.factories ================================================ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ com.netflix.conductor.client.spring.ConductorClientAutoConfiguration ================================================ FILE: client-spring/src/test/java/com/netflix/conductor/client/spring/ExampleClient.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; @SpringBootApplication public class ExampleClient { public static void main(String[] args) { SpringApplication.run(ExampleClient.class, args); } @Bean public Worker worker() { return new Worker() { @Override public String getTaskDefName() { return "taskDef"; } @Override public TaskResult execute(Task task) { return new TaskResult(task); } }; } } ================================================ FILE: client-spring/src/test/java/com/netflix/conductor/client/spring/Workers.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.spring; import java.util.Date; import org.springframework.stereotype.Component; import com.netflix.conductor.sdk.workflow.executor.task.TaskContext; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.WorkerTask; @Component public class Workers { @WorkerTask(value = "hello", threadCount = 3) public String helloWorld(@InputParam("name") String name) { TaskContext context = TaskContext.get(); System.out.println(new Date() + ":: Poll count: " + context.getPollCount()); if (context.getPollCount() < 5) { context.addLog("Not ready yet, poll count is only " + context.getPollCount()); context.setCallbackAfter(1); } return "Hello, " + name; } @WorkerTask(value = "hello_again", pollingInterval = 333) public String helloAgain(@InputParam("name") String name) { TaskContext context = TaskContext.get(); System.out.println(new Date() + ":: Poll count: " + context.getPollCount()); if (context.getPollCount() < 5) { context.addLog("Not ready yet, poll count is only " + context.getPollCount()); context.setCallbackAfter(1); } return "Hello (again), " + name; } } ================================================ FILE: client-spring/src/test/resources/application.properties ================================================ conductor.client.rootUri=http://localhost:8080/api/ conductor.worker.hello.threadCount=100 conductor.worker.hello_again.domain=test ================================================ FILE: common/build.gradle ================================================ configurations { annotationsProcessorCodegen } dependencies { implementation project(':conductor-annotations') annotationsProcessorCodegen project(':conductor-annotations-processor') compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.boot:spring-boot-starter-validation' compileOnly "org.springdoc:springdoc-openapi-ui:${revOpenapi}" implementation "org.apache.commons:commons-lang3" implementation "org.apache.bval:bval-jsr:${revBval}" implementation "com.google.protobuf:protobuf-java:${revProtoBuf}" implementation "com.fasterxml.jackson.core:jackson-databind:${revFasterXml}" implementation "com.fasterxml.jackson.core:jackson-core:${revFasterXml}" // https://github.com/FasterXML/jackson-modules-base/tree/master/afterburner implementation "com.fasterxml.jackson.module:jackson-module-afterburner:${revFasterXml}" testImplementation 'org.springframework.boot:spring-boot-starter-validation' } /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ task protogen(dependsOn: jar, type: JavaExec) { classpath configurations.annotationsProcessorCodegen mainClass = "com.netflix.conductor.annotationsprocessor.protogen.ProtoGenTask" args( "conductor.proto", "com.netflix.conductor.proto", "github.com/netflix/conductor/client/gogrpc/conductor/model", "${rootDir}/grpc/src/main/proto", "${rootDir}/grpc/src/main/java/com/netflix/conductor/grpc", "com.netflix.conductor.grpc", jar.archivePath, "com.netflix.conductor.common", ) } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/config/ObjectMapperBuilderConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.config; import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; @Configuration public class ObjectMapperBuilderConfiguration { /** Disable features like {@link ObjectMapperProvider#getObjectMapper()}. */ @Bean public Jackson2ObjectMapperBuilderCustomizer conductorJackson2ObjectMapperBuilderCustomizer() { return builder -> builder.featuresToDisable( FAIL_ON_UNKNOWN_PROPERTIES, FAIL_ON_IGNORED_PROPERTIES, FAIL_ON_NULL_FOR_PRIMITIVES); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/config/ObjectMapperConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.config; import javax.annotation.PostConstruct; import org.springframework.context.annotation.Configuration; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.afterburner.AfterburnerModule; @Configuration public class ObjectMapperConfiguration { private final ObjectMapper objectMapper; public ObjectMapperConfiguration(ObjectMapper objectMapper) { this.objectMapper = objectMapper; } /** Set default property inclusion like {@link ObjectMapperProvider#getObjectMapper()}. */ @PostConstruct public void customizeDefaultObjectMapper() { objectMapper.setDefaultPropertyInclusion( JsonInclude.Value.construct( JsonInclude.Include.NON_NULL, JsonInclude.Include.ALWAYS)); objectMapper.registerModule(new AfterburnerModule()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/config/ObjectMapperProvider.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.config; import com.netflix.conductor.common.jackson.JsonProtoModule; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.afterburner.AfterburnerModule; /** * A Factory class for creating a customized {@link ObjectMapper}. This is only used by the * conductor-client module and tests that rely on {@link ObjectMapper}. See * TestObjectMapperConfiguration. */ public class ObjectMapperProvider { /** * The customizations in this method are configured using {@link * org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration} * *

Customizations are spread across, 1. {@link ObjectMapperBuilderConfiguration} 2. {@link * ObjectMapperConfiguration} 3. {@link JsonProtoModule} * *

IMPORTANT: Changes in this method need to be also performed in the default {@link * ObjectMapper} that Spring Boot creates. * * @see org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration */ public ObjectMapper getObjectMapper() { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); objectMapper.setDefaultPropertyInclusion( JsonInclude.Value.construct( JsonInclude.Include.NON_NULL, JsonInclude.Include.ALWAYS)); objectMapper.registerModule(new JsonProtoModule()); objectMapper.registerModule(new AfterburnerModule()); return objectMapper; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/constraints/NoSemiColonConstraint.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.constraints; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import javax.validation.Constraint; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; import org.apache.commons.lang3.StringUtils; import static java.lang.annotation.ElementType.FIELD; import static java.lang.annotation.ElementType.PARAMETER; /** This constraint checks semi-colon is not allowed in a given string. */ @Documented @Constraint(validatedBy = NoSemiColonConstraint.NoSemiColonValidator.class) @Target({FIELD, PARAMETER}) @Retention(RetentionPolicy.RUNTIME) public @interface NoSemiColonConstraint { String message() default "String: cannot contain the following set of characters: ':'"; Class[] groups() default {}; Class[] payload() default {}; class NoSemiColonValidator implements ConstraintValidator { @Override public void initialize(NoSemiColonConstraint constraintAnnotation) {} @Override public boolean isValid(String value, ConstraintValidatorContext context) { boolean valid = true; if (!StringUtils.isEmpty(value) && value.contains(":")) { valid = false; } return valid; } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/constraints/OwnerEmailMandatoryConstraint.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.constraints; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import javax.validation.Constraint; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; import org.apache.commons.lang3.StringUtils; import static java.lang.annotation.ElementType.FIELD; import static java.lang.annotation.ElementType.TYPE; /** * This constraint class validates that owner email is non-empty, but only if configuration says * owner email is mandatory. */ @Documented @Constraint(validatedBy = OwnerEmailMandatoryConstraint.WorkflowTaskValidValidator.class) @Target({TYPE, FIELD}) @Retention(RetentionPolicy.RUNTIME) public @interface OwnerEmailMandatoryConstraint { String message() default "ownerEmail cannot be empty"; Class[] groups() default {}; Class[] payload() default {}; class WorkflowTaskValidValidator implements ConstraintValidator { @Override public void initialize(OwnerEmailMandatoryConstraint constraintAnnotation) {} @Override public boolean isValid(String ownerEmail, ConstraintValidatorContext context) { return !ownerEmailMandatory || !StringUtils.isEmpty(ownerEmail); } private static boolean ownerEmailMandatory = true; public static void setOwnerEmailMandatory(boolean ownerEmailMandatory) { WorkflowTaskValidValidator.ownerEmailMandatory = ownerEmailMandatory; } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/constraints/TaskReferenceNameUniqueConstraint.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.constraints; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.HashMap; import java.util.List; import javax.validation.Constraint; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; import org.apache.commons.lang3.mutable.MutableBoolean; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.ConstraintParamUtil; import static java.lang.annotation.ElementType.TYPE; /** * This constraint class validates following things. * *

    *
  • 1. WorkflowDef is valid or not *
  • 2. Make sure taskReferenceName used across different tasks are unique *
  • 3. Verify inputParameters points to correct tasks or not *
*/ @Documented @Constraint(validatedBy = TaskReferenceNameUniqueConstraint.TaskReferenceNameUniqueValidator.class) @Target({TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface TaskReferenceNameUniqueConstraint { String message() default ""; Class[] groups() default {}; Class[] payload() default {}; class TaskReferenceNameUniqueValidator implements ConstraintValidator { @Override public void initialize(TaskReferenceNameUniqueConstraint constraintAnnotation) {} @Override public boolean isValid(WorkflowDef workflowDef, ConstraintValidatorContext context) { context.disableDefaultConstraintViolation(); boolean valid = true; // check if taskReferenceNames are unique across tasks or not HashMap taskReferenceMap = new HashMap<>(); for (WorkflowTask workflowTask : workflowDef.collectTasks()) { if (taskReferenceMap.containsKey(workflowTask.getTaskReferenceName())) { String message = String.format( "taskReferenceName: %s should be unique across tasks for a given workflowDefinition: %s", workflowTask.getTaskReferenceName(), workflowDef.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } else { taskReferenceMap.put(workflowTask.getTaskReferenceName(), 1); } } // check inputParameters points to valid taskDef return valid & verifyTaskInputParameters(context, workflowDef); } private boolean verifyTaskInputParameters( ConstraintValidatorContext context, WorkflowDef workflow) { MutableBoolean valid = new MutableBoolean(); valid.setValue(true); if (workflow.getTasks() == null) { return valid.getValue(); } workflow.getTasks().stream() .filter(workflowTask -> workflowTask.getInputParameters() != null) .forEach( workflowTask -> { List errors = ConstraintParamUtil.validateInputParam( workflowTask.getInputParameters(), workflowTask.getName(), workflow); errors.forEach( message -> context.buildConstraintViolationWithTemplate( message) .addConstraintViolation()); if (errors.size() > 0) { valid.setValue(false); } }); return valid.getValue(); } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/constraints/TaskTimeoutConstraint.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.constraints; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import javax.validation.Constraint; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; import com.netflix.conductor.common.metadata.tasks.TaskDef; import static java.lang.annotation.ElementType.TYPE; /** * This constraint checks for a given task responseTimeoutSeconds should be less than * timeoutSeconds. */ @Documented @Constraint(validatedBy = TaskTimeoutConstraint.TaskTimeoutValidator.class) @Target({TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface TaskTimeoutConstraint { String message() default ""; Class[] groups() default {}; Class[] payload() default {}; class TaskTimeoutValidator implements ConstraintValidator { @Override public void initialize(TaskTimeoutConstraint constraintAnnotation) {} @Override public boolean isValid(TaskDef taskDef, ConstraintValidatorContext context) { context.disableDefaultConstraintViolation(); boolean valid = true; if (taskDef.getTimeoutSeconds() > 0) { if (taskDef.getResponseTimeoutSeconds() > taskDef.getTimeoutSeconds()) { valid = false; String message = String.format( "TaskDef: %s responseTimeoutSeconds: %d must be less than timeoutSeconds: %d", taskDef.getName(), taskDef.getResponseTimeoutSeconds(), taskDef.getTimeoutSeconds()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); } } return valid; } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/jackson/JsonProtoModule.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.jackson; import java.io.IOException; import org.springframework.stereotype.Component; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.protobuf.Any; import com.google.protobuf.ByteString; import com.google.protobuf.Message; /** * JsonProtoModule can be registered into an {@link ObjectMapper} to enable the serialization and * deserialization of ProtoBuf objects from/to JSON. * *

Right now this module only provides (de)serialization for the {@link Any} ProtoBuf type, as * this is the only ProtoBuf object which we're currently exposing through the REST API. * *

Annotated as {@link Component} so Spring can register it with {@link ObjectMapper} * * @see AnySerializer * @see AnyDeserializer * @see org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration */ @Component(JsonProtoModule.NAME) public class JsonProtoModule extends SimpleModule { public static final String NAME = "ConductorJsonProtoModule"; private static final String JSON_TYPE = "@type"; private static final String JSON_VALUE = "@value"; /** * AnySerializer converts a ProtoBuf {@link Any} object into its JSON representation. * *

This is not a canonical ProtoBuf JSON representation. Let us explain what we're * trying to accomplish here: * *

The {@link Any} ProtoBuf message is a type in the PB standard library that can store any * other arbitrary ProtoBuf message in a type-safe way, even when the server has no knowledge of * the schema of the stored message. * *

It accomplishes this by storing a tuple of information: an URL-like type declaration for * the stored message, and the serialized binary encoding of the stored message itself. Language * specific implementations of ProtoBuf provide helper methods to encode and decode arbitrary * messages into an {@link Any} object ({@link Any#pack(Message)} in Java). * *

We want to expose these {@link Any} objects in the REST API because they've been * introduced as part of the new GRPC interface to Conductor, but unfortunately we cannot encode * them using their canonical ProtoBuf JSON encoding. According to the docs: * *

The JSON representation of an `Any` value uses the regular representation of the * deserialized, embedded message, with an additional field `@type` which contains the type URL. * Example: * *

package google.profile; message Person { string first_name = 1; string last_name = 2; } { * "@type": "type.googleapis.com/google.profile.Person", "firstName": , "lastName": * } * *

In order to accomplish this representation, the PB-JSON encoder needs to have knowledge of * all the ProtoBuf messages that could be serialized inside the {@link Any} message. This is * not possible to accomplish inside the Conductor server, which is simply passing through * arbitrary payloads from/to clients. * *

Consequently, to actually expose the Message through the REST API, we must create a custom * encoding that contains the raw data of the serialized message, as we are not able to * deserialize it on the server. We simply return a dictionary with '@type' and '@value' keys, * where '@type' is identical to the canonical representation, but '@value' contains a base64 * encoded string with the binary data of the serialized message. * *

Since all the provided Conductor clients are required to know this encoding, it's always * possible to re-build the original {@link Any} message regardless of the client's language. * *

{@see AnyDeserializer} */ @SuppressWarnings("InnerClassMayBeStatic") protected class AnySerializer extends JsonSerializer { @Override public void serialize(Any value, JsonGenerator jgen, SerializerProvider provider) throws IOException { jgen.writeStartObject(); jgen.writeStringField(JSON_TYPE, value.getTypeUrl()); jgen.writeBinaryField(JSON_VALUE, value.getValue().toByteArray()); jgen.writeEndObject(); } } /** * AnyDeserializer converts the custom JSON representation of an {@link Any} value into its * original form. * *

{@see AnySerializer} for details on this representation. */ @SuppressWarnings("InnerClassMayBeStatic") protected class AnyDeserializer extends JsonDeserializer { @Override public Any deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { JsonNode root = p.getCodec().readTree(p); JsonNode type = root.get(JSON_TYPE); JsonNode value = root.get(JSON_VALUE); if (type == null || !type.isTextual()) { ctxt.reportMappingException( "invalid '@type' field when deserializing ProtoBuf Any object"); } if (value == null || !value.isTextual()) { ctxt.reportMappingException( "invalid '@value' field when deserializing ProtoBuf Any object"); } return Any.newBuilder() .setTypeUrl(type.textValue()) .setValue(ByteString.copyFrom(value.binaryValue())) .build(); } } public JsonProtoModule() { super(NAME); addSerializer(Any.class, new AnySerializer()); addDeserializer(Any.class, new AnyDeserializer()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/Auditable.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata; public abstract class Auditable { private String ownerApp; private Long createTime; private Long updateTime; private String createdBy; private String updatedBy; /** * @return the ownerApp */ public String getOwnerApp() { return ownerApp; } /** * @param ownerApp the ownerApp to set */ public void setOwnerApp(String ownerApp) { this.ownerApp = ownerApp; } /** * @return the createTime */ public Long getCreateTime() { return createTime; } /** * @param createTime the createTime to set */ public void setCreateTime(Long createTime) { this.createTime = createTime; } /** * @return the updateTime */ public Long getUpdateTime() { return updateTime; } /** * @param updateTime the updateTime to set */ public void setUpdateTime(Long updateTime) { this.updateTime = updateTime; } /** * @return the createdBy */ public String getCreatedBy() { return createdBy; } /** * @param createdBy the createdBy to set */ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; } /** * @return the updatedBy */ public String getUpdatedBy() { return updatedBy; } /** * @param updatedBy the updatedBy to set */ public void setUpdatedBy(String updatedBy) { this.updatedBy = updatedBy; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/BaseDef.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata; import java.util.Collections; import java.util.EnumMap; import java.util.Map; import com.netflix.conductor.common.metadata.acl.Permission; /** * A base class for {@link com.netflix.conductor.common.metadata.workflow.WorkflowDef} and {@link * com.netflix.conductor.common.metadata.tasks.TaskDef}. */ public abstract class BaseDef extends Auditable { private final Map accessPolicy = new EnumMap<>(Permission.class); public void addPermission(Permission permission, String allowedAuthority) { this.accessPolicy.put(permission, allowedAuthority); } public void addPermissionIfAbsent(Permission permission, String allowedAuthority) { this.accessPolicy.putIfAbsent(permission, allowedAuthority); } public void removePermission(Permission permission) { this.accessPolicy.remove(permission); } public String getAllowedAuthority(Permission permission) { return this.accessPolicy.get(permission); } public void clearAccessPolicy() { this.accessPolicy.clear(); } public Map getAccessPolicy() { return Collections.unmodifiableMap(this.accessPolicy); } public void setAccessPolicy(Map accessPolicy) { this.accessPolicy.putAll(accessPolicy); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/acl/Permission.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.acl; import com.netflix.conductor.annotations.protogen.ProtoEnum; @ProtoEnum public enum Permission { OWNER, OPERATOR } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/events/EventExecution.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.events; import java.util.HashMap; import java.util.Map; import java.util.Objects; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.events.EventHandler.Action; @ProtoMessage public class EventExecution { @ProtoEnum public enum Status { IN_PROGRESS, COMPLETED, FAILED, SKIPPED } @ProtoField(id = 1) private String id; @ProtoField(id = 2) private String messageId; @ProtoField(id = 3) private String name; @ProtoField(id = 4) private String event; @ProtoField(id = 5) private long created; @ProtoField(id = 6) private Status status; @ProtoField(id = 7) private Action.Type action; @ProtoField(id = 8) private Map output = new HashMap<>(); public EventExecution() {} public EventExecution(String id, String messageId) { this.id = id; this.messageId = messageId; } /** * @return the id */ public String getId() { return id; } /** * @param id the id to set */ public void setId(String id) { this.id = id; } /** * @return the messageId */ public String getMessageId() { return messageId; } /** * @param messageId the messageId to set */ public void setMessageId(String messageId) { this.messageId = messageId; } /** * @return the name */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the event */ public String getEvent() { return event; } /** * @param event the event to set */ public void setEvent(String event) { this.event = event; } /** * @return the created */ public long getCreated() { return created; } /** * @param created the created to set */ public void setCreated(long created) { this.created = created; } /** * @return the status */ public Status getStatus() { return status; } /** * @param status the status to set */ public void setStatus(Status status) { this.status = status; } /** * @return the action */ public Action.Type getAction() { return action; } /** * @param action the action to set */ public void setAction(Action.Type action) { this.action = action; } /** * @return the output */ public Map getOutput() { return output; } /** * @param output the output to set */ public void setOutput(Map output) { this.output = output; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } EventExecution execution = (EventExecution) o; return created == execution.created && Objects.equals(id, execution.id) && Objects.equals(messageId, execution.messageId) && Objects.equals(name, execution.name) && Objects.equals(event, execution.event) && status == execution.status && action == execution.action && Objects.equals(output, execution.output); } @Override public int hashCode() { return Objects.hash(id, messageId, name, event, created, status, action, output); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/events/EventHandler.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.events; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.google.protobuf.Any; import io.swagger.v3.oas.annotations.Hidden; /** Defines an event handler */ @ProtoMessage public class EventHandler { @ProtoField(id = 1) @NotEmpty(message = "Missing event handler name") private String name; @ProtoField(id = 2) @NotEmpty(message = "Missing event location") private String event; @ProtoField(id = 3) private String condition; @ProtoField(id = 4) @NotNull @NotEmpty(message = "No actions specified. Please specify at-least one action") private List<@Valid Action> actions = new LinkedList<>(); @ProtoField(id = 5) private boolean active; @ProtoField(id = 6) private String evaluatorType; public EventHandler() {} /** * @return the name MUST be unique within a conductor instance */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the event */ public String getEvent() { return event; } /** * @param event the event to set */ public void setEvent(String event) { this.event = event; } /** * @return the condition */ public String getCondition() { return condition; } /** * @param condition the condition to set */ public void setCondition(String condition) { this.condition = condition; } /** * @return the actions */ public List getActions() { return actions; } /** * @param actions the actions to set */ public void setActions(List actions) { this.actions = actions; } /** * @return the active */ public boolean isActive() { return active; } /** * @param active if set to false, the event handler is deactivated */ public void setActive(boolean active) { this.active = active; } /** * @return the evaluator type */ public String getEvaluatorType() { return evaluatorType; } /** * @param evaluatorType the evaluatorType to set */ public void setEvaluatorType(String evaluatorType) { this.evaluatorType = evaluatorType; } @ProtoMessage public static class Action { @ProtoEnum public enum Type { start_workflow, complete_task, fail_task } @ProtoField(id = 1) private Type action; @ProtoField(id = 2) private StartWorkflow start_workflow; @ProtoField(id = 3) private TaskDetails complete_task; @ProtoField(id = 4) private TaskDetails fail_task; @ProtoField(id = 5) private boolean expandInlineJSON; /** * @return the action */ public Type getAction() { return action; } /** * @param action the action to set */ public void setAction(Type action) { this.action = action; } /** * @return the start_workflow */ public StartWorkflow getStart_workflow() { return start_workflow; } /** * @param start_workflow the start_workflow to set */ public void setStart_workflow(StartWorkflow start_workflow) { this.start_workflow = start_workflow; } /** * @return the complete_task */ public TaskDetails getComplete_task() { return complete_task; } /** * @param complete_task the complete_task to set */ public void setComplete_task(TaskDetails complete_task) { this.complete_task = complete_task; } /** * @return the fail_task */ public TaskDetails getFail_task() { return fail_task; } /** * @param fail_task the fail_task to set */ public void setFail_task(TaskDetails fail_task) { this.fail_task = fail_task; } /** * @param expandInlineJSON when set to true, the in-lined JSON strings are expanded to a * full json document */ public void setExpandInlineJSON(boolean expandInlineJSON) { this.expandInlineJSON = expandInlineJSON; } /** * @return true if the json strings within the payload should be expanded. */ public boolean isExpandInlineJSON() { return expandInlineJSON; } } @ProtoMessage public static class TaskDetails { @ProtoField(id = 1) private String workflowId; @ProtoField(id = 2) private String taskRefName; @ProtoField(id = 3) private Map output = new HashMap<>(); @ProtoField(id = 4) @Hidden private Any outputMessage; @ProtoField(id = 5) private String taskId; /** * @return the workflowId */ public String getWorkflowId() { return workflowId; } /** * @param workflowId the workflowId to set */ public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } /** * @return the taskRefName */ public String getTaskRefName() { return taskRefName; } /** * @param taskRefName the taskRefName to set */ public void setTaskRefName(String taskRefName) { this.taskRefName = taskRefName; } /** * @return the output */ public Map getOutput() { return output; } /** * @param output the output to set */ public void setOutput(Map output) { this.output = output; } public Any getOutputMessage() { return outputMessage; } public void setOutputMessage(Any outputMessage) { this.outputMessage = outputMessage; } /** * @return the taskId */ public String getTaskId() { return taskId; } /** * @param taskId the taskId to set */ public void setTaskId(String taskId) { this.taskId = taskId; } } @ProtoMessage public static class StartWorkflow { @ProtoField(id = 1) private String name; @ProtoField(id = 2) private Integer version; @ProtoField(id = 3) private String correlationId; @ProtoField(id = 4) private Map input = new HashMap<>(); @ProtoField(id = 5) @Hidden private Any inputMessage; @ProtoField(id = 6) private Map taskToDomain; /** * @return the name */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the version */ public Integer getVersion() { return version; } /** * @param version the version to set */ public void setVersion(Integer version) { this.version = version; } /** * @return the correlationId */ public String getCorrelationId() { return correlationId; } /** * @param correlationId the correlationId to set */ public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } /** * @return the input */ public Map getInput() { return input; } /** * @param input the input to set */ public void setInput(Map input) { this.input = input; } public Any getInputMessage() { return inputMessage; } public void setInputMessage(Any inputMessage) { this.inputMessage = inputMessage; } public Map getTaskToDomain() { return taskToDomain; } public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/PollData.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.Objects; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; @ProtoMessage public class PollData { @ProtoField(id = 1) private String queueName; @ProtoField(id = 2) private String domain; @ProtoField(id = 3) private String workerId; @ProtoField(id = 4) private long lastPollTime; public PollData() { super(); } public PollData(String queueName, String domain, String workerId, long lastPollTime) { super(); this.queueName = queueName; this.domain = domain; this.workerId = workerId; this.lastPollTime = lastPollTime; } public String getQueueName() { return queueName; } public void setQueueName(String queueName) { this.queueName = queueName; } public String getDomain() { return domain; } public void setDomain(String domain) { this.domain = domain; } public String getWorkerId() { return workerId; } public void setWorkerId(String workerId) { this.workerId = workerId; } public long getLastPollTime() { return lastPollTime; } public void setLastPollTime(long lastPollTime) { this.lastPollTime = lastPollTime; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } PollData pollData = (PollData) o; return getLastPollTime() == pollData.getLastPollTime() && Objects.equals(getQueueName(), pollData.getQueueName()) && Objects.equals(getDomain(), pollData.getDomain()) && Objects.equals(getWorkerId(), pollData.getWorkerId()); } @Override public int hashCode() { return Objects.hash(getQueueName(), getDomain(), getWorkerId(), getLastPollTime()); } @Override public String toString() { return "PollData{" + "queueName='" + queueName + '\'' + ", domain='" + domain + '\'' + ", workerId='" + workerId + '\'' + ", lastPollTime=" + lastPollTime + '}'; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/Task.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.google.protobuf.Any; import io.swagger.v3.oas.annotations.Hidden; @ProtoMessage public class Task { @ProtoEnum public enum Status { IN_PROGRESS(false, true, true), CANCELED(true, false, false), FAILED(true, false, true), FAILED_WITH_TERMINAL_ERROR( true, false, false), // No retries even if retries are configured, the task and the related // workflow should be terminated COMPLETED(true, true, true), COMPLETED_WITH_ERRORS(true, true, true), SCHEDULED(false, true, true), TIMED_OUT(true, false, true), SKIPPED(true, true, false); private final boolean terminal; private final boolean successful; private final boolean retriable; Status(boolean terminal, boolean successful, boolean retriable) { this.terminal = terminal; this.successful = successful; this.retriable = retriable; } public boolean isTerminal() { return terminal; } public boolean isSuccessful() { return successful; } public boolean isRetriable() { return retriable; } } @ProtoField(id = 1) private String taskType; @ProtoField(id = 2) private Status status; @ProtoField(id = 3) private Map inputData = new HashMap<>(); @ProtoField(id = 4) private String referenceTaskName; @ProtoField(id = 5) private int retryCount; @ProtoField(id = 6) private int seq; @ProtoField(id = 7) private String correlationId; @ProtoField(id = 8) private int pollCount; @ProtoField(id = 9) private String taskDefName; /** Time when the task was scheduled */ @ProtoField(id = 10) private long scheduledTime; /** Time when the task was first polled */ @ProtoField(id = 11) private long startTime; /** Time when the task completed executing */ @ProtoField(id = 12) private long endTime; /** Time when the task was last updated */ @ProtoField(id = 13) private long updateTime; @ProtoField(id = 14) private int startDelayInSeconds; @ProtoField(id = 15) private String retriedTaskId; @ProtoField(id = 16) private boolean retried; @ProtoField(id = 17) private boolean executed; @ProtoField(id = 18) private boolean callbackFromWorker = true; @ProtoField(id = 19) private long responseTimeoutSeconds; @ProtoField(id = 20) private String workflowInstanceId; @ProtoField(id = 21) private String workflowType; @ProtoField(id = 22) private String taskId; @ProtoField(id = 23) private String reasonForIncompletion; @ProtoField(id = 24) private long callbackAfterSeconds; @ProtoField(id = 25) private String workerId; @ProtoField(id = 26) private Map outputData = new HashMap<>(); @ProtoField(id = 27) private WorkflowTask workflowTask; @ProtoField(id = 28) private String domain; @ProtoField(id = 29) @Hidden private Any inputMessage; @ProtoField(id = 30) @Hidden private Any outputMessage; // id 31 is reserved @ProtoField(id = 32) private int rateLimitPerFrequency; @ProtoField(id = 33) private int rateLimitFrequencyInSeconds; @ProtoField(id = 34) private String externalInputPayloadStoragePath; @ProtoField(id = 35) private String externalOutputPayloadStoragePath; @ProtoField(id = 36) private int workflowPriority; @ProtoField(id = 37) private String executionNameSpace; @ProtoField(id = 38) private String isolationGroupId; @ProtoField(id = 40) private int iteration; @ProtoField(id = 41) private String subWorkflowId; /** * Use to note that a sub workflow associated with SUB_WORKFLOW task has an action performed on * it directly. */ @ProtoField(id = 42) private boolean subworkflowChanged; public Task() {} /** * @return Type of the task * @see TaskType */ public String getTaskType() { return taskType; } public void setTaskType(String taskType) { this.taskType = taskType; } /** * @return Status of the task */ public Status getStatus() { return status; } /** * @param status Status of the task */ public void setStatus(Status status) { this.status = status; } public Map getInputData() { return inputData; } public void setInputData(Map inputData) { if (inputData == null) { inputData = new HashMap<>(); } this.inputData = inputData; } /** * @return the referenceTaskName */ public String getReferenceTaskName() { return referenceTaskName; } /** * @param referenceTaskName the referenceTaskName to set */ public void setReferenceTaskName(String referenceTaskName) { this.referenceTaskName = referenceTaskName; } /** * @return the correlationId */ public String getCorrelationId() { return correlationId; } /** * @param correlationId the correlationId to set */ public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } /** * @return the retryCount */ public int getRetryCount() { return retryCount; } /** * @param retryCount the retryCount to set */ public void setRetryCount(int retryCount) { this.retryCount = retryCount; } /** * @return the scheduledTime */ public long getScheduledTime() { return scheduledTime; } /** * @param scheduledTime the scheduledTime to set */ public void setScheduledTime(long scheduledTime) { this.scheduledTime = scheduledTime; } /** * @return the startTime */ public long getStartTime() { return startTime; } /** * @param startTime the startTime to set */ public void setStartTime(long startTime) { this.startTime = startTime; } /** * @return the endTime */ public long getEndTime() { return endTime; } /** * @param endTime the endTime to set */ public void setEndTime(long endTime) { this.endTime = endTime; } /** * @return the startDelayInSeconds */ public int getStartDelayInSeconds() { return startDelayInSeconds; } /** * @param startDelayInSeconds the startDelayInSeconds to set */ public void setStartDelayInSeconds(int startDelayInSeconds) { this.startDelayInSeconds = startDelayInSeconds; } /** * @return the retriedTaskId */ public String getRetriedTaskId() { return retriedTaskId; } /** * @param retriedTaskId the retriedTaskId to set */ public void setRetriedTaskId(String retriedTaskId) { this.retriedTaskId = retriedTaskId; } /** * @return the seq */ public int getSeq() { return seq; } /** * @param seq the seq to set */ public void setSeq(int seq) { this.seq = seq; } /** * @return the updateTime */ public long getUpdateTime() { return updateTime; } /** * @param updateTime the updateTime to set */ public void setUpdateTime(long updateTime) { this.updateTime = updateTime; } /** * @return the queueWaitTime */ public long getQueueWaitTime() { if (this.startTime > 0 && this.scheduledTime > 0) { if (this.updateTime > 0 && getCallbackAfterSeconds() > 0) { long waitTime = System.currentTimeMillis() - (this.updateTime + (getCallbackAfterSeconds() * 1000)); return waitTime > 0 ? waitTime : 0; } else { return this.startTime - this.scheduledTime; } } return 0L; } /** * @return True if the task has been retried after failure */ public boolean isRetried() { return retried; } /** * @param retried the retried to set */ public void setRetried(boolean retried) { this.retried = retried; } /** * @return True if the task has completed its lifecycle within conductor (from start to * completion to being updated in the datastore) */ public boolean isExecuted() { return executed; } /** * @param executed the executed value to set */ public void setExecuted(boolean executed) { this.executed = executed; } /** * @return No. of times task has been polled */ public int getPollCount() { return pollCount; } public void setPollCount(int pollCount) { this.pollCount = pollCount; } public void incrementPollCount() { ++this.pollCount; } public boolean isCallbackFromWorker() { return callbackFromWorker; } public void setCallbackFromWorker(boolean callbackFromWorker) { this.callbackFromWorker = callbackFromWorker; } /** * @return Name of the task definition */ public String getTaskDefName() { if (taskDefName == null || "".equals(taskDefName)) { taskDefName = taskType; } return taskDefName; } /** * @param taskDefName Name of the task definition */ public void setTaskDefName(String taskDefName) { this.taskDefName = taskDefName; } /** * @return the timeout for task to send response. After this timeout, the task will be re-queued */ public long getResponseTimeoutSeconds() { return responseTimeoutSeconds; } /** * @param responseTimeoutSeconds - timeout for task to send response. After this timeout, the * task will be re-queued */ public void setResponseTimeoutSeconds(long responseTimeoutSeconds) { this.responseTimeoutSeconds = responseTimeoutSeconds; } /** * @return the workflowInstanceId */ public String getWorkflowInstanceId() { return workflowInstanceId; } /** * @param workflowInstanceId the workflowInstanceId to set */ public void setWorkflowInstanceId(String workflowInstanceId) { this.workflowInstanceId = workflowInstanceId; } public String getWorkflowType() { return workflowType; } /** * @param workflowType the name of the workflow * @return the task object with the workflow type set */ public com.netflix.conductor.common.metadata.tasks.Task setWorkflowType(String workflowType) { this.workflowType = workflowType; return this; } /** * @return the taskId */ public String getTaskId() { return taskId; } /** * @param taskId the taskId to set */ public void setTaskId(String taskId) { this.taskId = taskId; } /** * @return the reasonForIncompletion */ public String getReasonForIncompletion() { return reasonForIncompletion; } /** * @param reasonForIncompletion the reasonForIncompletion to set */ public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = StringUtils.substring(reasonForIncompletion, 0, 500); } /** * @return the callbackAfterSeconds */ public long getCallbackAfterSeconds() { return callbackAfterSeconds; } /** * @param callbackAfterSeconds the callbackAfterSeconds to set */ public void setCallbackAfterSeconds(long callbackAfterSeconds) { this.callbackAfterSeconds = callbackAfterSeconds; } /** * @return the workerId */ public String getWorkerId() { return workerId; } /** * @param workerId the workerId to set */ public void setWorkerId(String workerId) { this.workerId = workerId; } /** * @return the outputData */ public Map getOutputData() { return outputData; } /** * @param outputData the outputData to set */ public void setOutputData(Map outputData) { if (outputData == null) { outputData = new HashMap<>(); } this.outputData = outputData; } /** * @return Workflow Task definition */ public WorkflowTask getWorkflowTask() { return workflowTask; } /** * @param workflowTask Task definition */ public void setWorkflowTask(WorkflowTask workflowTask) { this.workflowTask = workflowTask; } /** * @return the domain */ public String getDomain() { return domain; } /** * @param domain the Domain */ public void setDomain(String domain) { this.domain = domain; } public Any getInputMessage() { return inputMessage; } public void setInputMessage(Any inputMessage) { this.inputMessage = inputMessage; } public Any getOutputMessage() { return outputMessage; } public void setOutputMessage(Any outputMessage) { this.outputMessage = outputMessage; } /** * @return {@link Optional} containing the task definition if available */ public Optional getTaskDefinition() { return Optional.ofNullable(this.getWorkflowTask()).map(WorkflowTask::getTaskDefinition); } public int getRateLimitPerFrequency() { return rateLimitPerFrequency; } public void setRateLimitPerFrequency(int rateLimitPerFrequency) { this.rateLimitPerFrequency = rateLimitPerFrequency; } public int getRateLimitFrequencyInSeconds() { return rateLimitFrequencyInSeconds; } public void setRateLimitFrequencyInSeconds(int rateLimitFrequencyInSeconds) { this.rateLimitFrequencyInSeconds = rateLimitFrequencyInSeconds; } /** * @return the external storage path for the task input payload */ public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } /** * @param externalInputPayloadStoragePath the external storage path where the task input payload * is stored */ public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } /** * @return the external storage path for the task output payload */ public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } /** * @param externalOutputPayloadStoragePath the external storage path where the task output * payload is stored */ public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } public void setIsolationGroupId(String isolationGroupId) { this.isolationGroupId = isolationGroupId; } public String getIsolationGroupId() { return isolationGroupId; } public String getExecutionNameSpace() { return executionNameSpace; } public void setExecutionNameSpace(String executionNameSpace) { this.executionNameSpace = executionNameSpace; } /** * @return the iteration */ public int getIteration() { return iteration; } /** * @param iteration iteration */ public void setIteration(int iteration) { this.iteration = iteration; } public boolean isLoopOverTask() { return iteration > 0; } /** * @return the priority defined on workflow */ public int getWorkflowPriority() { return workflowPriority; } /** * @param workflowPriority Priority defined for workflow */ public void setWorkflowPriority(int workflowPriority) { this.workflowPriority = workflowPriority; } public boolean isSubworkflowChanged() { return subworkflowChanged; } public void setSubworkflowChanged(boolean subworkflowChanged) { this.subworkflowChanged = subworkflowChanged; } public String getSubWorkflowId() { // For backwards compatibility if (StringUtils.isNotBlank(subWorkflowId)) { return subWorkflowId; } else { return this.getOutputData() != null && this.getOutputData().get("subWorkflowId") != null ? (String) this.getOutputData().get("subWorkflowId") : this.getInputData() != null ? (String) this.getInputData().get("subWorkflowId") : null; } } public void setSubWorkflowId(String subWorkflowId) { this.subWorkflowId = subWorkflowId; // For backwards compatibility if (this.getOutputData() != null && this.getOutputData().containsKey("subWorkflowId")) { this.getOutputData().put("subWorkflowId", subWorkflowId); } } public Task copy() { Task copy = new Task(); copy.setCallbackAfterSeconds(callbackAfterSeconds); copy.setCallbackFromWorker(callbackFromWorker); copy.setCorrelationId(correlationId); copy.setInputData(inputData); copy.setOutputData(outputData); copy.setReferenceTaskName(referenceTaskName); copy.setStartDelayInSeconds(startDelayInSeconds); copy.setTaskDefName(taskDefName); copy.setTaskType(taskType); copy.setWorkflowInstanceId(workflowInstanceId); copy.setWorkflowType(workflowType); copy.setResponseTimeoutSeconds(responseTimeoutSeconds); copy.setStatus(status); copy.setRetryCount(retryCount); copy.setPollCount(pollCount); copy.setTaskId(taskId); copy.setWorkflowTask(workflowTask); copy.setDomain(domain); copy.setInputMessage(inputMessage); copy.setOutputMessage(outputMessage); copy.setRateLimitPerFrequency(rateLimitPerFrequency); copy.setRateLimitFrequencyInSeconds(rateLimitFrequencyInSeconds); copy.setExternalInputPayloadStoragePath(externalInputPayloadStoragePath); copy.setExternalOutputPayloadStoragePath(externalOutputPayloadStoragePath); copy.setWorkflowPriority(workflowPriority); copy.setIteration(iteration); copy.setExecutionNameSpace(executionNameSpace); copy.setIsolationGroupId(isolationGroupId); copy.setSubWorkflowId(getSubWorkflowId()); copy.setSubworkflowChanged(subworkflowChanged); return copy; } /** * @return a deep copy of the task instance To be used inside copy Workflow method to provide a * valid deep copied object. Note: This does not copy the following fields: *

    *
  • retried *
  • updateTime *
  • retriedTaskId *
*/ public Task deepCopy() { Task deepCopy = copy(); deepCopy.setStartTime(startTime); deepCopy.setScheduledTime(scheduledTime); deepCopy.setEndTime(endTime); deepCopy.setWorkerId(workerId); deepCopy.setReasonForIncompletion(reasonForIncompletion); deepCopy.setSeq(seq); return deepCopy; } @Override public String toString() { return "Task{" + "taskType='" + taskType + '\'' + ", status=" + status + ", inputData=" + inputData + ", referenceTaskName='" + referenceTaskName + '\'' + ", retryCount=" + retryCount + ", seq=" + seq + ", correlationId='" + correlationId + '\'' + ", pollCount=" + pollCount + ", taskDefName='" + taskDefName + '\'' + ", scheduledTime=" + scheduledTime + ", startTime=" + startTime + ", endTime=" + endTime + ", updateTime=" + updateTime + ", startDelayInSeconds=" + startDelayInSeconds + ", retriedTaskId='" + retriedTaskId + '\'' + ", retried=" + retried + ", executed=" + executed + ", callbackFromWorker=" + callbackFromWorker + ", responseTimeoutSeconds=" + responseTimeoutSeconds + ", workflowInstanceId='" + workflowInstanceId + '\'' + ", workflowType='" + workflowType + '\'' + ", taskId='" + taskId + '\'' + ", reasonForIncompletion='" + reasonForIncompletion + '\'' + ", callbackAfterSeconds=" + callbackAfterSeconds + ", workerId='" + workerId + '\'' + ", outputData=" + outputData + ", workflowTask=" + workflowTask + ", domain='" + domain + '\'' + ", inputMessage='" + inputMessage + '\'' + ", outputMessage='" + outputMessage + '\'' + ", rateLimitPerFrequency=" + rateLimitPerFrequency + ", rateLimitFrequencyInSeconds=" + rateLimitFrequencyInSeconds + ", workflowPriority=" + workflowPriority + ", externalInputPayloadStoragePath='" + externalInputPayloadStoragePath + '\'' + ", externalOutputPayloadStoragePath='" + externalOutputPayloadStoragePath + '\'' + ", isolationGroupId='" + isolationGroupId + '\'' + ", executionNameSpace='" + executionNameSpace + '\'' + ", subworkflowChanged='" + subworkflowChanged + '\'' + '}'; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Task task = (Task) o; return getRetryCount() == task.getRetryCount() && getSeq() == task.getSeq() && getPollCount() == task.getPollCount() && getScheduledTime() == task.getScheduledTime() && getStartTime() == task.getStartTime() && getEndTime() == task.getEndTime() && getUpdateTime() == task.getUpdateTime() && getStartDelayInSeconds() == task.getStartDelayInSeconds() && isRetried() == task.isRetried() && isExecuted() == task.isExecuted() && isCallbackFromWorker() == task.isCallbackFromWorker() && getResponseTimeoutSeconds() == task.getResponseTimeoutSeconds() && getCallbackAfterSeconds() == task.getCallbackAfterSeconds() && getRateLimitPerFrequency() == task.getRateLimitPerFrequency() && getRateLimitFrequencyInSeconds() == task.getRateLimitFrequencyInSeconds() && Objects.equals(getTaskType(), task.getTaskType()) && getStatus() == task.getStatus() && getIteration() == task.getIteration() && getWorkflowPriority() == task.getWorkflowPriority() && Objects.equals(getInputData(), task.getInputData()) && Objects.equals(getReferenceTaskName(), task.getReferenceTaskName()) && Objects.equals(getCorrelationId(), task.getCorrelationId()) && Objects.equals(getTaskDefName(), task.getTaskDefName()) && Objects.equals(getRetriedTaskId(), task.getRetriedTaskId()) && Objects.equals(getWorkflowInstanceId(), task.getWorkflowInstanceId()) && Objects.equals(getWorkflowType(), task.getWorkflowType()) && Objects.equals(getTaskId(), task.getTaskId()) && Objects.equals(getReasonForIncompletion(), task.getReasonForIncompletion()) && Objects.equals(getWorkerId(), task.getWorkerId()) && Objects.equals(getOutputData(), task.getOutputData()) && Objects.equals(getWorkflowTask(), task.getWorkflowTask()) && Objects.equals(getDomain(), task.getDomain()) && Objects.equals(getInputMessage(), task.getInputMessage()) && Objects.equals(getOutputMessage(), task.getOutputMessage()) && Objects.equals( getExternalInputPayloadStoragePath(), task.getExternalInputPayloadStoragePath()) && Objects.equals( getExternalOutputPayloadStoragePath(), task.getExternalOutputPayloadStoragePath()) && Objects.equals(getIsolationGroupId(), task.getIsolationGroupId()) && Objects.equals(getExecutionNameSpace(), task.getExecutionNameSpace()); } @Override public int hashCode() { return Objects.hash( getTaskType(), getStatus(), getInputData(), getReferenceTaskName(), getWorkflowPriority(), getRetryCount(), getSeq(), getCorrelationId(), getPollCount(), getTaskDefName(), getScheduledTime(), getStartTime(), getEndTime(), getUpdateTime(), getStartDelayInSeconds(), getRetriedTaskId(), isRetried(), isExecuted(), isCallbackFromWorker(), getResponseTimeoutSeconds(), getWorkflowInstanceId(), getWorkflowType(), getTaskId(), getReasonForIncompletion(), getCallbackAfterSeconds(), getWorkerId(), getOutputData(), getWorkflowTask(), getDomain(), getInputMessage(), getOutputMessage(), getRateLimitPerFrequency(), getRateLimitFrequencyInSeconds(), getExternalInputPayloadStoragePath(), getExternalOutputPayloadStoragePath(), getIsolationGroupId(), getExecutionNameSpace()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/TaskDef.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import javax.validation.Valid; import javax.validation.constraints.Email; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.constraints.OwnerEmailMandatoryConstraint; import com.netflix.conductor.common.constraints.TaskTimeoutConstraint; import com.netflix.conductor.common.metadata.BaseDef; @ProtoMessage @TaskTimeoutConstraint @Valid public class TaskDef extends BaseDef { @ProtoEnum public enum TimeoutPolicy { RETRY, TIME_OUT_WF, ALERT_ONLY } @ProtoEnum public enum RetryLogic { FIXED, EXPONENTIAL_BACKOFF, LINEAR_BACKOFF } public static final int ONE_HOUR = 60 * 60; /** Unique name identifying the task. The name is unique across */ @NotEmpty(message = "TaskDef name cannot be null or empty") @ProtoField(id = 1) private String name; @ProtoField(id = 2) private String description; @ProtoField(id = 3) @Min(value = 0, message = "TaskDef retryCount: {value} must be >= 0") private int retryCount = 3; // Default @ProtoField(id = 4) @NotNull private long timeoutSeconds; @ProtoField(id = 5) private List inputKeys = new ArrayList<>(); @ProtoField(id = 6) private List outputKeys = new ArrayList<>(); @ProtoField(id = 7) private TimeoutPolicy timeoutPolicy = TimeoutPolicy.TIME_OUT_WF; @ProtoField(id = 8) private RetryLogic retryLogic = RetryLogic.FIXED; @ProtoField(id = 9) private int retryDelaySeconds = 60; @ProtoField(id = 10) @Min( value = 1, message = "TaskDef responseTimeoutSeconds: ${validatedValue} should be minimum {value} second") private long responseTimeoutSeconds = ONE_HOUR; @ProtoField(id = 11) private Integer concurrentExecLimit; @ProtoField(id = 12) private Map inputTemplate = new HashMap<>(); // This field is deprecated, do not use id 13. // @ProtoField(id = 13) // private Integer rateLimitPerSecond; @ProtoField(id = 14) private Integer rateLimitPerFrequency; @ProtoField(id = 15) private Integer rateLimitFrequencyInSeconds; @ProtoField(id = 16) private String isolationGroupId; @ProtoField(id = 17) private String executionNameSpace; @ProtoField(id = 18) @OwnerEmailMandatoryConstraint @Email(message = "ownerEmail should be valid email address") private String ownerEmail; @ProtoField(id = 19) @Min(value = 0, message = "TaskDef pollTimeoutSeconds: {value} must be >= 0") private Integer pollTimeoutSeconds; @ProtoField(id = 20) @Min(value = 1, message = "Backoff scale factor. Applicable for LINEAR_BACKOFF") private Integer backoffScaleFactor = 1; public TaskDef() {} public TaskDef(String name) { this.name = name; } public TaskDef(String name, String description) { this.name = name; this.description = description; } public TaskDef(String name, String description, int retryCount, long timeoutSeconds) { this.name = name; this.description = description; this.retryCount = retryCount; this.timeoutSeconds = timeoutSeconds; } public TaskDef( String name, String description, String ownerEmail, int retryCount, long timeoutSeconds, long responseTimeoutSeconds) { this.name = name; this.description = description; this.ownerEmail = ownerEmail; this.retryCount = retryCount; this.timeoutSeconds = timeoutSeconds; this.responseTimeoutSeconds = responseTimeoutSeconds; } /** * @return the name */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the description */ public String getDescription() { return description; } /** * @param description the description to set */ public void setDescription(String description) { this.description = description; } /** * @return the retryCount */ public int getRetryCount() { return retryCount; } /** * @param retryCount the retryCount to set */ public void setRetryCount(int retryCount) { this.retryCount = retryCount; } /** * @return the timeoutSeconds */ public long getTimeoutSeconds() { return timeoutSeconds; } /** * @param timeoutSeconds the timeoutSeconds to set */ public void setTimeoutSeconds(long timeoutSeconds) { this.timeoutSeconds = timeoutSeconds; } /** * @return Returns the input keys */ public List getInputKeys() { return inputKeys; } /** * @param inputKeys Set of keys that the task accepts in the input map */ public void setInputKeys(List inputKeys) { this.inputKeys = inputKeys; } /** * @return Returns the output keys for the task when executed */ public List getOutputKeys() { return outputKeys; } /** * @param outputKeys Sets the output keys */ public void setOutputKeys(List outputKeys) { this.outputKeys = outputKeys; } /** * @return the timeoutPolicy */ public TimeoutPolicy getTimeoutPolicy() { return timeoutPolicy; } /** * @param timeoutPolicy the timeoutPolicy to set */ public void setTimeoutPolicy(TimeoutPolicy timeoutPolicy) { this.timeoutPolicy = timeoutPolicy; } /** * @return the retryLogic */ public RetryLogic getRetryLogic() { return retryLogic; } /** * @param retryLogic the retryLogic to set */ public void setRetryLogic(RetryLogic retryLogic) { this.retryLogic = retryLogic; } /** * @return the retryDelaySeconds */ public int getRetryDelaySeconds() { return retryDelaySeconds; } /** * @return the timeout for task to send response. After this timeout, the task will be re-queued */ public long getResponseTimeoutSeconds() { return responseTimeoutSeconds; } /** * @param responseTimeoutSeconds - timeout for task to send response. After this timeout, the * task will be re-queued */ public void setResponseTimeoutSeconds(long responseTimeoutSeconds) { this.responseTimeoutSeconds = responseTimeoutSeconds; } /** * @param retryDelaySeconds the retryDelaySeconds to set */ public void setRetryDelaySeconds(int retryDelaySeconds) { this.retryDelaySeconds = retryDelaySeconds; } /** * @return the inputTemplate */ public Map getInputTemplate() { return inputTemplate; } /** * @return rateLimitPerFrequency The max number of tasks that will be allowed to be executed per * rateLimitFrequencyInSeconds. */ public Integer getRateLimitPerFrequency() { return rateLimitPerFrequency == null ? 0 : rateLimitPerFrequency; } /** * @param rateLimitPerFrequency The max number of tasks that will be allowed to be executed per * rateLimitFrequencyInSeconds. Setting the value to 0 removes the rate limit */ public void setRateLimitPerFrequency(Integer rateLimitPerFrequency) { this.rateLimitPerFrequency = rateLimitPerFrequency; } /** * @return rateLimitFrequencyInSeconds: The time bucket that is used to rate limit tasks based * on {@link #getRateLimitPerFrequency()} If null or not set, then defaults to 1 second */ public Integer getRateLimitFrequencyInSeconds() { return rateLimitFrequencyInSeconds == null ? 1 : rateLimitFrequencyInSeconds; } /** * @param rateLimitFrequencyInSeconds: The time window/bucket for which the rate limit needs to * be applied. This will only have affect if {@link #getRateLimitPerFrequency()} is greater * than zero */ public void setRateLimitFrequencyInSeconds(Integer rateLimitFrequencyInSeconds) { this.rateLimitFrequencyInSeconds = rateLimitFrequencyInSeconds; } /** * @param concurrentExecLimit Limit of number of concurrent task that can be IN_PROGRESS at a * given time. Seting the value to 0 removes the limit. */ public void setConcurrentExecLimit(Integer concurrentExecLimit) { this.concurrentExecLimit = concurrentExecLimit; } /** * @return Limit of number of concurrent task that can be IN_PROGRESS at a given time */ public Integer getConcurrentExecLimit() { return concurrentExecLimit; } /** * @return concurrency limit */ public int concurrencyLimit() { return concurrentExecLimit == null ? 0 : concurrentExecLimit; } /** * @param inputTemplate the inputTemplate to set */ public void setInputTemplate(Map inputTemplate) { this.inputTemplate = inputTemplate; } public String getIsolationGroupId() { return isolationGroupId; } public void setIsolationGroupId(String isolationGroupId) { this.isolationGroupId = isolationGroupId; } public String getExecutionNameSpace() { return executionNameSpace; } public void setExecutionNameSpace(String executionNameSpace) { this.executionNameSpace = executionNameSpace; } /** * @return the email of the owner of this task definition */ public String getOwnerEmail() { return ownerEmail; } /** * @param ownerEmail the owner email to set */ public void setOwnerEmail(String ownerEmail) { this.ownerEmail = ownerEmail; } /** * @param pollTimeoutSeconds the poll timeout to set */ public void setPollTimeoutSeconds(Integer pollTimeoutSeconds) { this.pollTimeoutSeconds = pollTimeoutSeconds; } /** * @return the poll timeout of this task definition */ public Integer getPollTimeoutSeconds() { return pollTimeoutSeconds; } /** * @param backoffScaleFactor the backoff rate to set */ public void setBackoffScaleFactor(Integer backoffScaleFactor) { this.backoffScaleFactor = backoffScaleFactor; } /** * @return the backoff rate of this task definition */ public Integer getBackoffScaleFactor() { return backoffScaleFactor; } @Override public String toString() { return name; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } TaskDef taskDef = (TaskDef) o; return getRetryCount() == taskDef.getRetryCount() && getTimeoutSeconds() == taskDef.getTimeoutSeconds() && getRetryDelaySeconds() == taskDef.getRetryDelaySeconds() && getBackoffScaleFactor() == taskDef.getBackoffScaleFactor() && getResponseTimeoutSeconds() == taskDef.getResponseTimeoutSeconds() && Objects.equals(getName(), taskDef.getName()) && Objects.equals(getDescription(), taskDef.getDescription()) && Objects.equals(getInputKeys(), taskDef.getInputKeys()) && Objects.equals(getOutputKeys(), taskDef.getOutputKeys()) && getTimeoutPolicy() == taskDef.getTimeoutPolicy() && getRetryLogic() == taskDef.getRetryLogic() && Objects.equals(getConcurrentExecLimit(), taskDef.getConcurrentExecLimit()) && Objects.equals(getRateLimitPerFrequency(), taskDef.getRateLimitPerFrequency()) && Objects.equals(getInputTemplate(), taskDef.getInputTemplate()) && Objects.equals(getIsolationGroupId(), taskDef.getIsolationGroupId()) && Objects.equals(getExecutionNameSpace(), taskDef.getExecutionNameSpace()) && Objects.equals(getOwnerEmail(), taskDef.getOwnerEmail()); } @Override public int hashCode() { return Objects.hash( getName(), getDescription(), getRetryCount(), getTimeoutSeconds(), getInputKeys(), getOutputKeys(), getTimeoutPolicy(), getRetryLogic(), getRetryDelaySeconds(), getBackoffScaleFactor(), getResponseTimeoutSeconds(), getConcurrentExecLimit(), getRateLimitPerFrequency(), getInputTemplate(), getIsolationGroupId(), getExecutionNameSpace(), getOwnerEmail()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/TaskExecLog.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.Objects; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; /** Model that represents the task's execution log. */ @ProtoMessage public class TaskExecLog { @ProtoField(id = 1) private String log; @ProtoField(id = 2) private String taskId; @ProtoField(id = 3) private long createdTime; public TaskExecLog() {} public TaskExecLog(String log) { this.log = log; this.createdTime = System.currentTimeMillis(); } /** * @return Task Exec Log */ public String getLog() { return log; } /** * @param log The Log */ public void setLog(String log) { this.log = log; } /** * @return the taskId */ public String getTaskId() { return taskId; } /** * @param taskId the taskId to set */ public void setTaskId(String taskId) { this.taskId = taskId; } /** * @return the createdTime */ public long getCreatedTime() { return createdTime; } /** * @param createdTime the createdTime to set */ public void setCreatedTime(long createdTime) { this.createdTime = createdTime; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } TaskExecLog that = (TaskExecLog) o; return createdTime == that.createdTime && Objects.equals(log, that.log) && Objects.equals(taskId, that.taskId); } @Override public int hashCode() { return Objects.hash(log, taskId, createdTime); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/TaskResult.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import javax.validation.constraints.NotEmpty; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.google.protobuf.Any; import io.swagger.v3.oas.annotations.Hidden; /** Result of the task execution. */ @ProtoMessage public class TaskResult { @ProtoEnum public enum Status { IN_PROGRESS, FAILED, FAILED_WITH_TERMINAL_ERROR, COMPLETED } @NotEmpty(message = "Workflow Id cannot be null or empty") @ProtoField(id = 1) private String workflowInstanceId; @NotEmpty(message = "Task ID cannot be null or empty") @ProtoField(id = 2) private String taskId; @ProtoField(id = 3) private String reasonForIncompletion; @ProtoField(id = 4) private long callbackAfterSeconds; @ProtoField(id = 5) private String workerId; @ProtoField(id = 6) private Status status; @ProtoField(id = 7) private Map outputData = new HashMap<>(); @ProtoField(id = 8) @Hidden private Any outputMessage; private List logs = new CopyOnWriteArrayList<>(); private String externalOutputPayloadStoragePath; private String subWorkflowId; private boolean extendLease; public TaskResult(Task task) { this.workflowInstanceId = task.getWorkflowInstanceId(); this.taskId = task.getTaskId(); this.reasonForIncompletion = task.getReasonForIncompletion(); this.callbackAfterSeconds = task.getCallbackAfterSeconds(); this.workerId = task.getWorkerId(); this.outputData = task.getOutputData(); this.externalOutputPayloadStoragePath = task.getExternalOutputPayloadStoragePath(); this.subWorkflowId = task.getSubWorkflowId(); switch (task.getStatus()) { case CANCELED: case COMPLETED_WITH_ERRORS: case TIMED_OUT: case SKIPPED: this.status = Status.FAILED; break; case SCHEDULED: this.status = Status.IN_PROGRESS; break; default: this.status = Status.valueOf(task.getStatus().name()); break; } } public TaskResult() {} /** * @return Workflow instance id for which the task result is produced */ public String getWorkflowInstanceId() { return workflowInstanceId; } public void setWorkflowInstanceId(String workflowInstanceId) { this.workflowInstanceId = workflowInstanceId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getReasonForIncompletion() { return reasonForIncompletion; } public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = StringUtils.substring(reasonForIncompletion, 0, 500); } public long getCallbackAfterSeconds() { return callbackAfterSeconds; } /** * When set to non-zero values, the task remains in the queue for the specified seconds before * sent back to the worker when polled. Useful for the long running task, where the task is * updated as IN_PROGRESS and should not be polled out of the queue for a specified amount of * time. (delayed queue implementation) * * @param callbackAfterSeconds Amount of time in seconds the task should be held in the queue * before giving it to a polling worker. */ public void setCallbackAfterSeconds(long callbackAfterSeconds) { this.callbackAfterSeconds = callbackAfterSeconds; } public String getWorkerId() { return workerId; } /** * @param workerId a free form string identifying the worker host. Could be hostname, IP Address * or any other meaningful identifier that can help identify the host/process which executed * the task, in case of troubleshooting. */ public void setWorkerId(String workerId) { this.workerId = workerId; } /** * @return the status */ public Status getStatus() { return status; } /** * @param status Status of the task *

IN_PROGRESS: Use this for long running tasks, indicating the task is still in * progress and should be checked again at a later time. e.g. the worker checks the status * of the job in the DB, while the job is being executed by another process. *

FAILED, FAILED_WITH_TERMINAL_ERROR, COMPLETED: Terminal statuses for the task. * Use FAILED_WITH_TERMINAL_ERROR when you do not want the task to be retried. * @see #setCallbackAfterSeconds(long) */ public void setStatus(Status status) { this.status = status; } public Map getOutputData() { return outputData; } /** * @param outputData output data to be set for the task execution result */ public void setOutputData(Map outputData) { this.outputData = outputData; } /** * Adds output * * @param key output field * @param value value * @return current instance */ public TaskResult addOutputData(String key, Object value) { this.outputData.put(key, value); return this; } public Any getOutputMessage() { return outputMessage; } public void setOutputMessage(Any outputMessage) { this.outputMessage = outputMessage; } /** * @return Task execution logs */ public List getLogs() { return logs; } /** * @param logs Task execution logs */ public void setLogs(List logs) { this.logs = logs; } /** * @param log Log line to be added * @return Instance of TaskResult */ public TaskResult log(String log) { this.logs.add(new TaskExecLog(log)); return this; } /** * @return the path where the task output is stored in external storage */ public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } /** * @param externalOutputPayloadStoragePath path in the external storage where the task output is * stored */ public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } public String getSubWorkflowId() { return subWorkflowId; } public void setSubWorkflowId(String subWorkflowId) { this.subWorkflowId = subWorkflowId; } public boolean isExtendLease() { return extendLease; } public void setExtendLease(boolean extendLease) { this.extendLease = extendLease; } @Override public String toString() { return "TaskResult{" + "workflowInstanceId='" + workflowInstanceId + '\'' + ", taskId='" + taskId + '\'' + ", reasonForIncompletion='" + reasonForIncompletion + '\'' + ", callbackAfterSeconds=" + callbackAfterSeconds + ", workerId='" + workerId + '\'' + ", status=" + status + ", outputData=" + outputData + ", outputMessage=" + outputMessage + ", logs=" + logs + ", externalOutputPayloadStoragePath='" + externalOutputPayloadStoragePath + '\'' + ", subWorkflowId='" + subWorkflowId + '\'' + ", extendLease='" + extendLease + '\'' + '}'; } public static TaskResult complete() { return newTaskResult(Status.COMPLETED); } public static TaskResult failed() { return newTaskResult(Status.FAILED); } public static TaskResult failed(String failureReason) { TaskResult result = newTaskResult(Status.FAILED); result.setReasonForIncompletion(failureReason); return result; } public static TaskResult inProgress() { return newTaskResult(Status.IN_PROGRESS); } public static TaskResult newTaskResult(Status status) { TaskResult result = new TaskResult(); result.setStatus(status); return result; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/tasks/TaskType.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.tasks; import java.util.HashSet; import java.util.Set; import com.netflix.conductor.annotations.protogen.ProtoEnum; @ProtoEnum public enum TaskType { SIMPLE, DYNAMIC, FORK_JOIN, FORK_JOIN_DYNAMIC, DECISION, SWITCH, JOIN, DO_WHILE, SUB_WORKFLOW, START_WORKFLOW, EVENT, WAIT, HUMAN, USER_DEFINED, HTTP, LAMBDA, INLINE, EXCLUSIVE_JOIN, TERMINATE, KAFKA_PUBLISH, JSON_JQ_TRANSFORM, SET_VARIABLE, NOOP; /** * TaskType constants representing each of the possible enumeration values. Motivation: to not * have any hardcoded/inline strings used in the code. */ public static final String TASK_TYPE_DECISION = "DECISION"; public static final String TASK_TYPE_SWITCH = "SWITCH"; public static final String TASK_TYPE_DYNAMIC = "DYNAMIC"; public static final String TASK_TYPE_JOIN = "JOIN"; public static final String TASK_TYPE_DO_WHILE = "DO_WHILE"; public static final String TASK_TYPE_FORK_JOIN_DYNAMIC = "FORK_JOIN_DYNAMIC"; public static final String TASK_TYPE_EVENT = "EVENT"; public static final String TASK_TYPE_WAIT = "WAIT"; public static final String TASK_TYPE_HUMAN = "HUMAN"; public static final String TASK_TYPE_SUB_WORKFLOW = "SUB_WORKFLOW"; public static final String TASK_TYPE_START_WORKFLOW = "START_WORKFLOW"; public static final String TASK_TYPE_FORK_JOIN = "FORK_JOIN"; public static final String TASK_TYPE_SIMPLE = "SIMPLE"; public static final String TASK_TYPE_HTTP = "HTTP"; public static final String TASK_TYPE_LAMBDA = "LAMBDA"; public static final String TASK_TYPE_INLINE = "INLINE"; public static final String TASK_TYPE_EXCLUSIVE_JOIN = "EXCLUSIVE_JOIN"; public static final String TASK_TYPE_TERMINATE = "TERMINATE"; public static final String TASK_TYPE_KAFKA_PUBLISH = "KAFKA_PUBLISH"; public static final String TASK_TYPE_JSON_JQ_TRANSFORM = "JSON_JQ_TRANSFORM"; public static final String TASK_TYPE_SET_VARIABLE = "SET_VARIABLE"; public static final String TASK_TYPE_FORK = "FORK"; public static final String TASK_TYPE_NOOP = "NOOP"; private static final Set BUILT_IN_TASKS = new HashSet<>(); static { BUILT_IN_TASKS.add(TASK_TYPE_DECISION); BUILT_IN_TASKS.add(TASK_TYPE_SWITCH); BUILT_IN_TASKS.add(TASK_TYPE_FORK); BUILT_IN_TASKS.add(TASK_TYPE_JOIN); BUILT_IN_TASKS.add(TASK_TYPE_EXCLUSIVE_JOIN); BUILT_IN_TASKS.add(TASK_TYPE_DO_WHILE); } /** * Converts a task type string to {@link TaskType}. For an unknown string, the value is * defaulted to {@link TaskType#USER_DEFINED}. * *

NOTE: Use {@link Enum#valueOf(Class, String)} if the default of USER_DEFINED is not * necessary. * * @param taskType The task type string. * @return The {@link TaskType} enum. */ public static TaskType of(String taskType) { try { return TaskType.valueOf(taskType); } catch (IllegalArgumentException iae) { return TaskType.USER_DEFINED; } } public static boolean isBuiltIn(String taskType) { return BUILT_IN_TASKS.contains(taskType); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/DynamicForkJoinTask.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.HashMap; import java.util.Map; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.tasks.TaskType; @ProtoMessage public class DynamicForkJoinTask { @ProtoField(id = 1) private String taskName; @ProtoField(id = 2) private String workflowName; @ProtoField(id = 3) private String referenceName; @ProtoField(id = 4) private Map input = new HashMap<>(); @ProtoField(id = 5) private String type = TaskType.SIMPLE.name(); public DynamicForkJoinTask() {} public DynamicForkJoinTask( String taskName, String workflowName, String referenceName, Map input) { super(); this.taskName = taskName; this.workflowName = workflowName; this.referenceName = referenceName; this.input = input; } public DynamicForkJoinTask( String taskName, String workflowName, String referenceName, String type, Map input) { super(); this.taskName = taskName; this.workflowName = workflowName; this.referenceName = referenceName; this.input = input; this.type = type; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public String getWorkflowName() { return workflowName; } public void setWorkflowName(String workflowName) { this.workflowName = workflowName; } public String getReferenceName() { return referenceName; } public void setReferenceName(String referenceName) { this.referenceName = referenceName; } public Map getInput() { return input; } public void setInput(Map input) { this.input = input; } public String getType() { return type; } public void setType(String type) { this.type = type; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/DynamicForkJoinTaskList.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.ArrayList; import java.util.List; import java.util.Map; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; @ProtoMessage public class DynamicForkJoinTaskList { @ProtoField(id = 1) private List dynamicTasks = new ArrayList<>(); public void add( String taskName, String workflowName, String referenceName, Map input) { dynamicTasks.add(new DynamicForkJoinTask(taskName, workflowName, referenceName, input)); } public void add(DynamicForkJoinTask dtask) { dynamicTasks.add(dtask); } public List getDynamicTasks() { return dynamicTasks; } public void setDynamicTasks(List dynamicTasks) { this.dynamicTasks = dynamicTasks; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/RerunWorkflowRequest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.Map; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; @ProtoMessage public class RerunWorkflowRequest { @ProtoField(id = 1) private String reRunFromWorkflowId; @ProtoField(id = 2) private Map workflowInput; @ProtoField(id = 3) private String reRunFromTaskId; @ProtoField(id = 4) private Map taskInput; @ProtoField(id = 5) private String correlationId; public String getReRunFromWorkflowId() { return reRunFromWorkflowId; } public void setReRunFromWorkflowId(String reRunFromWorkflowId) { this.reRunFromWorkflowId = reRunFromWorkflowId; } public Map getWorkflowInput() { return workflowInput; } public void setWorkflowInput(Map workflowInput) { this.workflowInput = workflowInput; } public String getReRunFromTaskId() { return reRunFromTaskId; } public void setReRunFromTaskId(String reRunFromTaskId) { this.reRunFromTaskId = reRunFromTaskId; } public Map getTaskInput() { return taskInput; } public void setTaskInput(Map taskInput) { this.taskInput = taskInput; } public String getCorrelationId() { return correlationId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/SkipTaskRequest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.Map; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.google.protobuf.Any; import io.swagger.v3.oas.annotations.Hidden; @ProtoMessage(toProto = false) public class SkipTaskRequest { @ProtoField(id = 1) private Map taskInput; @ProtoField(id = 2) private Map taskOutput; @ProtoField(id = 3) @Hidden private Any taskInputMessage; @ProtoField(id = 4) @Hidden private Any taskOutputMessage; public Map getTaskInput() { return taskInput; } public void setTaskInput(Map taskInput) { this.taskInput = taskInput; } public Map getTaskOutput() { return taskOutput; } public void setTaskOutput(Map taskOutput) { this.taskOutput = taskOutput; } public Any getTaskInputMessage() { return taskInputMessage; } public void setTaskInputMessage(Any taskInputMessage) { this.taskInputMessage = taskInputMessage; } public Any getTaskOutputMessage() { return taskOutputMessage; } public void setTaskOutputMessage(Any taskOutputMessage) { this.taskOutputMessage = taskOutputMessage; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/StartWorkflowRequest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.HashMap; import java.util.Map; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; @ProtoMessage public class StartWorkflowRequest { @ProtoField(id = 1) @NotNull(message = "Workflow name cannot be null or empty") private String name; @ProtoField(id = 2) private Integer version; @ProtoField(id = 3) private String correlationId; @ProtoField(id = 4) private Map input = new HashMap<>(); @ProtoField(id = 5) private Map taskToDomain = new HashMap<>(); @ProtoField(id = 6) @Valid private WorkflowDef workflowDef; @ProtoField(id = 7) private String externalInputPayloadStoragePath; @ProtoField(id = 8) @Min(value = 0, message = "priority: ${validatedValue} should be minimum {value}") @Max(value = 99, message = "priority: ${validatedValue} should be maximum {value}") private Integer priority = 0; public String getName() { return name; } public void setName(String name) { this.name = name; } public StartWorkflowRequest withName(String name) { this.name = name; return this; } public Integer getVersion() { return version; } public void setVersion(Integer version) { this.version = version; } public StartWorkflowRequest withVersion(Integer version) { this.version = version; return this; } public String getCorrelationId() { return correlationId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public StartWorkflowRequest withCorrelationId(String correlationId) { this.correlationId = correlationId; return this; } public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } public StartWorkflowRequest withExternalInputPayloadStoragePath( String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; return this; } public Integer getPriority() { return priority; } public void setPriority(Integer priority) { this.priority = priority; } public StartWorkflowRequest withPriority(Integer priority) { this.priority = priority; return this; } public Map getInput() { return input; } public void setInput(Map input) { this.input = input; } public StartWorkflowRequest withInput(Map input) { this.input = input; return this; } public Map getTaskToDomain() { return taskToDomain; } public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } public StartWorkflowRequest withTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; return this; } public WorkflowDef getWorkflowDef() { return workflowDef; } public void setWorkflowDef(WorkflowDef workflowDef) { this.workflowDef = workflowDef; } public StartWorkflowRequest withWorkflowDef(WorkflowDef workflowDef) { this.workflowDef = workflowDef; return this; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/SubWorkflowParams.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.Map; import java.util.Objects; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonSetter; @ProtoMessage public class SubWorkflowParams { @ProtoField(id = 1) @NotNull(message = "SubWorkflowParams name cannot be null") @NotEmpty(message = "SubWorkflowParams name cannot be empty") private String name; @ProtoField(id = 2) private Integer version; @ProtoField(id = 3) private Map taskToDomain; // workaround as WorkflowDef cannot directly be used due to cyclic dependency issue in protobuf // imports @ProtoField(id = 4) private Object workflowDefinition; /** * @return the name */ public String getName() { if (workflowDefinition != null) { return getWorkflowDef().getName(); } else { return name; } } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the version */ public Integer getVersion() { if (workflowDefinition != null) { return getWorkflowDef().getVersion(); } else { return version; } } /** * @param version the version to set */ public void setVersion(Integer version) { this.version = version; } /** * @return the taskToDomain */ public Map getTaskToDomain() { return taskToDomain; } /** * @param taskToDomain the taskToDomain to set */ public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } /** * @return the workflowDefinition as an Object */ public Object getWorkflowDefinition() { return workflowDefinition; } /** * @return the workflowDefinition as a WorkflowDef */ @JsonGetter("workflowDefinition") public WorkflowDef getWorkflowDef() { return (WorkflowDef) workflowDefinition; } /** * @param workflowDef the workflowDefinition to set */ public void setWorkflowDefinition(Object workflowDef) { if (!(workflowDef == null || workflowDef instanceof WorkflowDef)) { throw new IllegalArgumentException( "workflowDefinition must be either null or WorkflowDef"); } this.workflowDefinition = workflowDef; } /** * @param workflowDef the workflowDefinition to set */ @JsonSetter("workflowDefinition") public void setWorkflowDef(WorkflowDef workflowDef) { this.workflowDefinition = workflowDef; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SubWorkflowParams that = (SubWorkflowParams) o; return Objects.equals(getName(), that.getName()) && Objects.equals(getVersion(), that.getVersion()) && Objects.equals(getTaskToDomain(), that.getTaskToDomain()) && Objects.equals(getWorkflowDefinition(), that.getWorkflowDefinition()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/WorkflowDef.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import javax.validation.Valid; import javax.validation.constraints.Email; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.constraints.NoSemiColonConstraint; import com.netflix.conductor.common.constraints.OwnerEmailMandatoryConstraint; import com.netflix.conductor.common.constraints.TaskReferenceNameUniqueConstraint; import com.netflix.conductor.common.metadata.BaseDef; import com.netflix.conductor.common.metadata.tasks.TaskType; @ProtoMessage @TaskReferenceNameUniqueConstraint public class WorkflowDef extends BaseDef { @ProtoEnum public enum TimeoutPolicy { TIME_OUT_WF, ALERT_ONLY } @NotEmpty(message = "WorkflowDef name cannot be null or empty") @ProtoField(id = 1) @NoSemiColonConstraint( message = "Workflow name cannot contain the following set of characters: ':'") private String name; @ProtoField(id = 2) private String description; @ProtoField(id = 3) private int version = 1; @ProtoField(id = 4) @NotNull @NotEmpty(message = "WorkflowTask list cannot be empty") private List<@Valid WorkflowTask> tasks = new LinkedList<>(); @ProtoField(id = 5) private List inputParameters = new LinkedList<>(); @ProtoField(id = 6) private Map outputParameters = new HashMap<>(); @ProtoField(id = 7) private String failureWorkflow; @ProtoField(id = 8) @Min(value = 2, message = "workflowDef schemaVersion: {value} is only supported") @Max(value = 2, message = "workflowDef schemaVersion: {value} is only supported") private int schemaVersion = 2; // By default, a workflow is restartable @ProtoField(id = 9) private boolean restartable = true; @ProtoField(id = 10) private boolean workflowStatusListenerEnabled = false; @ProtoField(id = 11) @OwnerEmailMandatoryConstraint @Email(message = "ownerEmail should be valid email address") private String ownerEmail; @ProtoField(id = 12) private TimeoutPolicy timeoutPolicy = TimeoutPolicy.ALERT_ONLY; @ProtoField(id = 13) @NotNull private long timeoutSeconds; @ProtoField(id = 14) private Map variables = new HashMap<>(); @ProtoField(id = 15) private Map inputTemplate = new HashMap<>(); /** * @return the name */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the description */ public String getDescription() { return description; } /** * @param description the description to set */ public void setDescription(String description) { this.description = description; } /** * @return the tasks */ public List getTasks() { return tasks; } /** * @param tasks the tasks to set */ public void setTasks(List<@Valid WorkflowTask> tasks) { this.tasks = tasks; } /** * @return the inputParameters */ public List getInputParameters() { return inputParameters; } /** * @param inputParameters the inputParameters to set */ public void setInputParameters(List inputParameters) { this.inputParameters = inputParameters; } /** * @return the outputParameters */ public Map getOutputParameters() { return outputParameters; } /** * @param outputParameters the outputParameters to set */ public void setOutputParameters(Map outputParameters) { this.outputParameters = outputParameters; } /** * @return the version */ public int getVersion() { return version; } /** * @return the failureWorkflow */ public String getFailureWorkflow() { return failureWorkflow; } /** * @param failureWorkflow the failureWorkflow to set */ public void setFailureWorkflow(String failureWorkflow) { this.failureWorkflow = failureWorkflow; } /** * @param version the version to set */ public void setVersion(int version) { this.version = version; } /** * This method determines if the workflow is restartable or not * * @return true: if the workflow is restartable false: if the workflow is non restartable */ public boolean isRestartable() { return restartable; } /** * This method is called only when the workflow definition is created * * @param restartable true: if the workflow is restartable false: if the workflow is non * restartable */ public void setRestartable(boolean restartable) { this.restartable = restartable; } /** * @return the schemaVersion */ public int getSchemaVersion() { return schemaVersion; } /** * @param schemaVersion the schemaVersion to set */ public void setSchemaVersion(int schemaVersion) { this.schemaVersion = schemaVersion; } /** * @return true is workflow listener will be invoked when workflow gets into a terminal state */ public boolean isWorkflowStatusListenerEnabled() { return workflowStatusListenerEnabled; } /** * Specify if workflow listener is enabled to invoke a callback for completed or terminated * workflows * * @param workflowStatusListenerEnabled */ public void setWorkflowStatusListenerEnabled(boolean workflowStatusListenerEnabled) { this.workflowStatusListenerEnabled = workflowStatusListenerEnabled; } /** * @return the email of the owner of this workflow definition */ public String getOwnerEmail() { return ownerEmail; } /** * @param ownerEmail the owner email to set */ public void setOwnerEmail(String ownerEmail) { this.ownerEmail = ownerEmail; } /** * @return the timeoutPolicy */ public TimeoutPolicy getTimeoutPolicy() { return timeoutPolicy; } /** * @param timeoutPolicy the timeoutPolicy to set */ public void setTimeoutPolicy(TimeoutPolicy timeoutPolicy) { this.timeoutPolicy = timeoutPolicy; } /** * @return the time after which a workflow is deemed to have timed out */ public long getTimeoutSeconds() { return timeoutSeconds; } /** * @param timeoutSeconds the timeout in seconds to set */ public void setTimeoutSeconds(long timeoutSeconds) { this.timeoutSeconds = timeoutSeconds; } /** * @return the global workflow variables */ public Map getVariables() { return variables; } /** * @param variables the set of global workflow variables to set */ public void setVariables(Map variables) { this.variables = variables; } public Map getInputTemplate() { return inputTemplate; } public void setInputTemplate(Map inputTemplate) { this.inputTemplate = inputTemplate; } public String key() { return getKey(name, version); } public static String getKey(String name, int version) { return name + "." + version; } public boolean containsType(String taskType) { return collectTasks().stream().anyMatch(t -> t.getType().equals(taskType)); } public WorkflowTask getNextTask(String taskReferenceName) { WorkflowTask workflowTask = getTaskByRefName(taskReferenceName); if (workflowTask != null && TaskType.TERMINATE.name().equals(workflowTask.getType())) { return null; } Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { WorkflowTask task = iterator.next(); if (task.getTaskReferenceName().equals(taskReferenceName)) { // If taskReferenceName matches, break out break; } WorkflowTask nextTask = task.next(taskReferenceName, null); if (nextTask != null) { return nextTask; } else if (TaskType.DO_WHILE.name().equals(task.getType()) && !task.getTaskReferenceName().equals(taskReferenceName) && task.has(taskReferenceName)) { // If the task is child of Loop Task and at last position, return null. return null; } if (task.has(taskReferenceName)) { break; } } if (iterator.hasNext()) { return iterator.next(); } return null; } public WorkflowTask getTaskByRefName(String taskReferenceName) { return collectTasks().stream() .filter( workflowTask -> workflowTask.getTaskReferenceName().equals(taskReferenceName)) .findFirst() .orElse(null); } public List collectTasks() { List tasks = new LinkedList<>(); for (WorkflowTask workflowTask : this.tasks) { tasks.addAll(workflowTask.collectTasks()); } return tasks; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } WorkflowDef that = (WorkflowDef) o; return getVersion() == that.getVersion() && getSchemaVersion() == that.getSchemaVersion() && Objects.equals(getName(), that.getName()) && Objects.equals(getDescription(), that.getDescription()) && Objects.equals(getTasks(), that.getTasks()) && Objects.equals(getInputParameters(), that.getInputParameters()) && Objects.equals(getOutputParameters(), that.getOutputParameters()) && Objects.equals(getFailureWorkflow(), that.getFailureWorkflow()) && Objects.equals(getOwnerEmail(), that.getOwnerEmail()) && Objects.equals(getTimeoutSeconds(), that.getTimeoutSeconds()); } @Override public int hashCode() { return Objects.hash( getName(), getDescription(), getVersion(), getTasks(), getInputParameters(), getOutputParameters(), getFailureWorkflow(), getSchemaVersion(), getOwnerEmail(), getTimeoutSeconds()); } @Override public String toString() { return "WorkflowDef{" + "name='" + name + '\'' + ", description='" + description + '\'' + ", version=" + version + ", tasks=" + tasks + ", inputParameters=" + inputParameters + ", outputParameters=" + outputParameters + ", failureWorkflow='" + failureWorkflow + '\'' + ", schemaVersion=" + schemaVersion + ", restartable=" + restartable + ", workflowStatusListenerEnabled=" + workflowStatusListenerEnabled + ", timeoutSeconds=" + timeoutSeconds + '}'; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/WorkflowDefSummary.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.Objects; import javax.validation.constraints.NotEmpty; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.constraints.NoSemiColonConstraint; @ProtoMessage public class WorkflowDefSummary implements Comparable { @NotEmpty(message = "WorkflowDef name cannot be null or empty") @ProtoField(id = 1) @NoSemiColonConstraint( message = "Workflow name cannot contain the following set of characters: ':'") private String name; @ProtoField(id = 2) private int version = 1; @ProtoField(id = 3) private Long createTime; /** * @return the version */ public int getVersion() { return version; } /** * @return the workflow name */ public String getName() { return name; } /** * @return the createTime */ public Long getCreateTime() { return createTime; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } WorkflowDefSummary that = (WorkflowDefSummary) o; return getVersion() == that.getVersion() && Objects.equals(getName(), that.getName()); } public void setName(String name) { this.name = name; } public void setVersion(int version) { this.version = version; } public void setCreateTime(Long createTime) { this.createTime = createTime; } @Override public int hashCode() { return Objects.hash(getName(), getVersion()); } @Override public String toString() { return "WorkflowDef{name='" + name + ", version=" + version + "}"; } @Override public int compareTo(WorkflowDefSummary o) { int res = this.name.compareTo(o.name); if (res != 0) { return res; } res = Integer.compare(this.version, o.version); return res; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/metadata/workflow/WorkflowTask.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.metadata.workflow; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.PositiveOrZero; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.fasterxml.jackson.annotation.JsonInclude; /** * This is the task definition definied as part of the {@link WorkflowDef}. The tasks definied in * the Workflow definition are saved as part of {@link WorkflowDef#getTasks} */ @ProtoMessage public class WorkflowTask { @ProtoField(id = 1) @NotEmpty(message = "WorkflowTask name cannot be empty or null") private String name; @ProtoField(id = 2) @NotEmpty(message = "WorkflowTask taskReferenceName name cannot be empty or null") private String taskReferenceName; @ProtoField(id = 3) private String description; @ProtoField(id = 4) private Map inputParameters = new HashMap<>(); @ProtoField(id = 5) private String type = TaskType.SIMPLE.name(); @ProtoField(id = 6) private String dynamicTaskNameParam; @Deprecated @ProtoField(id = 7) private String caseValueParam; @Deprecated @ProtoField(id = 8) private String caseExpression; @ProtoField(id = 22) private String scriptExpression; @ProtoMessage(wrapper = true) public static class WorkflowTaskList { public List getTasks() { return tasks; } public void setTasks(List tasks) { this.tasks = tasks; } @ProtoField(id = 1) private List tasks; } // Populates for the tasks of the decision type @ProtoField(id = 9) @JsonInclude(JsonInclude.Include.NON_EMPTY) private Map> decisionCases = new LinkedHashMap<>(); @Deprecated private String dynamicForkJoinTasksParam; @ProtoField(id = 10) private String dynamicForkTasksParam; @ProtoField(id = 11) private String dynamicForkTasksInputParamName; @ProtoField(id = 12) @JsonInclude(JsonInclude.Include.NON_EMPTY) private List<@Valid WorkflowTask> defaultCase = new LinkedList<>(); @ProtoField(id = 13) @JsonInclude(JsonInclude.Include.NON_EMPTY) private List<@Valid List<@Valid WorkflowTask>> forkTasks = new LinkedList<>(); @ProtoField(id = 14) @PositiveOrZero private int startDelay; // No. of seconds (at-least) to wait before starting a task. @ProtoField(id = 15) @Valid private SubWorkflowParams subWorkflowParam; @ProtoField(id = 16) @JsonInclude(JsonInclude.Include.NON_EMPTY) private List joinOn = new LinkedList<>(); @ProtoField(id = 17) private String sink; @ProtoField(id = 18) private boolean optional = false; @ProtoField(id = 19) private TaskDef taskDefinition; @ProtoField(id = 20) private Boolean rateLimited; @ProtoField(id = 21) @JsonInclude(JsonInclude.Include.NON_EMPTY) private List defaultExclusiveJoinTask = new LinkedList<>(); @ProtoField(id = 23) private Boolean asyncComplete = false; @ProtoField(id = 24) private String loopCondition; @ProtoField(id = 25) @JsonInclude(JsonInclude.Include.NON_EMPTY) private List loopOver = new LinkedList<>(); @ProtoField(id = 26) private Integer retryCount; @ProtoField(id = 27) private String evaluatorType; @ProtoField(id = 28) private String expression; /** * @return the name */ public String getName() { return name; } /** * @param name the name to set */ public void setName(String name) { this.name = name; } /** * @return the taskReferenceName */ public String getTaskReferenceName() { return taskReferenceName; } /** * @param taskReferenceName the taskReferenceName to set */ public void setTaskReferenceName(String taskReferenceName) { this.taskReferenceName = taskReferenceName; } /** * @return the description */ public String getDescription() { return description; } /** * @param description the description to set */ public void setDescription(String description) { this.description = description; } /** * @return the inputParameters */ public Map getInputParameters() { return inputParameters; } /** * @param inputParameters the inputParameters to set */ public void setInputParameters(Map inputParameters) { this.inputParameters = inputParameters; } /** * @return the type */ public String getType() { return type; } public void setWorkflowTaskType(TaskType type) { this.type = type.name(); } /** * @param type the type to set */ public void setType(@NotEmpty(message = "WorkTask type cannot be null or empty") String type) { this.type = type; } /** * @return the decisionCases */ public Map> getDecisionCases() { return decisionCases; } /** * @param decisionCases the decisionCases to set */ public void setDecisionCases(Map> decisionCases) { this.decisionCases = decisionCases; } /** * @return the defaultCase */ public List getDefaultCase() { return defaultCase; } /** * @param defaultCase the defaultCase to set */ public void setDefaultCase(List defaultCase) { this.defaultCase = defaultCase; } /** * @return the forkTasks */ public List> getForkTasks() { return forkTasks; } /** * @param forkTasks the forkTasks to set */ public void setForkTasks(List> forkTasks) { this.forkTasks = forkTasks; } /** * @return the startDelay in seconds */ public int getStartDelay() { return startDelay; } /** * @param startDelay the startDelay to set */ public void setStartDelay(int startDelay) { this.startDelay = startDelay; } /** * @return the retryCount */ public Integer getRetryCount() { return retryCount; } /** * @param retryCount the retryCount to set */ public void setRetryCount(final Integer retryCount) { this.retryCount = retryCount; } /** * @return the dynamicTaskNameParam */ public String getDynamicTaskNameParam() { return dynamicTaskNameParam; } /** * @param dynamicTaskNameParam the dynamicTaskNameParam to set to be used by DYNAMIC tasks */ public void setDynamicTaskNameParam(String dynamicTaskNameParam) { this.dynamicTaskNameParam = dynamicTaskNameParam; } /** * @deprecated Use {@link WorkflowTask#getEvaluatorType()} and {@link * WorkflowTask#getExpression()} combination. * @return the caseValueParam */ @Deprecated public String getCaseValueParam() { return caseValueParam; } @Deprecated public String getDynamicForkJoinTasksParam() { return dynamicForkJoinTasksParam; } @Deprecated public void setDynamicForkJoinTasksParam(String dynamicForkJoinTasksParam) { this.dynamicForkJoinTasksParam = dynamicForkJoinTasksParam; } public String getDynamicForkTasksParam() { return dynamicForkTasksParam; } public void setDynamicForkTasksParam(String dynamicForkTasksParam) { this.dynamicForkTasksParam = dynamicForkTasksParam; } public String getDynamicForkTasksInputParamName() { return dynamicForkTasksInputParamName; } public void setDynamicForkTasksInputParamName(String dynamicForkTasksInputParamName) { this.dynamicForkTasksInputParamName = dynamicForkTasksInputParamName; } /** * @param caseValueParam the caseValueParam to set * @deprecated Use {@link WorkflowTask#getEvaluatorType()} and {@link * WorkflowTask#getExpression()} combination. */ @Deprecated public void setCaseValueParam(String caseValueParam) { this.caseValueParam = caseValueParam; } /** * @return A javascript expression for decision cases. The result should be a scalar value that * is used to decide the case branches. * @see #getDecisionCases() * @deprecated Use {@link WorkflowTask#getEvaluatorType()} and {@link * WorkflowTask#getExpression()} combination. */ @Deprecated public String getCaseExpression() { return caseExpression; } /** * @param caseExpression A javascript expression for decision cases. The result should be a * scalar value that is used to decide the case branches. * @deprecated Use {@link WorkflowTask#getEvaluatorType()} and {@link * WorkflowTask#getExpression()} combination. */ @Deprecated public void setCaseExpression(String caseExpression) { this.caseExpression = caseExpression; } public String getScriptExpression() { return scriptExpression; } public void setScriptExpression(String expression) { this.scriptExpression = expression; } /** * @return the subWorkflow */ public SubWorkflowParams getSubWorkflowParam() { return subWorkflowParam; } /** * @param subWorkflow the subWorkflowParam to set */ public void setSubWorkflowParam(SubWorkflowParams subWorkflow) { this.subWorkflowParam = subWorkflow; } /** * @return the joinOn */ public List getJoinOn() { return joinOn; } /** * @param joinOn the joinOn to set */ public void setJoinOn(List joinOn) { this.joinOn = joinOn; } /** * @return the loopCondition */ public String getLoopCondition() { return loopCondition; } /** * @param loopCondition the expression to set */ public void setLoopCondition(String loopCondition) { this.loopCondition = loopCondition; } /** * @return the loopOver */ public List getLoopOver() { return loopOver; } /** * @param loopOver the loopOver to set */ public void setLoopOver(List loopOver) { this.loopOver = loopOver; } /** * @return Sink value for the EVENT type of task */ public String getSink() { return sink; } /** * @param sink Name of the sink */ public void setSink(String sink) { this.sink = sink; } /** * @return whether wait for an external event to complete the task, for EVENT and HTTP tasks */ public Boolean isAsyncComplete() { return asyncComplete; } public void setAsyncComplete(Boolean asyncComplete) { this.asyncComplete = asyncComplete; } /** * @return If the task is optional. When set to true, the workflow execution continues even when * the task is in failed status. */ public boolean isOptional() { return optional; } /** * @return Task definition associated to the Workflow Task */ public TaskDef getTaskDefinition() { return taskDefinition; } /** * @param taskDefinition Task definition */ public void setTaskDefinition(TaskDef taskDefinition) { this.taskDefinition = taskDefinition; } /** * @param optional when set to true, the task is marked as optional */ public void setOptional(boolean optional) { this.optional = optional; } public Boolean getRateLimited() { return rateLimited; } public void setRateLimited(Boolean rateLimited) { this.rateLimited = rateLimited; } public Boolean isRateLimited() { return rateLimited != null && rateLimited; } public List getDefaultExclusiveJoinTask() { return defaultExclusiveJoinTask; } public void setDefaultExclusiveJoinTask(List defaultExclusiveJoinTask) { this.defaultExclusiveJoinTask = defaultExclusiveJoinTask; } /** * @return the evaluatorType */ public String getEvaluatorType() { return evaluatorType; } /** * @param evaluatorType the evaluatorType to set */ public void setEvaluatorType(String evaluatorType) { this.evaluatorType = evaluatorType; } /** * @return An evaluation expression for switch cases evaluated by corresponding evaluator. The * result should be a scalar value that is used to decide the case branches. * @see #getDecisionCases() */ public String getExpression() { return expression; } /** * @param expression the expression to set */ public void setExpression(String expression) { this.expression = expression; } private Collection> children() { Collection> workflowTaskLists = new LinkedList<>(); switch (TaskType.of(type)) { case DECISION: case SWITCH: workflowTaskLists.addAll(decisionCases.values()); workflowTaskLists.add(defaultCase); break; case FORK_JOIN: workflowTaskLists.addAll(forkTasks); break; case DO_WHILE: workflowTaskLists.add(loopOver); break; default: break; } return workflowTaskLists; } public List collectTasks() { List tasks = new LinkedList<>(); tasks.add(this); for (List workflowTaskList : children()) { for (WorkflowTask workflowTask : workflowTaskList) { tasks.addAll(workflowTask.collectTasks()); } } return tasks; } public WorkflowTask next(String taskReferenceName, WorkflowTask parent) { TaskType taskType = TaskType.of(type); switch (taskType) { case DO_WHILE: case DECISION: case SWITCH: for (List workflowTasks : children()) { Iterator iterator = workflowTasks.iterator(); while (iterator.hasNext()) { WorkflowTask task = iterator.next(); if (task.getTaskReferenceName().equals(taskReferenceName)) { break; } WorkflowTask nextTask = task.next(taskReferenceName, this); if (nextTask != null) { return nextTask; } if (task.has(taskReferenceName)) { break; } } if (iterator.hasNext()) { return iterator.next(); } } if (taskType == TaskType.DO_WHILE && this.has(taskReferenceName)) { // come here means this is DO_WHILE task and `taskReferenceName` is the last // task in // this DO_WHILE task, because DO_WHILE task need to be executed to decide // whether to // schedule next iteration, so we just return the DO_WHILE task, and then ignore // generating this task again in deciderService.getNextTask() return this; } break; case FORK_JOIN: boolean found = false; for (List workflowTasks : children()) { Iterator iterator = workflowTasks.iterator(); while (iterator.hasNext()) { WorkflowTask task = iterator.next(); if (task.getTaskReferenceName().equals(taskReferenceName)) { found = true; break; } WorkflowTask nextTask = task.next(taskReferenceName, this); if (nextTask != null) { return nextTask; } if (task.has(taskReferenceName)) { break; } } if (iterator.hasNext()) { return iterator.next(); } if (found && parent != null) { return parent.next( this.taskReferenceName, parent); // we need to return join task... -- get my sibling from my // parent.. } } break; case DYNAMIC: case TERMINATE: case SIMPLE: return null; default: break; } return null; } public boolean has(String taskReferenceName) { if (this.getTaskReferenceName().equals(taskReferenceName)) { return true; } switch (TaskType.of(type)) { case DECISION: case SWITCH: case DO_WHILE: case FORK_JOIN: for (List childx : children()) { for (WorkflowTask child : childx) { if (child.has(taskReferenceName)) { return true; } } } break; default: break; } return false; } public WorkflowTask get(String taskReferenceName) { if (this.getTaskReferenceName().equals(taskReferenceName)) { return this; } for (List childx : children()) { for (WorkflowTask child : childx) { WorkflowTask found = child.get(taskReferenceName); if (found != null) { return found; } } } return null; } @Override public String toString() { return name + "/" + taskReferenceName; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } WorkflowTask that = (WorkflowTask) o; return getStartDelay() == that.getStartDelay() && isOptional() == that.isOptional() && Objects.equals(getName(), that.getName()) && Objects.equals(getTaskReferenceName(), that.getTaskReferenceName()) && Objects.equals(getDescription(), that.getDescription()) && Objects.equals(getInputParameters(), that.getInputParameters()) && Objects.equals(getType(), that.getType()) && Objects.equals(getDynamicTaskNameParam(), that.getDynamicTaskNameParam()) && Objects.equals(getCaseValueParam(), that.getCaseValueParam()) && Objects.equals(getEvaluatorType(), that.getEvaluatorType()) && Objects.equals(getExpression(), that.getExpression()) && Objects.equals(getCaseExpression(), that.getCaseExpression()) && Objects.equals(getDecisionCases(), that.getDecisionCases()) && Objects.equals( getDynamicForkJoinTasksParam(), that.getDynamicForkJoinTasksParam()) && Objects.equals(getDynamicForkTasksParam(), that.getDynamicForkTasksParam()) && Objects.equals( getDynamicForkTasksInputParamName(), that.getDynamicForkTasksInputParamName()) && Objects.equals(getDefaultCase(), that.getDefaultCase()) && Objects.equals(getForkTasks(), that.getForkTasks()) && Objects.equals(getSubWorkflowParam(), that.getSubWorkflowParam()) && Objects.equals(getJoinOn(), that.getJoinOn()) && Objects.equals(getSink(), that.getSink()) && Objects.equals(isAsyncComplete(), that.isAsyncComplete()) && Objects.equals(getDefaultExclusiveJoinTask(), that.getDefaultExclusiveJoinTask()) && Objects.equals(getRetryCount(), that.getRetryCount()); } @Override public int hashCode() { return Objects.hash( getName(), getTaskReferenceName(), getDescription(), getInputParameters(), getType(), getDynamicTaskNameParam(), getCaseValueParam(), getCaseExpression(), getEvaluatorType(), getExpression(), getDecisionCases(), getDynamicForkJoinTasksParam(), getDynamicForkTasksParam(), getDynamicForkTasksInputParamName(), getDefaultCase(), getForkTasks(), getStartDelay(), getSubWorkflowParam(), getJoinOn(), getSink(), isAsyncComplete(), isOptional(), getDefaultExclusiveJoinTask(), getRetryCount()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/model/BulkResponse.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.model; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; /** * Response object to return a list of succeeded entities and a map of failed ones, including error * message, for the bulk request. */ public class BulkResponse { /** Key - entityId Value - error message processing this entity */ private final Map bulkErrorResults; private final List bulkSuccessfulResults; private final String message = "Bulk Request has been processed."; public BulkResponse() { this.bulkSuccessfulResults = new ArrayList<>(); this.bulkErrorResults = new HashMap<>(); } public List getBulkSuccessfulResults() { return bulkSuccessfulResults; } public Map getBulkErrorResults() { return bulkErrorResults; } public void appendSuccessResponse(String id) { bulkSuccessfulResults.add(id); } public void appendFailedResponse(String id, String errorMessage) { bulkErrorResults.put(id, errorMessage); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof BulkResponse)) { return false; } BulkResponse that = (BulkResponse) o; return Objects.equals(bulkSuccessfulResults, that.bulkSuccessfulResults) && Objects.equals(bulkErrorResults, that.bulkErrorResults); } @Override public int hashCode() { return Objects.hash(bulkSuccessfulResults, bulkErrorResults, message); } @Override public String toString() { return "BulkResponse{" + "bulkSuccessfulResults=" + bulkSuccessfulResults + ", bulkErrorResults=" + bulkErrorResults + ", message='" + message + '\'' + '}'; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/ExternalStorageLocation.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; /** * Describes the location where the JSON payload is stored in external storage. * *

The location is described using the following fields: * *

    *
  • uri: The uri of the json file in external storage. *
  • path: The relative path of the file in external storage. *
*/ public class ExternalStorageLocation { private String uri; private String path; public String getUri() { return uri; } public void setUri(String uri) { this.uri = uri; } public String getPath() { return path; } public void setPath(String path) { this.path = path; } @Override public String toString() { return "ExternalStorageLocation{" + "uri='" + uri + '\'' + ", path='" + path + '\'' + '}'; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/SearchResult.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import java.util.List; public class SearchResult { private long totalHits; private List results; public SearchResult() {} public SearchResult(long totalHits, List results) { super(); this.totalHits = totalHits; this.results = results; } /** * @return the totalHits */ public long getTotalHits() { return totalHits; } /** * @return the results */ public List getResults() { return results; } /** * @param totalHits the totalHits to set */ public void setTotalHits(long totalHits) { this.totalHits = totalHits; } /** * @param results the results to set */ public void setResults(List results) { this.results = results; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/TaskSummary.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Objects; import java.util.TimeZone; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.utils.SummaryUtil; @ProtoMessage public class TaskSummary { /** The time should be stored as GMT */ private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); @ProtoField(id = 1) private String workflowId; @ProtoField(id = 2) private String workflowType; @ProtoField(id = 3) private String correlationId; @ProtoField(id = 4) private String scheduledTime; @ProtoField(id = 5) private String startTime; @ProtoField(id = 6) private String updateTime; @ProtoField(id = 7) private String endTime; @ProtoField(id = 8) private Task.Status status; @ProtoField(id = 9) private String reasonForIncompletion; @ProtoField(id = 10) private long executionTime; @ProtoField(id = 11) private long queueWaitTime; @ProtoField(id = 12) private String taskDefName; @ProtoField(id = 13) private String taskType; @ProtoField(id = 14) private String input; @ProtoField(id = 15) private String output; @ProtoField(id = 16) private String taskId; @ProtoField(id = 17) private String externalInputPayloadStoragePath; @ProtoField(id = 18) private String externalOutputPayloadStoragePath; @ProtoField(id = 19) private int workflowPriority; @ProtoField(id = 20) private String domain; public TaskSummary() {} public TaskSummary(Task task) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); sdf.setTimeZone(GMT); this.taskId = task.getTaskId(); this.taskDefName = task.getTaskDefName(); this.taskType = task.getTaskType(); this.workflowId = task.getWorkflowInstanceId(); this.workflowType = task.getWorkflowType(); this.workflowPriority = task.getWorkflowPriority(); this.correlationId = task.getCorrelationId(); this.scheduledTime = sdf.format(new Date(task.getScheduledTime())); this.startTime = sdf.format(new Date(task.getStartTime())); this.updateTime = sdf.format(new Date(task.getUpdateTime())); this.endTime = sdf.format(new Date(task.getEndTime())); this.status = task.getStatus(); this.reasonForIncompletion = task.getReasonForIncompletion(); this.queueWaitTime = task.getQueueWaitTime(); this.domain = task.getDomain(); if (task.getInputData() != null) { this.input = SummaryUtil.serializeInputOutput(task.getInputData()); } if (task.getOutputData() != null) { this.output = SummaryUtil.serializeInputOutput(task.getOutputData()); } if (task.getEndTime() > 0) { this.executionTime = task.getEndTime() - task.getStartTime(); } if (StringUtils.isNotBlank(task.getExternalInputPayloadStoragePath())) { this.externalInputPayloadStoragePath = task.getExternalInputPayloadStoragePath(); } if (StringUtils.isNotBlank(task.getExternalOutputPayloadStoragePath())) { this.externalOutputPayloadStoragePath = task.getExternalOutputPayloadStoragePath(); } } /** * @return the workflowId */ public String getWorkflowId() { return workflowId; } /** * @param workflowId the workflowId to set */ public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } /** * @return the workflowType */ public String getWorkflowType() { return workflowType; } /** * @param workflowType the workflowType to set */ public void setWorkflowType(String workflowType) { this.workflowType = workflowType; } /** * @return the correlationId */ public String getCorrelationId() { return correlationId; } /** * @param correlationId the correlationId to set */ public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } /** * @return the scheduledTime */ public String getScheduledTime() { return scheduledTime; } /** * @param scheduledTime the scheduledTime to set */ public void setScheduledTime(String scheduledTime) { this.scheduledTime = scheduledTime; } /** * @return the startTime */ public String getStartTime() { return startTime; } /** * @param startTime the startTime to set */ public void setStartTime(String startTime) { this.startTime = startTime; } /** * @return the updateTime */ public String getUpdateTime() { return updateTime; } /** * @param updateTime the updateTime to set */ public void setUpdateTime(String updateTime) { this.updateTime = updateTime; } /** * @return the endTime */ public String getEndTime() { return endTime; } /** * @param endTime the endTime to set */ public void setEndTime(String endTime) { this.endTime = endTime; } /** * @return the status */ public Status getStatus() { return status; } /** * @param status the status to set */ public void setStatus(Status status) { this.status = status; } /** * @return the reasonForIncompletion */ public String getReasonForIncompletion() { return reasonForIncompletion; } /** * @param reasonForIncompletion the reasonForIncompletion to set */ public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = reasonForIncompletion; } /** * @return the executionTime */ public long getExecutionTime() { return executionTime; } /** * @param executionTime the executionTime to set */ public void setExecutionTime(long executionTime) { this.executionTime = executionTime; } /** * @return the queueWaitTime */ public long getQueueWaitTime() { return queueWaitTime; } /** * @param queueWaitTime the queueWaitTime to set */ public void setQueueWaitTime(long queueWaitTime) { this.queueWaitTime = queueWaitTime; } /** * @return the taskDefName */ public String getTaskDefName() { return taskDefName; } /** * @param taskDefName the taskDefName to set */ public void setTaskDefName(String taskDefName) { this.taskDefName = taskDefName; } /** * @return the taskType */ public String getTaskType() { return taskType; } /** * @param taskType the taskType to set */ public void setTaskType(String taskType) { this.taskType = taskType; } /** * @return input to the task */ public String getInput() { return input; } /** * @param input input to the task */ public void setInput(String input) { this.input = input; } /** * @return output of the task */ public String getOutput() { return output; } /** * @param output Task output */ public void setOutput(String output) { this.output = output; } /** * @return the taskId */ public String getTaskId() { return taskId; } /** * @param taskId the taskId to set */ public void setTaskId(String taskId) { this.taskId = taskId; } /** * @return the external storage path for the task input payload */ public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } /** * @param externalInputPayloadStoragePath the external storage path where the task input payload * is stored */ public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } /** * @return the external storage path for the task output payload */ public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } /** * @param externalOutputPayloadStoragePath the external storage path where the task output * payload is stored */ public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } /** * @return the priority defined on workflow */ public int getWorkflowPriority() { return workflowPriority; } /** * @param workflowPriority Priority defined for workflow */ public void setWorkflowPriority(int workflowPriority) { this.workflowPriority = workflowPriority; } /** * @return the domain that the task was scheduled in */ public String getDomain() { return domain; } /** * @param domain The domain that the task was scheduled in */ public void setDomain(String domain) { this.domain = domain; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } TaskSummary that = (TaskSummary) o; return getExecutionTime() == that.getExecutionTime() && getQueueWaitTime() == that.getQueueWaitTime() && getWorkflowPriority() == that.getWorkflowPriority() && getWorkflowId().equals(that.getWorkflowId()) && getWorkflowType().equals(that.getWorkflowType()) && Objects.equals(getCorrelationId(), that.getCorrelationId()) && getScheduledTime().equals(that.getScheduledTime()) && Objects.equals(getStartTime(), that.getStartTime()) && Objects.equals(getUpdateTime(), that.getUpdateTime()) && Objects.equals(getEndTime(), that.getEndTime()) && getStatus() == that.getStatus() && Objects.equals(getReasonForIncompletion(), that.getReasonForIncompletion()) && Objects.equals(getTaskDefName(), that.getTaskDefName()) && getTaskType().equals(that.getTaskType()) && getTaskId().equals(that.getTaskId()) && Objects.equals(getDomain(), that.getDomain()); } @Override public int hashCode() { return Objects.hash( getWorkflowId(), getWorkflowType(), getCorrelationId(), getScheduledTime(), getStartTime(), getUpdateTime(), getEndTime(), getStatus(), getReasonForIncompletion(), getExecutionTime(), getQueueWaitTime(), getTaskDefName(), getTaskType(), getTaskId(), getWorkflowPriority(), getDomain()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/Workflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import java.util.*; import java.util.stream.Collectors; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.annotations.protogen.ProtoEnum; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.metadata.Auditable; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @ProtoMessage public class Workflow extends Auditable { @ProtoEnum public enum WorkflowStatus { RUNNING(false, false), COMPLETED(true, true), FAILED(true, false), TIMED_OUT(true, false), TERMINATED(true, false), PAUSED(false, true); private final boolean terminal; private final boolean successful; WorkflowStatus(boolean terminal, boolean successful) { this.terminal = terminal; this.successful = successful; } public boolean isTerminal() { return terminal; } public boolean isSuccessful() { return successful; } } @ProtoField(id = 1) private WorkflowStatus status = WorkflowStatus.RUNNING; @ProtoField(id = 2) private long endTime; @ProtoField(id = 3) private String workflowId; @ProtoField(id = 4) private String parentWorkflowId; @ProtoField(id = 5) private String parentWorkflowTaskId; @ProtoField(id = 6) private List tasks = new LinkedList<>(); @ProtoField(id = 8) private Map input = new HashMap<>(); @ProtoField(id = 9) private Map output = new HashMap<>(); // ids 10,11 are reserved @ProtoField(id = 12) private String correlationId; @ProtoField(id = 13) private String reRunFromWorkflowId; @ProtoField(id = 14) private String reasonForIncompletion; // id 15 is reserved @ProtoField(id = 16) private String event; @ProtoField(id = 17) private Map taskToDomain = new HashMap<>(); @ProtoField(id = 18) private Set failedReferenceTaskNames = new HashSet<>(); @ProtoField(id = 19) private WorkflowDef workflowDefinition; @ProtoField(id = 20) private String externalInputPayloadStoragePath; @ProtoField(id = 21) private String externalOutputPayloadStoragePath; @ProtoField(id = 22) @Min(value = 0, message = "workflow priority: ${validatedValue} should be minimum {value}") @Max(value = 99, message = "workflow priority: ${validatedValue} should be maximum {value}") private int priority; @ProtoField(id = 23) private Map variables = new HashMap<>(); @ProtoField(id = 24) private long lastRetriedTime; @ProtoField(id = 25) private Set failedTaskNames = new HashSet<>(); public Workflow() {} /** * @return the status */ public WorkflowStatus getStatus() { return status; } /** * @param status the status to set */ public void setStatus(WorkflowStatus status) { this.status = status; } /** * @return the startTime */ public long getStartTime() { return getCreateTime(); } /** * @param startTime the startTime to set */ public void setStartTime(long startTime) { this.setCreateTime(startTime); } /** * @return the endTime */ public long getEndTime() { return endTime; } /** * @param endTime the endTime to set */ public void setEndTime(long endTime) { this.endTime = endTime; } /** * @return the workflowId */ public String getWorkflowId() { return workflowId; } /** * @param workflowId the workflowId to set */ public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } /** * @return the tasks which are scheduled, in progress or completed. */ public List getTasks() { return tasks; } /** * @param tasks the tasks to set */ public void setTasks(List tasks) { this.tasks = tasks; } /** * @return the input */ public Map getInput() { return input; } /** * @param input the input to set */ public void setInput(Map input) { if (input == null) { input = new HashMap<>(); } this.input = input; } /** * @return the task to domain map */ public Map getTaskToDomain() { return taskToDomain; } /** * @param taskToDomain the task to domain map */ public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } /** * @return the output */ public Map getOutput() { return output; } /** * @param output the output to set */ public void setOutput(Map output) { if (output == null) { output = new HashMap<>(); } this.output = output; } /** * @return The correlation id used when starting the workflow */ public String getCorrelationId() { return correlationId; } /** * @param correlationId the correlation id */ public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public String getReRunFromWorkflowId() { return reRunFromWorkflowId; } public void setReRunFromWorkflowId(String reRunFromWorkflowId) { this.reRunFromWorkflowId = reRunFromWorkflowId; } public String getReasonForIncompletion() { return reasonForIncompletion; } public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = reasonForIncompletion; } /** * @return the parentWorkflowId */ public String getParentWorkflowId() { return parentWorkflowId; } /** * @param parentWorkflowId the parentWorkflowId to set */ public void setParentWorkflowId(String parentWorkflowId) { this.parentWorkflowId = parentWorkflowId; } /** * @return the parentWorkflowTaskId */ public String getParentWorkflowTaskId() { return parentWorkflowTaskId; } /** * @param parentWorkflowTaskId the parentWorkflowTaskId to set */ public void setParentWorkflowTaskId(String parentWorkflowTaskId) { this.parentWorkflowTaskId = parentWorkflowTaskId; } /** * @return Name of the event that started the workflow */ public String getEvent() { return event; } /** * @param event Name of the event that started the workflow */ public void setEvent(String event) { this.event = event; } public Set getFailedReferenceTaskNames() { return failedReferenceTaskNames; } public void setFailedReferenceTaskNames(Set failedReferenceTaskNames) { this.failedReferenceTaskNames = failedReferenceTaskNames; } public Set getFailedTaskNames() { return failedTaskNames; } public void setFailedTaskNames(Set failedTaskNames) { this.failedTaskNames = failedTaskNames; } public WorkflowDef getWorkflowDefinition() { return workflowDefinition; } public void setWorkflowDefinition(WorkflowDef workflowDefinition) { this.workflowDefinition = workflowDefinition; } /** * @return the external storage path of the workflow input payload */ public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } /** * @param externalInputPayloadStoragePath the external storage path where the workflow input * payload is stored */ public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } /** * @return the external storage path of the workflow output payload */ public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } /** * @return the priority to define on tasks */ public int getPriority() { return priority; } /** * @param priority priority of tasks (between 0 and 99) */ public void setPriority(int priority) { if (priority < 0 || priority > 99) { throw new IllegalArgumentException("priority MUST be between 0 and 99 (inclusive)"); } this.priority = priority; } /** * Convenience method for accessing the workflow definition name. * * @return the workflow definition name. */ public String getWorkflowName() { if (workflowDefinition == null) { throw new NullPointerException("Workflow definition is null"); } return workflowDefinition.getName(); } /** * Convenience method for accessing the workflow definition version. * * @return the workflow definition version. */ public int getWorkflowVersion() { if (workflowDefinition == null) { throw new NullPointerException("Workflow definition is null"); } return workflowDefinition.getVersion(); } /** * @param externalOutputPayloadStoragePath the external storage path where the workflow output * payload is stored */ public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } /** * @return the global workflow variables */ public Map getVariables() { return variables; } /** * @param variables the set of global workflow variables to set */ public void setVariables(Map variables) { this.variables = variables; } /** * Captures the last time the workflow was retried * * @return the last retried time of the workflow */ public long getLastRetriedTime() { return lastRetriedTime; } /** * @param lastRetriedTime time in milliseconds when the workflow is retried */ public void setLastRetriedTime(long lastRetriedTime) { this.lastRetriedTime = lastRetriedTime; } public boolean hasParent() { return StringUtils.isNotEmpty(parentWorkflowId); } public Task getTaskByRefName(String refName) { if (refName == null) { throw new RuntimeException( "refName passed is null. Check the workflow execution. For dynamic tasks, make sure referenceTaskName is set to a not null value"); } LinkedList found = new LinkedList<>(); for (Task t : tasks) { if (t.getReferenceTaskName() == null) { throw new RuntimeException( "Task " + t.getTaskDefName() + ", seq=" + t.getSeq() + " does not have reference name specified."); } if (t.getReferenceTaskName().equals(refName)) { found.add(t); } } if (found.isEmpty()) { return null; } return found.getLast(); } /** * @return a deep copy of the workflow instance */ public Workflow copy() { Workflow copy = new Workflow(); copy.setInput(input); copy.setOutput(output); copy.setStatus(status); copy.setWorkflowId(workflowId); copy.setParentWorkflowId(parentWorkflowId); copy.setParentWorkflowTaskId(parentWorkflowTaskId); copy.setReRunFromWorkflowId(reRunFromWorkflowId); copy.setCorrelationId(correlationId); copy.setEvent(event); copy.setReasonForIncompletion(reasonForIncompletion); copy.setWorkflowDefinition(workflowDefinition); copy.setPriority(priority); copy.setTasks(tasks.stream().map(Task::deepCopy).collect(Collectors.toList())); copy.setVariables(variables); copy.setEndTime(endTime); copy.setLastRetriedTime(lastRetriedTime); copy.setTaskToDomain(taskToDomain); copy.setFailedReferenceTaskNames(failedReferenceTaskNames); copy.setFailedTaskNames(failedTaskNames); copy.setExternalInputPayloadStoragePath(externalInputPayloadStoragePath); copy.setExternalOutputPayloadStoragePath(externalOutputPayloadStoragePath); return copy; } @Override public String toString() { String name = workflowDefinition != null ? workflowDefinition.getName() : null; Integer version = workflowDefinition != null ? workflowDefinition.getVersion() : null; return String.format("%s.%s/%s.%s", name, version, workflowId, status); } /** * A string representation of all relevant fields that identify this workflow. Intended for use * in log and other system generated messages. */ public String toShortString() { String name = workflowDefinition != null ? workflowDefinition.getName() : null; Integer version = workflowDefinition != null ? workflowDefinition.getVersion() : null; return String.format("%s.%s/%s", name, version, workflowId); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Workflow workflow = (Workflow) o; return getEndTime() == workflow.getEndTime() && getWorkflowVersion() == workflow.getWorkflowVersion() && getStatus() == workflow.getStatus() && Objects.equals(getWorkflowId(), workflow.getWorkflowId()) && Objects.equals(getParentWorkflowId(), workflow.getParentWorkflowId()) && Objects.equals(getParentWorkflowTaskId(), workflow.getParentWorkflowTaskId()) && Objects.equals(getTasks(), workflow.getTasks()) && Objects.equals(getInput(), workflow.getInput()) && Objects.equals(getOutput(), workflow.getOutput()) && Objects.equals(getWorkflowName(), workflow.getWorkflowName()) && Objects.equals(getCorrelationId(), workflow.getCorrelationId()) && Objects.equals(getReRunFromWorkflowId(), workflow.getReRunFromWorkflowId()) && Objects.equals(getReasonForIncompletion(), workflow.getReasonForIncompletion()) && Objects.equals(getEvent(), workflow.getEvent()) && Objects.equals(getTaskToDomain(), workflow.getTaskToDomain()) && Objects.equals( getFailedReferenceTaskNames(), workflow.getFailedReferenceTaskNames()) && Objects.equals(getFailedTaskNames(), workflow.getFailedTaskNames()) && Objects.equals( getExternalInputPayloadStoragePath(), workflow.getExternalInputPayloadStoragePath()) && Objects.equals( getExternalOutputPayloadStoragePath(), workflow.getExternalOutputPayloadStoragePath()) && Objects.equals(getPriority(), workflow.getPriority()) && Objects.equals(getWorkflowDefinition(), workflow.getWorkflowDefinition()) && Objects.equals(getVariables(), workflow.getVariables()) && Objects.equals(getLastRetriedTime(), workflow.getLastRetriedTime()); } @Override public int hashCode() { return Objects.hash( getStatus(), getEndTime(), getWorkflowId(), getParentWorkflowId(), getParentWorkflowTaskId(), getTasks(), getInput(), getOutput(), getWorkflowName(), getWorkflowVersion(), getCorrelationId(), getReRunFromWorkflowId(), getReasonForIncompletion(), getEvent(), getTaskToDomain(), getFailedReferenceTaskNames(), getFailedTaskNames(), getWorkflowDefinition(), getExternalInputPayloadStoragePath(), getExternalOutputPayloadStoragePath(), getPriority(), getVariables(), getLastRetriedTime()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/WorkflowSummary.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashSet; import java.util.Objects; import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.annotations.protogen.ProtoField; import com.netflix.conductor.annotations.protogen.ProtoMessage; import com.netflix.conductor.common.run.Workflow.WorkflowStatus; import com.netflix.conductor.common.utils.SummaryUtil; /** Captures workflow summary info to be indexed in Elastic Search. */ @ProtoMessage public class WorkflowSummary { /** The time should be stored as GMT */ private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); @ProtoField(id = 1) private String workflowType; @ProtoField(id = 2) private int version; @ProtoField(id = 3) private String workflowId; @ProtoField(id = 4) private String correlationId; @ProtoField(id = 5) private String startTime; @ProtoField(id = 6) private String updateTime; @ProtoField(id = 7) private String endTime; @ProtoField(id = 8) private Workflow.WorkflowStatus status; @ProtoField(id = 9) private String input; @ProtoField(id = 10) private String output; @ProtoField(id = 11) private String reasonForIncompletion; @ProtoField(id = 12) private long executionTime; @ProtoField(id = 13) private String event; @ProtoField(id = 14) private String failedReferenceTaskNames = ""; @ProtoField(id = 15) private String externalInputPayloadStoragePath; @ProtoField(id = 16) private String externalOutputPayloadStoragePath; @ProtoField(id = 17) private int priority; @ProtoField(id = 18) private Set failedTaskNames = new HashSet<>(); public WorkflowSummary() {} public WorkflowSummary(Workflow workflow) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); sdf.setTimeZone(GMT); this.workflowType = workflow.getWorkflowName(); this.version = workflow.getWorkflowVersion(); this.workflowId = workflow.getWorkflowId(); this.priority = workflow.getPriority(); this.correlationId = workflow.getCorrelationId(); if (workflow.getCreateTime() != null) { this.startTime = sdf.format(new Date(workflow.getCreateTime())); } if (workflow.getEndTime() > 0) { this.endTime = sdf.format(new Date(workflow.getEndTime())); } if (workflow.getUpdateTime() != null) { this.updateTime = sdf.format(new Date(workflow.getUpdateTime())); } this.status = workflow.getStatus(); if (workflow.getInput() != null) { this.input = SummaryUtil.serializeInputOutput(workflow.getInput()); } if (workflow.getOutput() != null) { this.output = SummaryUtil.serializeInputOutput(workflow.getOutput()); } this.reasonForIncompletion = workflow.getReasonForIncompletion(); if (workflow.getEndTime() > 0) { this.executionTime = workflow.getEndTime() - workflow.getStartTime(); } this.event = workflow.getEvent(); this.failedReferenceTaskNames = workflow.getFailedReferenceTaskNames().stream().collect(Collectors.joining(",")); this.failedTaskNames = workflow.getFailedTaskNames(); if (StringUtils.isNotBlank(workflow.getExternalInputPayloadStoragePath())) { this.externalInputPayloadStoragePath = workflow.getExternalInputPayloadStoragePath(); } if (StringUtils.isNotBlank(workflow.getExternalOutputPayloadStoragePath())) { this.externalOutputPayloadStoragePath = workflow.getExternalOutputPayloadStoragePath(); } } /** * @return the workflowType */ public String getWorkflowType() { return workflowType; } /** * @return the version */ public int getVersion() { return version; } /** * @return the workflowId */ public String getWorkflowId() { return workflowId; } /** * @return the correlationId */ public String getCorrelationId() { return correlationId; } /** * @return the startTime */ public String getStartTime() { return startTime; } /** * @return the endTime */ public String getEndTime() { return endTime; } /** * @return the status */ public WorkflowStatus getStatus() { return status; } /** * @return the input */ public String getInput() { return input; } public long getInputSize() { return input != null ? input.length() : 0; } /** * @return the output */ public String getOutput() { return output; } public long getOutputSize() { return output != null ? output.length() : 0; } /** * @return the reasonForIncompletion */ public String getReasonForIncompletion() { return reasonForIncompletion; } /** * @return the executionTime */ public long getExecutionTime() { return executionTime; } /** * @return the updateTime */ public String getUpdateTime() { return updateTime; } /** * @return The event */ public String getEvent() { return event; } /** * @param event The event */ public void setEvent(String event) { this.event = event; } public String getFailedReferenceTaskNames() { return failedReferenceTaskNames; } public void setFailedReferenceTaskNames(String failedReferenceTaskNames) { this.failedReferenceTaskNames = failedReferenceTaskNames; } public Set getFailedTaskNames() { return failedTaskNames; } public void setFailedTaskNames(Set failedTaskNames) { this.failedTaskNames = failedTaskNames; } public void setWorkflowType(String workflowType) { this.workflowType = workflowType; } public void setVersion(int version) { this.version = version; } public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public void setStartTime(String startTime) { this.startTime = startTime; } public void setUpdateTime(String updateTime) { this.updateTime = updateTime; } public void setEndTime(String endTime) { this.endTime = endTime; } public void setStatus(WorkflowStatus status) { this.status = status; } public void setInput(String input) { this.input = input; } public void setOutput(String output) { this.output = output; } public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = reasonForIncompletion; } public void setExecutionTime(long executionTime) { this.executionTime = executionTime; } /** * @return the external storage path of the workflow input payload */ public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } /** * @param externalInputPayloadStoragePath the external storage path where the workflow input * payload is stored */ public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } /** * @return the external storage path of the workflow output payload */ public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } /** * @param externalOutputPayloadStoragePath the external storage path where the workflow output * payload is stored */ public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } /** * @return the priority to define on tasks */ public int getPriority() { return priority; } /** * @param priority priority of tasks (between 0 and 99) */ public void setPriority(int priority) { this.priority = priority; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } WorkflowSummary that = (WorkflowSummary) o; return getVersion() == that.getVersion() && getExecutionTime() == that.getExecutionTime() && getPriority() == that.getPriority() && getWorkflowType().equals(that.getWorkflowType()) && getWorkflowId().equals(that.getWorkflowId()) && Objects.equals(getCorrelationId(), that.getCorrelationId()) && StringUtils.equals(getStartTime(), that.getStartTime()) && StringUtils.equals(getUpdateTime(), that.getUpdateTime()) && StringUtils.equals(getEndTime(), that.getEndTime()) && getStatus() == that.getStatus() && Objects.equals(getReasonForIncompletion(), that.getReasonForIncompletion()) && Objects.equals(getEvent(), that.getEvent()); } @Override public int hashCode() { return Objects.hash( getWorkflowType(), getVersion(), getWorkflowId(), getCorrelationId(), getStartTime(), getUpdateTime(), getEndTime(), getStatus(), getReasonForIncompletion(), getExecutionTime(), getEvent(), getPriority()); } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/run/WorkflowTestRequest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import java.util.HashMap; import java.util.List; import java.util.Map; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; public class WorkflowTestRequest extends StartWorkflowRequest { // Map of task reference name to mock output for the task private Map> taskRefToMockOutput = new HashMap<>(); // If there are sub-workflows inside the workflow // The map of task reference name to the mock for the sub-workflow private Map subWorkflowTestRequest = new HashMap<>(); public static class TaskMock { private TaskResult.Status status = TaskResult.Status.COMPLETED; private Map output; private long executionTime; // Time in millis for the execution of the task. Useful for // simulating timeout conditions private long queueWaitTime; // Time in millis for the wait time in the queue. public TaskMock() {} public TaskMock(TaskResult.Status status, Map output) { this.status = status; this.output = output; } public TaskResult.Status getStatus() { return status; } public void setStatus(TaskResult.Status status) { this.status = status; } public Map getOutput() { return output; } public void setOutput(Map output) { this.output = output; } public long getExecutionTime() { return executionTime; } public void setExecutionTime(long executionTime) { this.executionTime = executionTime; } public long getQueueWaitTime() { return queueWaitTime; } public void setQueueWaitTime(long queueWaitTime) { this.queueWaitTime = queueWaitTime; } } public Map> getTaskRefToMockOutput() { return taskRefToMockOutput; } public void setTaskRefToMockOutput(Map> taskRefToMockOutput) { this.taskRefToMockOutput = taskRefToMockOutput; } public Map getSubWorkflowTestRequest() { return subWorkflowTestRequest; } public void setSubWorkflowTestRequest(Map subWorkflowTestRequest) { this.subWorkflowTestRequest = subWorkflowTestRequest; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/utils/ConstraintParamUtil.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.EnvUtils.SystemParameters; @SuppressWarnings("unchecked") public class ConstraintParamUtil { /** * Validates inputParam and returns a list of errors if input is not valid. * * @param input {@link Map} of inputParameters * @param taskName TaskName of inputParameters * @param workflow WorkflowDef * @return {@link List} of error strings. */ public static List validateInputParam( Map input, String taskName, WorkflowDef workflow) { ArrayList errorList = new ArrayList<>(); for (Entry e : input.entrySet()) { Object value = e.getValue(); if (value instanceof String) { errorList.addAll( extractParamPathComponentsFromString( e.getKey(), value.toString(), taskName, workflow)); } else if (value instanceof Map) { // recursive call errorList.addAll( validateInputParam((Map) value, taskName, workflow)); } else if (value instanceof List) { errorList.addAll( extractListInputParam(e.getKey(), (List) value, taskName, workflow)); } else { e.setValue(value); } } return errorList; } private static List extractListInputParam( String key, List values, String taskName, WorkflowDef workflow) { ArrayList errorList = new ArrayList<>(); for (Object listVal : values) { if (listVal instanceof String) { errorList.addAll( extractParamPathComponentsFromString( key, listVal.toString(), taskName, workflow)); } else if (listVal instanceof Map) { errorList.addAll( validateInputParam((Map) listVal, taskName, workflow)); } else if (listVal instanceof List) { errorList.addAll(extractListInputParam(key, (List) listVal, taskName, workflow)); } } return errorList; } private static List extractParamPathComponentsFromString( String key, String value, String taskName, WorkflowDef workflow) { ArrayList errorList = new ArrayList<>(); if (value == null) { String message = String.format("key: %s input parameter value: is null", key); errorList.add(message); return errorList; } String[] values = value.split("(?=(? * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.util.Optional; public class EnvUtils { public enum SystemParameters { CPEWF_TASK_ID, NETFLIX_ENV, NETFLIX_STACK } public static boolean isEnvironmentVariable(String test) { for (SystemParameters c : SystemParameters.values()) { if (c.name().equals(test)) { return true; } } String value = Optional.ofNullable(System.getProperty(test)).orElseGet(() -> System.getenv(test)); return value != null; } public static String getSystemParametersValue(String sysParam, String taskId) { if ("CPEWF_TASK_ID".equals(sysParam)) { return taskId; } String value = System.getenv(sysParam); if (value == null) { value = System.getProperty(sysParam); } return value; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/utils/ExternalPayloadStorage.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.io.InputStream; import com.netflix.conductor.common.run.ExternalStorageLocation; /** * Interface used to externalize the storage of large JSON payloads in workflow and task * input/output */ public interface ExternalPayloadStorage { enum Operation { READ, WRITE } enum PayloadType { WORKFLOW_INPUT, WORKFLOW_OUTPUT, TASK_INPUT, TASK_OUTPUT } /** * Obtain a uri used to store/access a json payload in external storage. * * @param operation the type of {@link Operation} to be performed with the uri * @param payloadType the {@link PayloadType} that is being accessed at the uri * @param path (optional) the relative path for which the external storage location object is to * be populated. If path is not specified, it will be computed and populated. * @return a {@link ExternalStorageLocation} object which contains the uri and the path for the * json payload */ ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path); /** * Obtain an uri used to store/access a json payload in external storage with deduplication of * data based on payloadBytes digest. * * @param operation the type of {@link Operation} to be performed with the uri * @param payloadType the {@link PayloadType} that is being accessed at the uri * @param path (optional) the relative path for which the external storage location object is to * be populated. If path is not specified, it will be computed and populated. * @param payloadBytes for calculating digest which is used for objectKey * @return a {@link ExternalStorageLocation} object which contains the uri and the path for the * json payload */ default ExternalStorageLocation getLocation( Operation operation, PayloadType payloadType, String path, byte[] payloadBytes) { return getLocation(operation, payloadType, path); } /** * Upload a json payload to the specified external storage location. * * @param path the location to which the object is to be uploaded * @param payload an {@link InputStream} containing the json payload which is to be uploaded * @param payloadSize the size of the json payload in bytes */ void upload(String path, InputStream payload, long payloadSize); /** * Download the json payload from the specified external storage location. * * @param path the location from where the object is to be downloaded * @return an {@link InputStream} of the json payload at the specified location */ InputStream download(String path); } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/utils/SummaryUtil.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.util.Map; import javax.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @Component public class SummaryUtil { private static final Logger logger = LoggerFactory.getLogger(SummaryUtil.class); private static final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); private static boolean isSummaryInputOutputJsonSerializationEnabled; @Value("${conductor.app.summary-input-output-json-serialization.enabled:false}") private boolean isJsonSerializationEnabled; @PostConstruct public void init() { isSummaryInputOutputJsonSerializationEnabled = isJsonSerializationEnabled; } /** * Serializes the Workflow or Task's Input/Output object by Java's toString (default), or by a * Json ObjectMapper (@see Configuration.isSummaryInputOutputJsonSerializationEnabled) * * @param object the Input or Output Object to serialize * @return the serialized string of the Input or Output object */ public static String serializeInputOutput(Map object) { if (!isSummaryInputOutputJsonSerializationEnabled) { return object.toString(); } try { return objectMapper.writeValueAsString(object); } catch (JsonProcessingException e) { logger.error( "The provided value ({}) could not be serialized as Json", object.toString(), e); throw new RuntimeException(e); } } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/utils/TaskUtils.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; public class TaskUtils { private static final String LOOP_TASK_DELIMITER = "__"; public static String appendIteration(String name, int iteration) { return name + LOOP_TASK_DELIMITER + iteration; } public static String getLoopOverTaskRefNameSuffix(int iteration) { return LOOP_TASK_DELIMITER + iteration; } public static String removeIterationFromTaskRefName(String referenceTaskName) { String[] tokens = referenceTaskName.split(TaskUtils.LOOP_TASK_DELIMITER); return tokens.length > 0 ? tokens[0] : referenceTaskName; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/validation/ErrorResponse.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.validation; import java.util.List; public class ErrorResponse { private int status; private String code; private String message; private String instance; private boolean retryable; private List validationErrors; public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public List getValidationErrors() { return validationErrors; } public void setValidationErrors(List validationErrors) { this.validationErrors = validationErrors; } public boolean isRetryable() { return retryable; } public void setRetryable(boolean retryable) { this.retryable = retryable; } public String getCode() { return code; } public void setCode(String code) { this.code = code; } public String getMessage() { return message; } public void setMessage(String message) { this.message = message; } public String getInstance() { return instance; } public void setInstance(String instance) { this.instance = instance; } } ================================================ FILE: common/src/main/java/com/netflix/conductor/common/validation/ValidationError.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.validation; import java.util.StringJoiner; /** Captures a validation error that can be returned in {@link ErrorResponse}. */ public class ValidationError { private String path; private String message; private String invalidValue; public ValidationError() {} public ValidationError(String path, String message, String invalidValue) { this.path = path; this.message = message; this.invalidValue = invalidValue; } public String getPath() { return path; } public String getMessage() { return message; } public String getInvalidValue() { return invalidValue; } public void setPath(String path) { this.path = path; } public void setMessage(String message) { this.message = message; } public void setInvalidValue(String invalidValue) { this.invalidValue = invalidValue; } @Override public String toString() { return new StringJoiner(", ", ValidationError.class.getSimpleName() + "[", "]") .add("path='" + path + "'") .add("message='" + message + "'") .add("invalidValue='" + invalidValue + "'") .toString(); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/config/TestObjectMapperConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.fasterxml.jackson.databind.ObjectMapper; /** Supplies the standard Conductor {@link ObjectMapper} for tests that need them. */ @Configuration public class TestObjectMapperConfiguration { @Bean public ObjectMapper testObjectMapper() { return new ObjectMapperProvider().getObjectMapper(); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/events/EventHandlerTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.events; import java.util.ArrayList; import java.util.List; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.junit.Test; import com.netflix.conductor.common.metadata.events.EventHandler; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class EventHandlerTest { @Test public void testWorkflowTaskName() { EventHandler taskDef = new EventHandler(); // name is null ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(taskDef); assertEquals(3, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("Missing event handler name")); assertTrue(validationErrors.contains("Missing event location")); assertTrue( validationErrors.contains( "No actions specified. Please specify at-least one action")); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/run/TaskSummaryTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.run; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.Task; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertNotNull; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class TaskSummaryTest { @Autowired private ObjectMapper objectMapper; @Test public void testJsonSerializing() throws Exception { Task task = new Task(); TaskSummary taskSummary = new TaskSummary(task); String json = objectMapper.writeValueAsString(taskSummary); TaskSummary read = objectMapper.readValue(json, TaskSummary.class); assertNotNull(read); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/tasks/TaskDefTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.tasks; import java.util.ArrayList; import java.util.List; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TaskDefTest { private Validator validator; @Before public void setup() { ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); this.validator = factory.getValidator(); } @Test public void test() { String name = "test1"; String description = "desc"; int retryCount = 10; int timeout = 100; TaskDef def = new TaskDef(name, description, retryCount, timeout); assertEquals(36_00, def.getResponseTimeoutSeconds()); assertEquals(name, def.getName()); assertEquals(description, def.getDescription()); assertEquals(retryCount, def.getRetryCount()); assertEquals(timeout, def.getTimeoutSeconds()); } @Test public void testTaskDef() { TaskDef taskDef = new TaskDef(); taskDef.setName("task1"); taskDef.setRetryCount(-1); taskDef.setTimeoutSeconds(1000); taskDef.setResponseTimeoutSeconds(1001); Set> result = validator.validate(taskDef); assertEquals(3, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "TaskDef: task1 responseTimeoutSeconds: 1001 must be less than timeoutSeconds: 1000")); assertTrue(validationErrors.contains("TaskDef retryCount: 0 must be >= 0")); assertTrue(validationErrors.contains("ownerEmail cannot be empty")); } @Test public void testTaskDefNameAndOwnerNotSet() { TaskDef taskDef = new TaskDef(); taskDef.setRetryCount(-1); taskDef.setTimeoutSeconds(1000); taskDef.setResponseTimeoutSeconds(1); Set> result = validator.validate(taskDef); assertEquals(3, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("TaskDef retryCount: 0 must be >= 0")); assertTrue(validationErrors.contains("TaskDef name cannot be null or empty")); assertTrue(validationErrors.contains("ownerEmail cannot be empty")); } @Test public void testTaskDefInvalidEmail() { TaskDef taskDef = new TaskDef(); taskDef.setName("test-task"); taskDef.setRetryCount(1); taskDef.setTimeoutSeconds(1000); taskDef.setResponseTimeoutSeconds(1); taskDef.setOwnerEmail("owner"); Set> result = validator.validate(taskDef); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("ownerEmail should be valid email address")); } @Test public void testTaskDefValidEmail() { TaskDef taskDef = new TaskDef(); taskDef.setName("test-task"); taskDef.setRetryCount(1); taskDef.setTimeoutSeconds(1000); taskDef.setResponseTimeoutSeconds(1); taskDef.setOwnerEmail("owner@test.com"); Set> result = validator.validate(taskDef); assertEquals(0, result.size()); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/tasks/TaskResultTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.tasks; import java.util.HashMap; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import static org.junit.Assert.assertEquals; public class TaskResultTest { private Task task; private TaskResult taskResult; @Before public void setUp() { task = new Task(); task.setWorkflowInstanceId("workflow-id"); task.setTaskId("task-id"); task.setReasonForIncompletion("reason"); task.setCallbackAfterSeconds(10); task.setWorkerId("worker-id"); task.setOutputData(new HashMap<>()); task.setExternalOutputPayloadStoragePath("externalOutput"); } @Test public void testCanceledTask() { task.setStatus(Task.Status.CANCELED); taskResult = new TaskResult(task); validateTaskResult(); assertEquals(TaskResult.Status.FAILED, taskResult.getStatus()); } @Test public void testCompletedWithErrorsTask() { task.setStatus(Task.Status.COMPLETED_WITH_ERRORS); taskResult = new TaskResult(task); validateTaskResult(); assertEquals(TaskResult.Status.FAILED, taskResult.getStatus()); } @Test public void testScheduledTask() { task.setStatus(Task.Status.SCHEDULED); taskResult = new TaskResult(task); validateTaskResult(); assertEquals(TaskResult.Status.IN_PROGRESS, taskResult.getStatus()); } @Test public void testCompltetedTask() { task.setStatus(Task.Status.COMPLETED); taskResult = new TaskResult(task); validateTaskResult(); assertEquals(TaskResult.Status.COMPLETED, taskResult.getStatus()); } private void validateTaskResult() { assertEquals(task.getWorkflowInstanceId(), taskResult.getWorkflowInstanceId()); assertEquals(task.getTaskId(), taskResult.getTaskId()); assertEquals(task.getReasonForIncompletion(), taskResult.getReasonForIncompletion()); assertEquals(task.getCallbackAfterSeconds(), taskResult.getCallbackAfterSeconds()); assertEquals(task.getWorkerId(), taskResult.getWorkerId()); assertEquals(task.getOutputData(), taskResult.getOutputData()); assertEquals( task.getExternalOutputPayloadStoragePath(), taskResult.getExternalOutputPayloadStoragePath()); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/tasks/TaskTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.tasks; import java.util.Arrays; import java.util.HashMap; import java.util.Set; import java.util.stream.Collectors; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.google.protobuf.Any; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class TaskTest { @Test public void test() { Task task = new Task(); task.setStatus(Status.FAILED); assertEquals(Status.FAILED, task.getStatus()); Set resultStatues = Arrays.stream(TaskResult.Status.values()) .map(Enum::name) .collect(Collectors.toSet()); for (Status status : Status.values()) { if (resultStatues.contains(status.name())) { TaskResult.Status trStatus = TaskResult.Status.valueOf(status.name()); assertEquals(status.name(), trStatus.name()); task = new Task(); task.setStatus(status); assertEquals(status, task.getStatus()); } } } @Test public void testTaskDefinitionIfAvailable() { Task task = new Task(); task.setStatus(Status.FAILED); assertEquals(Status.FAILED, task.getStatus()); assertNull(task.getWorkflowTask()); assertFalse(task.getTaskDefinition().isPresent()); WorkflowTask workflowTask = new WorkflowTask(); TaskDef taskDefinition = new TaskDef(); workflowTask.setTaskDefinition(taskDefinition); task.setWorkflowTask(workflowTask); assertTrue(task.getTaskDefinition().isPresent()); assertEquals(taskDefinition, task.getTaskDefinition().get()); } @Test public void testTaskQueueWaitTime() { Task task = new Task(); long currentTimeMillis = System.currentTimeMillis(); task.setScheduledTime(currentTimeMillis - 30_000); // 30 seconds ago task.setStartTime(currentTimeMillis - 25_000); long queueWaitTime = task.getQueueWaitTime(); assertEquals(5000L, queueWaitTime); task.setUpdateTime(currentTimeMillis - 20_000); task.setCallbackAfterSeconds(10); queueWaitTime = task.getQueueWaitTime(); assertTrue(queueWaitTime > 0); } @Test public void testDeepCopyTask() { final Task task = new Task(); // In order to avoid forgetting putting inside the copy method the newly added fields check // the number of declared fields. final int expectedTaskFieldsNumber = 40; final int declaredFieldsNumber = task.getClass().getDeclaredFields().length; assertEquals(expectedTaskFieldsNumber, declaredFieldsNumber); task.setCallbackAfterSeconds(111L); task.setCallbackFromWorker(false); task.setCorrelationId("correlation_id"); task.setInputData(new HashMap<>()); task.setOutputData(new HashMap<>()); task.setReferenceTaskName("ref_task_name"); task.setStartDelayInSeconds(1); task.setTaskDefName("task_def_name"); task.setTaskType("dummy_task_type"); task.setWorkflowInstanceId("workflowInstanceId"); task.setWorkflowType("workflowType"); task.setResponseTimeoutSeconds(11L); task.setStatus(Status.COMPLETED); task.setRetryCount(0); task.setPollCount(0); task.setTaskId("taskId"); task.setWorkflowTask(new WorkflowTask()); task.setDomain("domain"); task.setInputMessage(Any.getDefaultInstance()); task.setOutputMessage(Any.getDefaultInstance()); task.setRateLimitPerFrequency(11); task.setRateLimitFrequencyInSeconds(11); task.setExternalInputPayloadStoragePath("externalInputPayloadStoragePath"); task.setExternalOutputPayloadStoragePath("externalOutputPayloadStoragePath"); task.setWorkflowPriority(0); task.setIteration(1); task.setExecutionNameSpace("name_space"); task.setIsolationGroupId("groupId"); task.setStartTime(12L); task.setEndTime(20L); task.setScheduledTime(7L); task.setRetried(false); task.setReasonForIncompletion(""); task.setWorkerId(""); task.setSubWorkflowId(""); task.setSubworkflowChanged(false); final Task copy = task.deepCopy(); assertEquals(task, copy); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/utils/ConstraintParamUtilTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import static org.junit.Assert.assertEquals; public class ConstraintParamUtilTest { @Before public void before() { System.setProperty("NETFLIX_STACK", "test"); System.setProperty("NETFLIX_ENVIRONMENT", "test"); System.setProperty("TEST_ENV", "test"); } private WorkflowDef constructWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); return workflowDef; } @Test public void testExtractParamPathComponents() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithMissingEnvVariable() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${NETFLIX_STACK}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithValidEnvVariable() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithValidMap() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status}"); Map envInputParam = new HashMap<>(); envInputParam.put("packageId", "${workflow.input.packageId}"); envInputParam.put("taskId", "${CPEWF_TASK_ID}"); envInputParam.put("NETFLIX_STACK", "${NETFLIX_STACK}"); envInputParam.put("NETFLIX_ENVIRONMENT", "${NETFLIX_ENVIRONMENT}"); envInputParam.put("TEST_ENV", "${TEST_ENV}"); inputParam.put("env", envInputParam); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithInvalidEnv() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status}"); Map envInputParam = new HashMap<>(); envInputParam.put("packageId", "${workflow.input.packageId}"); envInputParam.put("taskId", "${CPEWF_TASK_ID}"); envInputParam.put("TEST_ENV1", "${TEST_ENV1}"); inputParam.put("env", envInputParam); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 1); } @Test public void testExtractParamPathComponentsWithInputParamEmpty() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", ""); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithListInputParamWithEmptyString() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", new String[] {""}); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithInputFieldWithSpace() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status sta}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 1); } @Test public void testExtractParamPathComponentsWithPredefineEnums() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("NETFLIX_ENV", "${CPEWF_TASK_ID}"); inputParam.put( "entryPoint", "/tools/pdfwatermarker_mux.py ${NETFLIX_ENV} ${CPEWF_TASK_ID} alpha"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } @Test public void testExtractParamPathComponentsWithEscapedChar() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "$${expression with spaces}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/utils/SummaryUtilTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.utils; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.jupiter.api.Assertions.assertEquals; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, SummaryUtilTest.SummaryUtilTestConfiguration.class }) @RunWith(SpringRunner.class) public class SummaryUtilTest { @Configuration static class SummaryUtilTestConfiguration { @Bean public SummaryUtil summaryUtil() { return new SummaryUtil(); } } @Autowired private ObjectMapper objectMapper; private Map testObject; @Before public void init() { Map child = new HashMap<>(); child.put("testStr", "childTestStr"); Map obj = new HashMap<>(); obj.put("testStr", "stringValue"); obj.put("testArray", new ArrayList<>(Arrays.asList(1, 2, 3))); obj.put("testObj", child); obj.put("testNull", null); testObject = obj; } @Test public void testSerializeInputOutput_defaultToString() throws Exception { new ApplicationContextRunner() .withPropertyValues( "conductor.app.summary-input-output-json-serialization.enabled:false") .withUserConfiguration(SummaryUtilTestConfiguration.class) .run( context -> { String serialized = SummaryUtil.serializeInputOutput(this.testObject); assertEquals( this.testObject.toString(), serialized, "The Java.toString() Serialization should match the serialized Test Object"); }); } @Test public void testSerializeInputOutput_jsonSerializationEnabled() throws Exception { new ApplicationContextRunner() .withPropertyValues( "conductor.app.summary-input-output-json-serialization.enabled:true") .withUserConfiguration(SummaryUtilTestConfiguration.class) .run( context -> { String serialized = SummaryUtil.serializeInputOutput(testObject); assertEquals( objectMapper.writeValueAsString(testObject), serialized, "The ObjectMapper Json Serialization should match the serialized Test Object"); }); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/workflow/SubWorkflowParamsTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.workflow; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class SubWorkflowParamsTest { @Autowired private ObjectMapper objectMapper; @Test public void testWorkflowTaskName() { SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); // name is null ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(subWorkflowParams); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("SubWorkflowParams name cannot be null")); assertTrue(validationErrors.contains("SubWorkflowParams name cannot be empty")); } @Test public void testWorkflowSetTaskToDomain() { SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); Map taskToDomain = new HashMap<>(); taskToDomain.put("unit", "test"); subWorkflowParams.setTaskToDomain(taskToDomain); assertEquals(taskToDomain, subWorkflowParams.getTaskToDomain()); } @Test(expected = IllegalArgumentException.class) public void testSetWorkflowDefinition() { SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("dummy-name"); subWorkflowParams.setWorkflowDefinition(new Object()); } @Test public void testGetWorkflowDef() { SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("dummy-name"); WorkflowDef def = new WorkflowDef(); def.setName("test_workflow"); def.setVersion(1); WorkflowTask task = new WorkflowTask(); task.setName("test_task"); task.setTaskReferenceName("t1"); def.getTasks().add(task); subWorkflowParams.setWorkflowDefinition(def); assertEquals(def, subWorkflowParams.getWorkflowDefinition()); assertEquals(def, subWorkflowParams.getWorkflowDef()); } @Test public void testWorkflowDefJson() throws Exception { SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("dummy-name"); WorkflowDef def = new WorkflowDef(); def.setName("test_workflow"); def.setVersion(1); WorkflowTask task = new WorkflowTask(); task.setName("test_task"); task.setTaskReferenceName("t1"); def.getTasks().add(task); subWorkflowParams.setWorkflowDefinition(def); objectMapper.enable(SerializationFeature.INDENT_OUTPUT); objectMapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); objectMapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); String serializedParams = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(subWorkflowParams); SubWorkflowParams deserializedParams = objectMapper.readValue(serializedParams, SubWorkflowParams.class); assertEquals(def, deserializedParams.getWorkflowDefinition()); assertEquals(def, deserializedParams.getWorkflowDef()); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/workflow/WorkflowDefValidatorTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.workflow; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class WorkflowDefValidatorTest { @Before public void before() { System.setProperty("NETFLIX_STACK", "test"); System.setProperty("NETFLIX_ENVIRONMENT", "test"); System.setProperty("TEST_ENV", "test"); } @Test public void testWorkflowDefConstraints() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(3, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("WorkflowDef name cannot be null or empty")); assertTrue(validationErrors.contains("WorkflowTask list cannot be empty")); assertTrue(validationErrors.contains("ownerEmail cannot be empty")); // assertTrue(validationErrors.contains("workflowDef schemaVersion: 1 should be >= 2")); } @Test public void testWorkflowDefConstraintsWithMultipleEnvVariable() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID}"); inputParam.put( "entryPoint", "${NETFLIX_ENVIRONMENT} ${NETFLIX_STACK} ${CPEWF_TASK_ID} ${workflow.input.status}"); workflowTask_1.setInputParameters(inputParam); WorkflowTask workflowTask_2 = new WorkflowTask(); workflowTask_2.setName("task_2"); workflowTask_2.setTaskReferenceName("task_2"); workflowTask_2.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam2 = new HashMap<>(); inputParam2.put("env", inputParam); workflowTask_2.setInputParameters(inputParam2); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); tasks.add(workflowTask_2); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowDefConstraintsSingleEnvVariable() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowDefConstraintsDualEnvVariable() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${NETFLIX_STACK}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowDefConstraintsWithMapAsInputParam() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${NETFLIX_STACK}"); Map envInputParam = new HashMap<>(); envInputParam.put("packageId", "${workflow.input.packageId}"); envInputParam.put("taskId", "${CPEWF_TASK_ID}"); envInputParam.put("NETFLIX_STACK", "${NETFLIX_STACK}"); envInputParam.put("NETFLIX_ENVIRONMENT", "${NETFLIX_ENVIRONMENT}"); inputParam.put("env", envInputParam); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowTaskInputParamInvalid() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask = new WorkflowTask(); // name is null workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", "${workflow.input.Space Value}"); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "key: blabla input parameter value: workflow.input.Space Value is not valid")); } @Test public void testWorkflowTaskEmptyStringInputParamValue() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask = new WorkflowTask(); // name is null workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", ""); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowTasklistInputParamWithEmptyString() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(2); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask = new WorkflowTask(); // name is null workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", ""); map.put("foo", new String[] {""}); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test public void testWorkflowSchemaVersion1() { WorkflowDef workflowDef = new WorkflowDef(); // name is null workflowDef.setSchemaVersion(3); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", ""); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("workflowDef schemaVersion: 2 is only supported")); } @Test public void testWorkflowOwnerInvalidEmail() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner"); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", ""); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("ownerEmail should be valid email address")); } @Test public void testWorkflowOwnerValidEmail() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test_env"); workflowDef.setOwnerEmail("owner@test.com"); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("t1"); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName("t1"); Map map = new HashMap<>(); map.put("blabla", ""); workflowTask.setInputParameters(map); workflowDef.getTasks().add(workflowTask); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } } ================================================ FILE: common/src/test/java/com/netflix/conductor/common/workflow/WorkflowTaskTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.workflow; import java.util.ArrayList; import java.util.List; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class WorkflowTaskTest { @Test public void test() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setWorkflowTaskType(TaskType.DECISION); assertNotNull(workflowTask.getType()); assertEquals(TaskType.DECISION.name(), workflowTask.getType()); workflowTask = new WorkflowTask(); workflowTask.setWorkflowTaskType(TaskType.SWITCH); assertNotNull(workflowTask.getType()); assertEquals(TaskType.SWITCH.name(), workflowTask.getType()); } @Test public void testOptional() { WorkflowTask task = new WorkflowTask(); assertFalse(task.isOptional()); task.setOptional(Boolean.FALSE); assertFalse(task.isOptional()); task.setOptional(Boolean.TRUE); assertTrue(task.isOptional()); } @Test public void testWorkflowTaskName() { WorkflowTask taskDef = new WorkflowTask(); // name is null ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(taskDef); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("WorkflowTask name cannot be empty or null")); assertTrue( validationErrors.contains( "WorkflowTask taskReferenceName name cannot be empty or null")); } } ================================================ FILE: core/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ apply plugin: 'groovy' dependencies { implementation project(':conductor-common') compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.boot:spring-boot-starter-validation' compileOnly 'org.springframework.retry:spring-retry' implementation "com.fasterxml.jackson.core:jackson-annotations:${revFasterXml}" implementation "com.fasterxml.jackson.core:jackson-databind:${revFasterXml}" implementation "commons-io:commons-io:${revCommonsIo}" implementation "com.google.protobuf:protobuf-java:${revProtoBuf}" implementation "org.apache.commons:commons-lang3" implementation "com.fasterxml.jackson.core:jackson-core:${revFasterXml}" implementation "com.spotify:completable-futures:${revSpotifyCompletableFutures}" implementation "com.jayway.jsonpath:json-path:${revJsonPath}" implementation "io.reactivex:rxjava:${revRxJava}" implementation "com.netflix.spectator:spectator-api:${revSpectator}" implementation "org.apache.bval:bval-jsr:${revBval}" implementation "com.github.ben-manes.caffeine:caffeine" implementation "org.openjdk.nashorn:nashorn-core:15.4" // JAXB is not bundled with Java 11, dependencies added explicitly // These are needed by Apache BVAL implementation "jakarta.xml.bind:jakarta.xml.bind-api:${revJAXB}" implementation "jakarta.activation:jakarta.activation-api:${revActivation}" // Only add it as a test dependency. The actual jaxb runtime provider is provided when building the server. testImplementation "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" testImplementation 'org.springframework.boot:spring-boot-starter-validation' testImplementation 'org.springframework.retry:spring-retry' testImplementation project(':conductor-common').sourceSets.test.output testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" testImplementation "org.junit.vintage:junit-vintage-engine" } ================================================ FILE: core/src/main/java/com/netflix/conductor/annotations/Audit.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations; import java.lang.annotation.Retention; import java.lang.annotation.Target; import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; /** Mark service for custom audit implementation */ @Target({TYPE}) @Retention(RUNTIME) public @interface Audit {} ================================================ FILE: core/src/main/java/com/netflix/conductor/annotations/Trace.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations; import java.lang.annotation.Retention; import java.lang.annotation.Target; import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; @Target({TYPE}) @Retention(RUNTIME) public @interface Trace {} ================================================ FILE: core/src/main/java/com/netflix/conductor/annotations/VisibleForTesting.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.annotations; import java.lang.annotation.*; /** * Annotates a program element that exists, or is more widely visible than otherwise necessary, only * for use in test code. */ @Retention(RetentionPolicy.CLASS) @Target({ElementType.FIELD, ElementType.TYPE, ElementType.METHOD}) @Documented public @interface VisibleForTesting {} ================================================ FILE: core/src/main/java/com/netflix/conductor/core/LifecycleAwareComponent.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.SmartLifecycle; public abstract class LifecycleAwareComponent implements SmartLifecycle { private volatile boolean running = false; private static final Logger LOGGER = LoggerFactory.getLogger(LifecycleAwareComponent.class); @Override public final void start() { running = true; LOGGER.info("{} started.", getClass().getSimpleName()); doStart(); } @Override public final void stop() { running = false; LOGGER.info("{} stopped.", getClass().getSimpleName()); doStop(); } @Override public final boolean isRunning() { return running; } public void doStart() {} public void doStop() {} } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/WorkflowContext.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core; /** Store the authentication context, app or username or both */ public class WorkflowContext { public static final ThreadLocal THREAD_LOCAL = InheritableThreadLocal.withInitial(() -> new WorkflowContext("", "")); private final String clientApp; private final String userName; public WorkflowContext(String clientApp) { this.clientApp = clientApp; this.userName = null; } public WorkflowContext(String clientApp, String userName) { this.clientApp = clientApp; this.userName = userName; } public static WorkflowContext get() { return THREAD_LOCAL.get(); } public static void set(WorkflowContext ctx) { THREAD_LOCAL.set(ctx); } public static void unset() { THREAD_LOCAL.remove(); } /** * @return the clientApp */ public String getClientApp() { return clientApp; } /** * @return the username */ public String getUserName() { return userName; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.config; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.stream.Collectors; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.events.EventQueueProvider; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.mapper.TaskMapper; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.listener.TaskStatusListener; import com.netflix.conductor.core.listener.TaskStatusListenerStub; import com.netflix.conductor.core.listener.WorkflowStatusListener; import com.netflix.conductor.core.listener.WorkflowStatusListenerStub; import com.netflix.conductor.core.storage.DummyPayloadStorage; import com.netflix.conductor.core.sync.Lock; import com.netflix.conductor.core.sync.noop.NoopLock; import static com.netflix.conductor.core.events.EventQueues.EVENT_QUEUE_PROVIDERS_QUALIFIER; import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; import static java.util.function.Function.identity; @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(ConductorProperties.class) public class ConductorCoreConfiguration { private static final Logger LOGGER = LoggerFactory.getLogger(ConductorCoreConfiguration.class); @ConditionalOnProperty( name = "conductor.workflow-execution-lock.type", havingValue = "noop_lock", matchIfMissing = true) @Bean public Lock provideLock() { return new NoopLock(); } @ConditionalOnProperty( name = "conductor.external-payload-storage.type", havingValue = "dummy", matchIfMissing = true) @Bean public ExternalPayloadStorage dummyExternalPayloadStorage() { LOGGER.info("Initialized dummy payload storage!"); return new DummyPayloadStorage(); } @ConditionalOnProperty( name = "conductor.workflow-status-listener.type", havingValue = "stub", matchIfMissing = true) @Bean public WorkflowStatusListener workflowStatusListener() { return new WorkflowStatusListenerStub(); } @ConditionalOnProperty( name = "conductor.task-status-listener.type", havingValue = "stub", matchIfMissing = true) @Bean public TaskStatusListener taskStatusListener() { return new TaskStatusListenerStub(); } @Bean public ExecutorService executorService(ConductorProperties conductorProperties) { ThreadFactory threadFactory = new BasicThreadFactory.Builder() .namingPattern("conductor-worker-%d") .daemon(true) .build(); return Executors.newFixedThreadPool( conductorProperties.getExecutorServiceMaxThreadCount(), threadFactory); } @Bean @Qualifier("taskMappersByTaskType") public Map getTaskMappers(List taskMappers) { return taskMappers.stream().collect(Collectors.toMap(TaskMapper::getTaskType, identity())); } @Bean @Qualifier(ASYNC_SYSTEM_TASKS_QUALIFIER) public Set asyncSystemTasks(Set allSystemTasks) { return allSystemTasks.stream() .filter(WorkflowSystemTask::isAsync) .collect(Collectors.toUnmodifiableSet()); } @Bean @Qualifier(EVENT_QUEUE_PROVIDERS_QUALIFIER) public Map getEventQueueProviders( List eventQueueProviders) { return eventQueueProviders.stream() .collect(Collectors.toMap(EventQueueProvider::getQueueType, identity())); } @Bean public RetryTemplate onTransientErrorRetryTemplate() { return RetryTemplate.builder() .retryOn(TransientException.class) .maxAttempts(3) .noBackoff() .build(); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/config/ConductorProperties.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.config; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DataSizeUnit; import org.springframework.boot.convert.DurationUnit; import org.springframework.util.unit.DataSize; import org.springframework.util.unit.DataUnit; @ConfigurationProperties("conductor.app") public class ConductorProperties { /** * Name of the stack within which the app is running. e.g. devint, testintg, staging, prod etc. */ private String stack = "test"; /** The id with the app has been registered. */ private String appId = "conductor"; /** The maximum number of threads to be allocated to the executor service threadpool. */ private int executorServiceMaxThreadCount = 50; /** The timeout duration to set when a workflow is pushed to the decider queue. */ @DurationUnit(ChronoUnit.SECONDS) private Duration workflowOffsetTimeout = Duration.ofSeconds(30); /** The number of threads to use to do background sweep on active workflows. */ private int sweeperThreadCount = Runtime.getRuntime().availableProcessors() * 2; /** The timeout (in milliseconds) for the polling of workflows to be swept. */ private Duration sweeperWorkflowPollTimeout = Duration.ofMillis(2000); /** The number of threads to configure the threadpool in the event processor. */ private int eventProcessorThreadCount = 2; /** Used to enable/disable the indexing of messages within event payloads. */ private boolean eventMessageIndexingEnabled = true; /** Used to enable/disable the indexing of event execution results. */ private boolean eventExecutionIndexingEnabled = true; /** Used to enable/disable the workflow execution lock. */ private boolean workflowExecutionLockEnabled = false; /** The time (in milliseconds) for which the lock is leased for. */ private Duration lockLeaseTime = Duration.ofMillis(60000); /** * The time (in milliseconds) for which the thread will block in an attempt to acquire the lock. */ private Duration lockTimeToTry = Duration.ofMillis(500); /** * The time (in seconds) that is used to consider if a worker is actively polling for a task. */ @DurationUnit(ChronoUnit.SECONDS) private Duration activeWorkerLastPollTimeout = Duration.ofSeconds(10); /** * The time (in seconds) for which a task execution will be postponed if being rate limited or * concurrent execution limited. */ @DurationUnit(ChronoUnit.SECONDS) private Duration taskExecutionPostponeDuration = Duration.ofSeconds(60); /** Used to enable/disable the indexing of task execution logs. */ private boolean taskExecLogIndexingEnabled = true; /** Used to enable/disable asynchronous indexing to elasticsearch. */ private boolean asyncIndexingEnabled = false; /** The number of threads to be used within the threadpool for system task workers. */ private int systemTaskWorkerThreadCount = Runtime.getRuntime().availableProcessors() * 2; /** * The interval (in seconds) after which a system task will be checked by the system task worker * for completion. */ @DurationUnit(ChronoUnit.SECONDS) private Duration systemTaskWorkerCallbackDuration = Duration.ofSeconds(30); /** * The interval (in milliseconds) at which system task queues will be polled by the system task * workers. */ private Duration systemTaskWorkerPollInterval = Duration.ofMillis(50); /** The namespace for the system task workers to provide instance level isolation. */ private String systemTaskWorkerExecutionNamespace = ""; /** * The number of threads to be used within the threadpool for system task workers in each * isolation group. */ private int isolatedSystemTaskWorkerThreadCount = 1; /** * The duration of workflow execution which qualifies a workflow as a short-running workflow * when async indexing to elasticsearch is enabled. */ @DurationUnit(ChronoUnit.SECONDS) private Duration asyncUpdateShortRunningWorkflowDuration = Duration.ofSeconds(30); /** * The delay with which short-running workflows will be updated in the elasticsearch index when * async indexing is enabled. */ @DurationUnit(ChronoUnit.SECONDS) private Duration asyncUpdateDelay = Duration.ofSeconds(60); /** * Used to control the validation for owner email field as mandatory within workflow and task * definitions. */ private boolean ownerEmailMandatory = true; /** * The number of threads to be usde in Scheduler used for polling events from multiple event * queues. By default, a thread count equal to the number of CPU cores is chosen. */ private int eventQueueSchedulerPollThreadCount = Runtime.getRuntime().availableProcessors(); /** The time interval (in milliseconds) at which the default event queues will be polled. */ private Duration eventQueuePollInterval = Duration.ofMillis(100); /** The number of messages to be polled from a default event queue in a single operation. */ private int eventQueuePollCount = 10; /** The timeout (in milliseconds) for the poll operation on the default event queue. */ private Duration eventQueueLongPollTimeout = Duration.ofMillis(1000); /** * The threshold of the workflow input payload size in KB beyond which the payload will be * stored in {@link com.netflix.conductor.common.utils.ExternalPayloadStorage}. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize workflowInputPayloadSizeThreshold = DataSize.ofKilobytes(5120L); /** * The maximum threshold of the workflow input payload size in KB beyond which input will be * rejected and the workflow will be marked as FAILED. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize maxWorkflowInputPayloadSizeThreshold = DataSize.ofKilobytes(10240L); /** * The threshold of the workflow output payload size in KB beyond which the payload will be * stored in {@link com.netflix.conductor.common.utils.ExternalPayloadStorage}. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize workflowOutputPayloadSizeThreshold = DataSize.ofKilobytes(5120L); /** * The maximum threshold of the workflow output payload size in KB beyond which output will be * rejected and the workflow will be marked as FAILED. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize maxWorkflowOutputPayloadSizeThreshold = DataSize.ofKilobytes(10240L); /** * The threshold of the task input payload size in KB beyond which the payload will be stored in * {@link com.netflix.conductor.common.utils.ExternalPayloadStorage}. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize taskInputPayloadSizeThreshold = DataSize.ofKilobytes(3072L); /** * The maximum threshold of the task input payload size in KB beyond which the task input will * be rejected and the task will be marked as FAILED_WITH_TERMINAL_ERROR. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize maxTaskInputPayloadSizeThreshold = DataSize.ofKilobytes(10240L); /** * The threshold of the task output payload size in KB beyond which the payload will be stored * in {@link com.netflix.conductor.common.utils.ExternalPayloadStorage}. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize taskOutputPayloadSizeThreshold = DataSize.ofKilobytes(3072L); /** * The maximum threshold of the task output payload size in KB beyond which the task input will * be rejected and the task will be marked as FAILED_WITH_TERMINAL_ERROR. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize maxTaskOutputPayloadSizeThreshold = DataSize.ofKilobytes(10240L); /** * The maximum threshold of the workflow variables payload size in KB beyond which the task * changes will be rejected and the task will be marked as FAILED_WITH_TERMINAL_ERROR. */ @DataSizeUnit(DataUnit.KILOBYTES) private DataSize maxWorkflowVariablesPayloadSizeThreshold = DataSize.ofKilobytes(256L); /** Used to limit the size of task execution logs. */ private int taskExecLogSizeLimit = 10; public String getStack() { return stack; } public void setStack(String stack) { this.stack = stack; } public String getAppId() { return appId; } public void setAppId(String appId) { this.appId = appId; } public int getExecutorServiceMaxThreadCount() { return executorServiceMaxThreadCount; } public void setExecutorServiceMaxThreadCount(int executorServiceMaxThreadCount) { this.executorServiceMaxThreadCount = executorServiceMaxThreadCount; } public Duration getWorkflowOffsetTimeout() { return workflowOffsetTimeout; } public void setWorkflowOffsetTimeout(Duration workflowOffsetTimeout) { this.workflowOffsetTimeout = workflowOffsetTimeout; } public int getSweeperThreadCount() { return sweeperThreadCount; } public void setSweeperThreadCount(int sweeperThreadCount) { this.sweeperThreadCount = sweeperThreadCount; } public Duration getSweeperWorkflowPollTimeout() { return sweeperWorkflowPollTimeout; } public void setSweeperWorkflowPollTimeout(Duration sweeperWorkflowPollTimeout) { this.sweeperWorkflowPollTimeout = sweeperWorkflowPollTimeout; } public int getEventProcessorThreadCount() { return eventProcessorThreadCount; } public void setEventProcessorThreadCount(int eventProcessorThreadCount) { this.eventProcessorThreadCount = eventProcessorThreadCount; } public boolean isEventMessageIndexingEnabled() { return eventMessageIndexingEnabled; } public void setEventMessageIndexingEnabled(boolean eventMessageIndexingEnabled) { this.eventMessageIndexingEnabled = eventMessageIndexingEnabled; } public boolean isEventExecutionIndexingEnabled() { return eventExecutionIndexingEnabled; } public void setEventExecutionIndexingEnabled(boolean eventExecutionIndexingEnabled) { this.eventExecutionIndexingEnabled = eventExecutionIndexingEnabled; } public boolean isWorkflowExecutionLockEnabled() { return workflowExecutionLockEnabled; } public void setWorkflowExecutionLockEnabled(boolean workflowExecutionLockEnabled) { this.workflowExecutionLockEnabled = workflowExecutionLockEnabled; } public Duration getLockLeaseTime() { return lockLeaseTime; } public void setLockLeaseTime(Duration lockLeaseTime) { this.lockLeaseTime = lockLeaseTime; } public Duration getLockTimeToTry() { return lockTimeToTry; } public void setLockTimeToTry(Duration lockTimeToTry) { this.lockTimeToTry = lockTimeToTry; } public Duration getActiveWorkerLastPollTimeout() { return activeWorkerLastPollTimeout; } public void setActiveWorkerLastPollTimeout(Duration activeWorkerLastPollTimeout) { this.activeWorkerLastPollTimeout = activeWorkerLastPollTimeout; } public Duration getTaskExecutionPostponeDuration() { return taskExecutionPostponeDuration; } public void setTaskExecutionPostponeDuration(Duration taskExecutionPostponeDuration) { this.taskExecutionPostponeDuration = taskExecutionPostponeDuration; } public boolean isTaskExecLogIndexingEnabled() { return taskExecLogIndexingEnabled; } public void setTaskExecLogIndexingEnabled(boolean taskExecLogIndexingEnabled) { this.taskExecLogIndexingEnabled = taskExecLogIndexingEnabled; } public boolean isAsyncIndexingEnabled() { return asyncIndexingEnabled; } public void setAsyncIndexingEnabled(boolean asyncIndexingEnabled) { this.asyncIndexingEnabled = asyncIndexingEnabled; } public int getSystemTaskWorkerThreadCount() { return systemTaskWorkerThreadCount; } public void setSystemTaskWorkerThreadCount(int systemTaskWorkerThreadCount) { this.systemTaskWorkerThreadCount = systemTaskWorkerThreadCount; } public Duration getSystemTaskWorkerCallbackDuration() { return systemTaskWorkerCallbackDuration; } public void setSystemTaskWorkerCallbackDuration(Duration systemTaskWorkerCallbackDuration) { this.systemTaskWorkerCallbackDuration = systemTaskWorkerCallbackDuration; } public Duration getSystemTaskWorkerPollInterval() { return systemTaskWorkerPollInterval; } public void setSystemTaskWorkerPollInterval(Duration systemTaskWorkerPollInterval) { this.systemTaskWorkerPollInterval = systemTaskWorkerPollInterval; } public String getSystemTaskWorkerExecutionNamespace() { return systemTaskWorkerExecutionNamespace; } public void setSystemTaskWorkerExecutionNamespace(String systemTaskWorkerExecutionNamespace) { this.systemTaskWorkerExecutionNamespace = systemTaskWorkerExecutionNamespace; } public int getIsolatedSystemTaskWorkerThreadCount() { return isolatedSystemTaskWorkerThreadCount; } public void setIsolatedSystemTaskWorkerThreadCount(int isolatedSystemTaskWorkerThreadCount) { this.isolatedSystemTaskWorkerThreadCount = isolatedSystemTaskWorkerThreadCount; } public Duration getAsyncUpdateShortRunningWorkflowDuration() { return asyncUpdateShortRunningWorkflowDuration; } public void setAsyncUpdateShortRunningWorkflowDuration( Duration asyncUpdateShortRunningWorkflowDuration) { this.asyncUpdateShortRunningWorkflowDuration = asyncUpdateShortRunningWorkflowDuration; } public Duration getAsyncUpdateDelay() { return asyncUpdateDelay; } public void setAsyncUpdateDelay(Duration asyncUpdateDelay) { this.asyncUpdateDelay = asyncUpdateDelay; } public boolean isOwnerEmailMandatory() { return ownerEmailMandatory; } public void setOwnerEmailMandatory(boolean ownerEmailMandatory) { this.ownerEmailMandatory = ownerEmailMandatory; } public int getEventQueueSchedulerPollThreadCount() { return eventQueueSchedulerPollThreadCount; } public void setEventQueueSchedulerPollThreadCount(int eventQueueSchedulerPollThreadCount) { this.eventQueueSchedulerPollThreadCount = eventQueueSchedulerPollThreadCount; } public Duration getEventQueuePollInterval() { return eventQueuePollInterval; } public void setEventQueuePollInterval(Duration eventQueuePollInterval) { this.eventQueuePollInterval = eventQueuePollInterval; } public int getEventQueuePollCount() { return eventQueuePollCount; } public void setEventQueuePollCount(int eventQueuePollCount) { this.eventQueuePollCount = eventQueuePollCount; } public Duration getEventQueueLongPollTimeout() { return eventQueueLongPollTimeout; } public void setEventQueueLongPollTimeout(Duration eventQueueLongPollTimeout) { this.eventQueueLongPollTimeout = eventQueueLongPollTimeout; } public DataSize getWorkflowInputPayloadSizeThreshold() { return workflowInputPayloadSizeThreshold; } public void setWorkflowInputPayloadSizeThreshold(DataSize workflowInputPayloadSizeThreshold) { this.workflowInputPayloadSizeThreshold = workflowInputPayloadSizeThreshold; } public DataSize getMaxWorkflowInputPayloadSizeThreshold() { return maxWorkflowInputPayloadSizeThreshold; } public void setMaxWorkflowInputPayloadSizeThreshold( DataSize maxWorkflowInputPayloadSizeThreshold) { this.maxWorkflowInputPayloadSizeThreshold = maxWorkflowInputPayloadSizeThreshold; } public DataSize getWorkflowOutputPayloadSizeThreshold() { return workflowOutputPayloadSizeThreshold; } public void setWorkflowOutputPayloadSizeThreshold(DataSize workflowOutputPayloadSizeThreshold) { this.workflowOutputPayloadSizeThreshold = workflowOutputPayloadSizeThreshold; } public DataSize getMaxWorkflowOutputPayloadSizeThreshold() { return maxWorkflowOutputPayloadSizeThreshold; } public void setMaxWorkflowOutputPayloadSizeThreshold( DataSize maxWorkflowOutputPayloadSizeThreshold) { this.maxWorkflowOutputPayloadSizeThreshold = maxWorkflowOutputPayloadSizeThreshold; } public DataSize getTaskInputPayloadSizeThreshold() { return taskInputPayloadSizeThreshold; } public void setTaskInputPayloadSizeThreshold(DataSize taskInputPayloadSizeThreshold) { this.taskInputPayloadSizeThreshold = taskInputPayloadSizeThreshold; } public DataSize getMaxTaskInputPayloadSizeThreshold() { return maxTaskInputPayloadSizeThreshold; } public void setMaxTaskInputPayloadSizeThreshold(DataSize maxTaskInputPayloadSizeThreshold) { this.maxTaskInputPayloadSizeThreshold = maxTaskInputPayloadSizeThreshold; } public DataSize getTaskOutputPayloadSizeThreshold() { return taskOutputPayloadSizeThreshold; } public void setTaskOutputPayloadSizeThreshold(DataSize taskOutputPayloadSizeThreshold) { this.taskOutputPayloadSizeThreshold = taskOutputPayloadSizeThreshold; } public DataSize getMaxTaskOutputPayloadSizeThreshold() { return maxTaskOutputPayloadSizeThreshold; } public void setMaxTaskOutputPayloadSizeThreshold(DataSize maxTaskOutputPayloadSizeThreshold) { this.maxTaskOutputPayloadSizeThreshold = maxTaskOutputPayloadSizeThreshold; } public DataSize getMaxWorkflowVariablesPayloadSizeThreshold() { return maxWorkflowVariablesPayloadSizeThreshold; } public void setMaxWorkflowVariablesPayloadSizeThreshold( DataSize maxWorkflowVariablesPayloadSizeThreshold) { this.maxWorkflowVariablesPayloadSizeThreshold = maxWorkflowVariablesPayloadSizeThreshold; } public int getTaskExecLogSizeLimit() { return taskExecLogSizeLimit; } public void setTaskExecLogSizeLimit(int taskExecLogSizeLimit) { this.taskExecLogSizeLimit = taskExecLogSizeLimit; } /** * @return Returns all the configurations in a map. */ public Map getAll() { Map map = new HashMap<>(); Properties props = System.getProperties(); props.forEach((key, value) -> map.put(key.toString(), value)); return map; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/config/SchedulerConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.config; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.SchedulingConfigurer; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.scheduling.config.ScheduledTaskRegistrar; import rx.Scheduler; import rx.schedulers.Schedulers; @Configuration(proxyBeanMethods = false) @EnableScheduling @EnableAsync public class SchedulerConfiguration implements SchedulingConfigurer { public static final String SWEEPER_EXECUTOR_NAME = "WorkflowSweeperExecutor"; /** * Used by some {@link com.netflix.conductor.core.events.queue.ObservableQueue} implementations. * * @see com.netflix.conductor.core.events.queue.ConductorObservableQueue */ @Bean public Scheduler scheduler(ConductorProperties properties) { ThreadFactory threadFactory = new BasicThreadFactory.Builder() .namingPattern("event-queue-poll-scheduler-thread-%d") .build(); Executor executorService = Executors.newFixedThreadPool( properties.getEventQueueSchedulerPollThreadCount(), threadFactory); return Schedulers.from(executorService); } @Bean(SWEEPER_EXECUTOR_NAME) public Executor sweeperExecutor(ConductorProperties properties) { if (properties.getSweeperThreadCount() <= 0) { throw new IllegalStateException( "conductor.app.sweeper-thread-count must be greater than 0."); } ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("sweeper-thread-%d").build(); return Executors.newFixedThreadPool(properties.getSweeperThreadCount(), threadFactory); } @Override public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler(); threadPoolTaskScheduler.setPoolSize(3); // equal to the number of scheduled jobs threadPoolTaskScheduler.setThreadNamePrefix("scheduled-task-pool-"); threadPoolTaskScheduler.initialize(); taskRegistrar.setTaskScheduler(threadPoolTaskScheduler); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/dal/ExecutionDAOFacade.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.dal; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.annotation.PreDestroy; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.dao.*; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; /** * Service that acts as a facade for accessing execution data from the {@link ExecutionDAO}, {@link * RateLimitingDAO} and {@link IndexDAO} storage layers */ @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") @Component public class ExecutionDAOFacade { private static final Logger LOGGER = LoggerFactory.getLogger(ExecutionDAOFacade.class); private static final String ARCHIVED_FIELD = "archived"; private static final String RAW_JSON_FIELD = "rawJSON"; private final ExecutionDAO executionDAO; private final QueueDAO queueDAO; private final IndexDAO indexDAO; private final RateLimitingDAO rateLimitingDao; private final ConcurrentExecutionLimitDAO concurrentExecutionLimitDAO; private final PollDataDAO pollDataDAO; private final ObjectMapper objectMapper; private final ConductorProperties properties; private final ExternalPayloadStorageUtils externalPayloadStorageUtils; private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; public ExecutionDAOFacade( ExecutionDAO executionDAO, QueueDAO queueDAO, IndexDAO indexDAO, RateLimitingDAO rateLimitingDao, ConcurrentExecutionLimitDAO concurrentExecutionLimitDAO, PollDataDAO pollDataDAO, ObjectMapper objectMapper, ConductorProperties properties, ExternalPayloadStorageUtils externalPayloadStorageUtils) { this.executionDAO = executionDAO; this.queueDAO = queueDAO; this.indexDAO = indexDAO; this.rateLimitingDao = rateLimitingDao; this.concurrentExecutionLimitDAO = concurrentExecutionLimitDAO; this.pollDataDAO = pollDataDAO; this.objectMapper = objectMapper; this.properties = properties; this.externalPayloadStorageUtils = externalPayloadStorageUtils; this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( 4, (runnable, executor) -> { LOGGER.warn( "Request {} to delay updating index dropped in executor {}", runnable, executor); Monitors.recordDiscardedIndexingCount("delayQueue"); }); this.scheduledThreadPoolExecutor.setRemoveOnCancelPolicy(true); } @PreDestroy public void shutdownExecutorService() { try { LOGGER.info("Gracefully shutdown executor service"); scheduledThreadPoolExecutor.shutdown(); if (scheduledThreadPoolExecutor.awaitTermination( properties.getAsyncUpdateDelay().getSeconds(), TimeUnit.SECONDS)) { LOGGER.debug("tasks completed, shutting down"); } else { LOGGER.warn( "Forcing shutdown after waiting for {} seconds", properties.getAsyncUpdateDelay()); scheduledThreadPoolExecutor.shutdownNow(); } } catch (InterruptedException ie) { LOGGER.warn( "Shutdown interrupted, invoking shutdownNow on scheduledThreadPoolExecutor for delay queue"); scheduledThreadPoolExecutor.shutdownNow(); Thread.currentThread().interrupt(); } } public WorkflowModel getWorkflowModel(String workflowId, boolean includeTasks) { WorkflowModel workflowModel = getWorkflowModelFromDataStore(workflowId, includeTasks); populateWorkflowAndTaskPayloadData(workflowModel); return workflowModel; } /** * Fetches the {@link Workflow} object from the data store given the id. Attempts to fetch from * {@link ExecutionDAO} first, if not found, attempts to fetch from {@link IndexDAO}. * * @param workflowId the id of the workflow to be fetched * @param includeTasks if true, fetches the {@link Task} data in the workflow. * @return the {@link Workflow} object * @throws NotFoundException no such {@link Workflow} is found. * @throws TransientException parsing the {@link Workflow} object fails. */ public Workflow getWorkflow(String workflowId, boolean includeTasks) { return getWorkflowModelFromDataStore(workflowId, includeTasks).toWorkflow(); } private WorkflowModel getWorkflowModelFromDataStore(String workflowId, boolean includeTasks) { WorkflowModel workflow = executionDAO.getWorkflow(workflowId, includeTasks); if (workflow == null) { LOGGER.debug("Workflow {} not found in executionDAO, checking indexDAO", workflowId); String json = indexDAO.get(workflowId, RAW_JSON_FIELD); if (json == null) { String errorMsg = String.format("No such workflow found by id: %s", workflowId); LOGGER.error(errorMsg); throw new NotFoundException(errorMsg); } try { workflow = objectMapper.readValue(json, WorkflowModel.class); if (!includeTasks) { workflow.getTasks().clear(); } } catch (IOException e) { String errorMsg = String.format("Error reading workflow: %s", workflowId); LOGGER.error(errorMsg); throw new TransientException(errorMsg, e); } } return workflow; } /** * Retrieve all workflow executions with the given correlationId and workflow type Uses the * {@link IndexDAO} to search across workflows if the {@link ExecutionDAO} cannot perform * searches across workflows. * * @param workflowName, workflow type to be queried * @param correlationId the correlation id to be queried * @param includeTasks if true, fetches the {@link Task} data within the workflows * @return the list of {@link Workflow} executions matching the correlationId */ public List getWorkflowsByCorrelationId( String workflowName, String correlationId, boolean includeTasks) { if (!executionDAO.canSearchAcrossWorkflows()) { String query = "correlationId='" + correlationId + "' AND workflowType='" + workflowName + "'"; SearchResult result = indexDAO.searchWorkflows(query, "*", 0, 1000, null); return result.getResults().stream() .parallel() .map( workflowId -> { try { return getWorkflow(workflowId, includeTasks); } catch (NotFoundException e) { // This might happen when the workflow archival failed and the // workflow was removed from primary datastore LOGGER.error( "Error getting the workflow: {} for correlationId: {} from datastore/index", workflowId, correlationId, e); return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); } return executionDAO .getWorkflowsByCorrelationId(workflowName, correlationId, includeTasks) .stream() .map(WorkflowModel::toWorkflow) .collect(Collectors.toList()); } public List getWorkflowsByName(String workflowName, Long startTime, Long endTime) { return executionDAO.getWorkflowsByType(workflowName, startTime, endTime).stream() .map(WorkflowModel::toWorkflow) .collect(Collectors.toList()); } public List getPendingWorkflowsByName(String workflowName, int version) { return executionDAO.getPendingWorkflowsByType(workflowName, version).stream() .map(WorkflowModel::toWorkflow) .collect(Collectors.toList()); } public List getRunningWorkflowIds(String workflowName, int version) { return executionDAO.getRunningWorkflowIds(workflowName, version); } public long getPendingWorkflowCount(String workflowName) { return executionDAO.getPendingWorkflowCount(workflowName); } /** * Creates a new workflow in the data store * * @param workflowModel the workflow to be created * @return the id of the created workflow */ public String createWorkflow(WorkflowModel workflowModel) { externalizeWorkflowData(workflowModel); executionDAO.createWorkflow(workflowModel); // Add to decider queue queueDAO.push( DECIDER_QUEUE, workflowModel.getWorkflowId(), workflowModel.getPriority(), properties.getWorkflowOffsetTimeout().getSeconds()); if (properties.isAsyncIndexingEnabled()) { indexDAO.asyncIndexWorkflow(new WorkflowSummary(workflowModel.toWorkflow())); } else { indexDAO.indexWorkflow(new WorkflowSummary(workflowModel.toWorkflow())); } return workflowModel.getWorkflowId(); } private void externalizeTaskData(TaskModel taskModel) { externalPayloadStorageUtils.verifyAndUpload( taskModel, ExternalPayloadStorage.PayloadType.TASK_INPUT); externalPayloadStorageUtils.verifyAndUpload( taskModel, ExternalPayloadStorage.PayloadType.TASK_OUTPUT); } private void externalizeWorkflowData(WorkflowModel workflowModel) { externalPayloadStorageUtils.verifyAndUpload( workflowModel, ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT); externalPayloadStorageUtils.verifyAndUpload( workflowModel, ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT); } /** * Updates the given workflow in the data store * * @param workflowModel the workflow tp be updated * @return the id of the updated workflow */ public String updateWorkflow(WorkflowModel workflowModel) { workflowModel.setUpdatedTime(System.currentTimeMillis()); if (workflowModel.getStatus().isTerminal()) { workflowModel.setEndTime(System.currentTimeMillis()); } externalizeWorkflowData(workflowModel); executionDAO.updateWorkflow(workflowModel); if (properties.isAsyncIndexingEnabled()) { if (workflowModel.getStatus().isTerminal() && workflowModel.getEndTime() - workflowModel.getCreateTime() < properties.getAsyncUpdateShortRunningWorkflowDuration().toMillis()) { final String workflowId = workflowModel.getWorkflowId(); DelayWorkflowUpdate delayWorkflowUpdate = new DelayWorkflowUpdate(workflowId); LOGGER.debug( "Delayed updating workflow: {} in the index by {} seconds", workflowId, properties.getAsyncUpdateDelay()); scheduledThreadPoolExecutor.schedule( delayWorkflowUpdate, properties.getAsyncUpdateDelay().getSeconds(), TimeUnit.SECONDS); Monitors.recordWorkerQueueSize( "delayQueue", scheduledThreadPoolExecutor.getQueue().size()); } else { indexDAO.asyncIndexWorkflow(new WorkflowSummary(workflowModel.toWorkflow())); } if (workflowModel.getStatus().isTerminal()) { workflowModel .getTasks() .forEach( taskModel -> indexDAO.asyncIndexTask( new TaskSummary(taskModel.toTask()))); } } else { indexDAO.indexWorkflow(new WorkflowSummary(workflowModel.toWorkflow())); } return workflowModel.getWorkflowId(); } public void removeFromPendingWorkflow(String workflowType, String workflowId) { executionDAO.removeFromPendingWorkflow(workflowType, workflowId); } /** * Removes the workflow from the data store. * * @param workflowId the id of the workflow to be removed * @param archiveWorkflow if true, the workflow and associated tasks will be archived in the * {@link IndexDAO} after removal from {@link ExecutionDAO}. */ public void removeWorkflow(String workflowId, boolean archiveWorkflow) { WorkflowModel workflow = getWorkflowModelFromDataStore(workflowId, true); executionDAO.removeWorkflow(workflowId); try { removeWorkflowIndex(workflow, archiveWorkflow); } catch (JsonProcessingException e) { throw new TransientException("Workflow can not be serialized to json", e); } workflow.getTasks() .forEach( task -> { try { removeTaskIndex(workflow, task, archiveWorkflow); } catch (JsonProcessingException e) { throw new TransientException( String.format( "Task %s of workflow %s can not be serialized to json", task.getTaskId(), workflow.getWorkflowId()), e); } try { queueDAO.remove(QueueUtils.getQueueName(task), task.getTaskId()); } catch (Exception e) { LOGGER.info( "Error removing task: {} of workflow: {} from {} queue", workflowId, task.getTaskId(), QueueUtils.getQueueName(task), e); } }); try { queueDAO.remove(DECIDER_QUEUE, workflowId); } catch (Exception e) { LOGGER.info("Error removing workflow: {} from decider queue", workflowId, e); } } private void removeWorkflowIndex(WorkflowModel workflow, boolean archiveWorkflow) throws JsonProcessingException { if (archiveWorkflow) { if (workflow.getStatus().isTerminal()) { // Only allow archival if workflow is in terminal state // DO NOT archive async, since if archival errors out, workflow data will be lost indexDAO.updateWorkflow( workflow.getWorkflowId(), new String[] {RAW_JSON_FIELD, ARCHIVED_FIELD}, new Object[] {objectMapper.writeValueAsString(workflow), true}); } else { throw new IllegalArgumentException( String.format( "Cannot archive workflow: %s with status: %s", workflow.getWorkflowId(), workflow.getStatus())); } } else { // Not archiving, also remove workflow from index indexDAO.asyncRemoveWorkflow(workflow.getWorkflowId()); } } public void removeWorkflowWithExpiry( String workflowId, boolean archiveWorkflow, int ttlSeconds) { try { WorkflowModel workflow = getWorkflowModelFromDataStore(workflowId, true); removeWorkflowIndex(workflow, archiveWorkflow); // remove workflow from DAO with TTL executionDAO.removeWorkflowWithExpiry(workflowId, ttlSeconds); } catch (Exception e) { Monitors.recordDaoError("executionDao", "removeWorkflow"); throw new TransientException("Error removing workflow: " + workflowId, e); } } /** * Reset the workflow state by removing from the {@link ExecutionDAO} and removing this workflow * from the {@link IndexDAO}. * * @param workflowId the workflow id to be reset */ public void resetWorkflow(String workflowId) { getWorkflowModelFromDataStore(workflowId, true); executionDAO.removeWorkflow(workflowId); try { if (properties.isAsyncIndexingEnabled()) { indexDAO.asyncRemoveWorkflow(workflowId); } else { indexDAO.removeWorkflow(workflowId); } } catch (Exception e) { throw new TransientException("Error resetting workflow state: " + workflowId, e); } } public List createTasks(List tasks) { tasks.forEach(this::externalizeTaskData); return executionDAO.createTasks(tasks); } public List getTasksForWorkflow(String workflowId) { return executionDAO.getTasksForWorkflow(workflowId).stream() .map(TaskModel::toTask) .collect(Collectors.toList()); } public TaskModel getTaskModel(String taskId) { TaskModel taskModel = getTaskFromDatastore(taskId); if (taskModel != null) { populateTaskData(taskModel); } return taskModel; } public Task getTask(String taskId) { TaskModel taskModel = getTaskFromDatastore(taskId); if (taskModel != null) { return taskModel.toTask(); } return null; } private TaskModel getTaskFromDatastore(String taskId) { return executionDAO.getTask(taskId); } public List getTasksByName(String taskName, String startKey, int count) { return executionDAO.getTasks(taskName, startKey, count).stream() .map(TaskModel::toTask) .collect(Collectors.toList()); } public List getPendingTasksForTaskType(String taskType) { return executionDAO.getPendingTasksForTaskType(taskType).stream() .map(TaskModel::toTask) .collect(Collectors.toList()); } public long getInProgressTaskCount(String taskDefName) { return executionDAO.getInProgressTaskCount(taskDefName); } /** * Sets the update time for the task. Sets the end time for the task (if task is in terminal * state and end time is not set). Updates the task in the {@link ExecutionDAO} first, then * stores it in the {@link IndexDAO}. * * @param taskModel the task to be updated in the data store * @throws TransientException if the {@link IndexDAO} or {@link ExecutionDAO} operations fail. * @throws com.netflix.conductor.core.exception.NonTransientException if the externalization of * payload fails. */ public void updateTask(TaskModel taskModel) { if (taskModel.getStatus() != null) { if (!taskModel.getStatus().isTerminal() || (taskModel.getStatus().isTerminal() && taskModel.getUpdateTime() == 0)) { taskModel.setUpdateTime(System.currentTimeMillis()); } if (taskModel.getStatus().isTerminal() && taskModel.getEndTime() == 0) { taskModel.setEndTime(System.currentTimeMillis()); } } externalizeTaskData(taskModel); executionDAO.updateTask(taskModel); try { /* * Indexing a task for every update adds a lot of volume. That is ok but if async indexing * is enabled and tasks are stored in memory until a block has completed, we would lose a lot * of tasks on a system failure. So only index for each update if async indexing is not enabled. * If it *is* enabled, tasks will be indexed only when a workflow is in terminal state. */ if (!properties.isAsyncIndexingEnabled()) { indexDAO.indexTask(new TaskSummary(taskModel.toTask())); } } catch (TerminateWorkflowException e) { // re-throw it so we can terminate the workflow throw e; } catch (Exception e) { String errorMsg = String.format( "Error updating task: %s in workflow: %s", taskModel.getTaskId(), taskModel.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } public void updateTasks(List tasks) { tasks.forEach(this::updateTask); } public void removeTask(String taskId) { executionDAO.removeTask(taskId); } private void removeTaskIndex(WorkflowModel workflow, TaskModel task, boolean archiveTask) throws JsonProcessingException { if (archiveTask) { if (task.getStatus().isTerminal()) { // Only allow archival if task is in terminal state // DO NOT archive async, since if archival errors out, task data will be lost indexDAO.updateTask( workflow.getWorkflowId(), task.getTaskId(), new String[] {ARCHIVED_FIELD}, new Object[] {true}); } else { throw new IllegalArgumentException( String.format( "Cannot archive task: %s of workflow: %s with status: %s", task.getTaskId(), workflow.getWorkflowId(), task.getStatus())); } } else { // Not archiving, remove task from index indexDAO.asyncRemoveTask(workflow.getWorkflowId(), task.getTaskId()); } } public void extendLease(TaskModel taskModel) { taskModel.setUpdateTime(System.currentTimeMillis()); executionDAO.updateTask(taskModel); } public List getTaskPollData(String taskName) { return pollDataDAO.getPollData(taskName); } public List getAllPollData() { return pollDataDAO.getAllPollData(); } public PollData getTaskPollDataByDomain(String taskName, String domain) { try { return pollDataDAO.getPollData(taskName, domain); } catch (Exception e) { LOGGER.error( "Error fetching pollData for task: '{}', domain: '{}'", taskName, domain, e); return null; } } public void updateTaskLastPoll(String taskName, String domain, String workerId) { try { pollDataDAO.updateLastPollData(taskName, domain, workerId); } catch (Exception e) { LOGGER.error( "Error updating PollData for task: {} in domain: {} from worker: {}", taskName, domain, workerId, e); Monitors.error(this.getClass().getCanonicalName(), "updateTaskLastPoll"); } } /** * Save the {@link EventExecution} to the data store Saves to {@link ExecutionDAO} first, if * this succeeds then saves to the {@link IndexDAO}. * * @param eventExecution the {@link EventExecution} to be saved * @return true if save succeeds, false otherwise. */ public boolean addEventExecution(EventExecution eventExecution) { boolean added = executionDAO.addEventExecution(eventExecution); if (added) { indexEventExecution(eventExecution); } return added; } public void updateEventExecution(EventExecution eventExecution) { executionDAO.updateEventExecution(eventExecution); indexEventExecution(eventExecution); } private void indexEventExecution(EventExecution eventExecution) { if (properties.isEventExecutionIndexingEnabled()) { if (properties.isAsyncIndexingEnabled()) { indexDAO.asyncAddEventExecution(eventExecution); } else { indexDAO.addEventExecution(eventExecution); } } } public void removeEventExecution(EventExecution eventExecution) { executionDAO.removeEventExecution(eventExecution); } public boolean exceedsInProgressLimit(TaskModel task) { return concurrentExecutionLimitDAO.exceedsLimit(task); } public boolean exceedsRateLimitPerFrequency(TaskModel task, TaskDef taskDef) { return rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef); } public void addTaskExecLog(List logs) { if (properties.isTaskExecLogIndexingEnabled() && !logs.isEmpty()) { Monitors.recordTaskExecLogSize(logs.size()); int taskExecLogSizeLimit = properties.getTaskExecLogSizeLimit(); if (logs.size() > taskExecLogSizeLimit) { LOGGER.warn( "Task Execution log size: {} for taskId: {} exceeds the limit: {}", logs.size(), logs.get(0).getTaskId(), taskExecLogSizeLimit); logs = logs.stream().limit(taskExecLogSizeLimit).collect(Collectors.toList()); } if (properties.isAsyncIndexingEnabled()) { indexDAO.asyncAddTaskExecutionLogs(logs); } else { indexDAO.addTaskExecutionLogs(logs); } } } public void addMessage(String queue, Message message) { if (properties.isAsyncIndexingEnabled()) { indexDAO.asyncAddMessage(queue, message); } else { indexDAO.addMessage(queue, message); } } public SearchResult searchWorkflows( String query, String freeText, int start, int count, List sort) { return indexDAO.searchWorkflows(query, freeText, start, count, sort); } public SearchResult searchWorkflowSummary( String query, String freeText, int start, int count, List sort) { return indexDAO.searchWorkflowSummary(query, freeText, start, count, sort); } public SearchResult searchTasks( String query, String freeText, int start, int count, List sort) { return indexDAO.searchTasks(query, freeText, start, count, sort); } public SearchResult searchTaskSummary( String query, String freeText, int start, int count, List sort) { return indexDAO.searchTaskSummary(query, freeText, start, count, sort); } public List getTaskExecutionLogs(String taskId) { return properties.isTaskExecLogIndexingEnabled() ? indexDAO.getTaskExecutionLogs(taskId) : Collections.emptyList(); } /** * Populates the workflow input data and the tasks input/output data if stored in external * payload storage. * * @param workflowModel the workflowModel for which the payload data needs to be populated from * external storage (if applicable) */ public void populateWorkflowAndTaskPayloadData(WorkflowModel workflowModel) { if (StringUtils.isNotBlank(workflowModel.getExternalInputPayloadStoragePath())) { Map workflowInputParams = externalPayloadStorageUtils.downloadPayload( workflowModel.getExternalInputPayloadStoragePath()); Monitors.recordExternalPayloadStorageUsage( workflowModel.getWorkflowName(), ExternalPayloadStorage.Operation.READ.toString(), ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT.toString()); workflowModel.internalizeInput(workflowInputParams); } if (StringUtils.isNotBlank(workflowModel.getExternalOutputPayloadStoragePath())) { Map workflowOutputParams = externalPayloadStorageUtils.downloadPayload( workflowModel.getExternalOutputPayloadStoragePath()); Monitors.recordExternalPayloadStorageUsage( workflowModel.getWorkflowName(), ExternalPayloadStorage.Operation.READ.toString(), ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT.toString()); workflowModel.internalizeOutput(workflowOutputParams); } workflowModel.getTasks().forEach(this::populateTaskData); } public void populateTaskData(TaskModel taskModel) { if (StringUtils.isNotBlank(taskModel.getExternalOutputPayloadStoragePath())) { Map outputData = externalPayloadStorageUtils.downloadPayload( taskModel.getExternalOutputPayloadStoragePath()); taskModel.internalizeOutput(outputData); Monitors.recordExternalPayloadStorageUsage( taskModel.getTaskDefName(), ExternalPayloadStorage.Operation.READ.toString(), ExternalPayloadStorage.PayloadType.TASK_OUTPUT.toString()); } if (StringUtils.isNotBlank(taskModel.getExternalInputPayloadStoragePath())) { Map inputData = externalPayloadStorageUtils.downloadPayload( taskModel.getExternalInputPayloadStoragePath()); taskModel.internalizeInput(inputData); Monitors.recordExternalPayloadStorageUsage( taskModel.getTaskDefName(), ExternalPayloadStorage.Operation.READ.toString(), ExternalPayloadStorage.PayloadType.TASK_INPUT.toString()); } } class DelayWorkflowUpdate implements Runnable { private final String workflowId; DelayWorkflowUpdate(String workflowId) { this.workflowId = workflowId; } @Override public void run() { try { WorkflowModel workflowModel = executionDAO.getWorkflow(workflowId, false); indexDAO.asyncIndexWorkflow(new WorkflowSummary(workflowModel.toWorkflow())); } catch (Exception e) { LOGGER.error("Unable to update workflow: {}", workflowId, e); } } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/event/WorkflowCreationEvent.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.event; import java.io.Serializable; import com.netflix.conductor.core.execution.StartWorkflowInput; public class WorkflowCreationEvent implements Serializable { private final StartWorkflowInput startWorkflowInput; public WorkflowCreationEvent(StartWorkflowInput startWorkflowInput) { this.startWorkflowInput = startWorkflowInput; } public StartWorkflowInput getStartWorkflowInput() { return startWorkflowInput; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/event/WorkflowEvaluationEvent.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.event; import java.io.Serializable; import com.netflix.conductor.model.WorkflowModel; public final class WorkflowEvaluationEvent implements Serializable { private final WorkflowModel workflowModel; public WorkflowEvaluationEvent(WorkflowModel workflowModel) { this.workflowModel = workflowModel; } public WorkflowModel getWorkflowModel() { return workflowModel; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/ActionProcessor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.Map; import com.netflix.conductor.common.metadata.events.EventHandler; public interface ActionProcessor { Map execute( EventHandler.Action action, Object payloadObject, String event, String messageId); } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/DefaultEventProcessor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.retry.support.RetryTemplate; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.events.EventExecution.Status; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.events.EventHandler.Action; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.utils.JsonUtils; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.service.ExecutionService; import com.netflix.conductor.service.MetadataService; import com.fasterxml.jackson.databind.ObjectMapper; import com.spotify.futures.CompletableFutures; import static com.netflix.conductor.core.utils.Utils.isTransientException; /** * Event Processor is used to dispatch actions configured in the event handlers, based on incoming * events to the event queues. * *

Set conductor.default-event-processor.enabled=false to disable event processing. */ @Component @ConditionalOnProperty( name = "conductor.default-event-processor.enabled", havingValue = "true", matchIfMissing = true) public class DefaultEventProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultEventProcessor.class); private final MetadataService metadataService; private final ExecutionService executionService; private final ActionProcessor actionProcessor; private final ExecutorService eventActionExecutorService; private final ObjectMapper objectMapper; private final JsonUtils jsonUtils; private final boolean isEventMessageIndexingEnabled; private final Map evaluators; private final RetryTemplate retryTemplate; public DefaultEventProcessor( ExecutionService executionService, MetadataService metadataService, ActionProcessor actionProcessor, JsonUtils jsonUtils, ConductorProperties properties, ObjectMapper objectMapper, Map evaluators, @Qualifier("onTransientErrorRetryTemplate") RetryTemplate retryTemplate) { this.executionService = executionService; this.metadataService = metadataService; this.actionProcessor = actionProcessor; this.objectMapper = objectMapper; this.jsonUtils = jsonUtils; this.evaluators = evaluators; this.retryTemplate = retryTemplate; if (properties.getEventProcessorThreadCount() <= 0) { throw new IllegalStateException( "Cannot set event processor thread count to <=0. To disable event " + "processing, set conductor.default-event-processor.enabled=false."); } ThreadFactory threadFactory = new BasicThreadFactory.Builder() .namingPattern("event-action-executor-thread-%d") .build(); eventActionExecutorService = Executors.newFixedThreadPool( properties.getEventProcessorThreadCount(), threadFactory); this.isEventMessageIndexingEnabled = properties.isEventMessageIndexingEnabled(); LOGGER.info("Event Processing is ENABLED"); } public void handle(ObservableQueue queue, Message msg) { List transientFailures = null; boolean executionFailed = false; try { if (isEventMessageIndexingEnabled) { executionService.addMessage(queue.getName(), msg); } String event = queue.getType() + ":" + queue.getName(); LOGGER.debug("Evaluating message: {} for event: {}", msg.getId(), event); transientFailures = executeEvent(event, msg); } catch (Exception e) { executionFailed = true; LOGGER.error("Error handling message: {} on queue:{}", msg, queue.getName(), e); Monitors.recordEventQueueMessagesError(queue.getType(), queue.getName()); } finally { if (!executionFailed && CollectionUtils.isEmpty(transientFailures)) { queue.ack(Collections.singletonList(msg)); LOGGER.debug("Message: {} acked on queue: {}", msg.getId(), queue.getName()); } else if (queue.rePublishIfNoAck() || !CollectionUtils.isEmpty(transientFailures)) { // re-submit this message to the queue, to be retried later // This is needed for queues with no unack timeout, since messages are removed // from the queue queue.publish(Collections.singletonList(msg)); LOGGER.debug("Message: {} published to queue: {}", msg.getId(), queue.getName()); } else { queue.nack(Collections.singletonList(msg)); LOGGER.debug("Message: {} nacked on queue: {}", msg.getId(), queue.getName()); } Monitors.recordEventQueueMessagesHandled(queue.getType(), queue.getName()); } } /** * Executes all the actions configured on all the event handlers triggered by the {@link * Message} on the queue If any of the actions on an event handler fails due to a transient * failure, the execution is not persisted such that it can be retried * * @return a list of {@link EventExecution} that failed due to transient failures. */ protected List executeEvent(String event, Message msg) throws Exception { List eventHandlerList; List transientFailures = new ArrayList<>(); try { eventHandlerList = metadataService.getEventHandlersForEvent(event, true); } catch (TransientException transientException) { transientFailures.add(new EventExecution(event, msg.getId())); return transientFailures; } Object payloadObject = getPayloadObject(msg.getPayload()); for (EventHandler eventHandler : eventHandlerList) { String condition = eventHandler.getCondition(); String evaluatorType = eventHandler.getEvaluatorType(); // Set default to true so that if condition is not specified, it falls through // to process the event. boolean success = true; if (StringUtils.isNotEmpty(condition) && evaluators.get(evaluatorType) != null) { Object result = evaluators .get(evaluatorType) .evaluate(condition, jsonUtils.expand(payloadObject)); success = ScriptEvaluator.toBoolean(result); } else if (StringUtils.isNotEmpty(condition)) { LOGGER.debug("Checking condition: {} for event: {}", condition, event); success = ScriptEvaluator.evalBool(condition, jsonUtils.expand(payloadObject)); } if (!success) { String id = msg.getId() + "_" + 0; EventExecution eventExecution = new EventExecution(id, msg.getId()); eventExecution.setCreated(System.currentTimeMillis()); eventExecution.setEvent(eventHandler.getEvent()); eventExecution.setName(eventHandler.getName()); eventExecution.setStatus(Status.SKIPPED); eventExecution.getOutput().put("msg", msg.getPayload()); eventExecution.getOutput().put("condition", condition); executionService.addEventExecution(eventExecution); LOGGER.debug( "Condition: {} not successful for event: {} with payload: {}", condition, eventHandler.getEvent(), msg.getPayload()); continue; } CompletableFuture> future = executeActionsForEventHandler(eventHandler, msg); future.whenComplete( (result, error) -> result.forEach( eventExecution -> { if (error != null || eventExecution.getStatus() == Status.IN_PROGRESS) { transientFailures.add(eventExecution); } else { executionService.updateEventExecution( eventExecution); } })) .get(); } return processTransientFailures(transientFailures); } /** * Remove the event executions which failed temporarily. * * @param eventExecutions The event executions which failed with a transient error. * @return The event executions which failed with a transient error. */ protected List processTransientFailures(List eventExecutions) { eventExecutions.forEach(executionService::removeEventExecution); return eventExecutions; } /** * @param eventHandler the {@link EventHandler} for which the actions are to be executed * @param msg the {@link Message} that triggered the event * @return a {@link CompletableFuture} holding a list of {@link EventExecution}s for the {@link * Action}s executed in the event handler */ protected CompletableFuture> executeActionsForEventHandler( EventHandler eventHandler, Message msg) { List> futuresList = new ArrayList<>(); int i = 0; for (Action action : eventHandler.getActions()) { String id = msg.getId() + "_" + i++; EventExecution eventExecution = new EventExecution(id, msg.getId()); eventExecution.setCreated(System.currentTimeMillis()); eventExecution.setEvent(eventHandler.getEvent()); eventExecution.setName(eventHandler.getName()); eventExecution.setAction(action.getAction()); eventExecution.setStatus(Status.IN_PROGRESS); if (executionService.addEventExecution(eventExecution)) { futuresList.add( CompletableFuture.supplyAsync( () -> execute( eventExecution, action, getPayloadObject(msg.getPayload())), eventActionExecutorService)); } else { LOGGER.warn("Duplicate delivery/execution of message: {}", msg.getId()); } } return CompletableFutures.allAsList(futuresList); } /** * @param eventExecution the instance of {@link EventExecution} * @param action the {@link Action} to be executed for the event * @param payload the {@link Message#getPayload()} * @return the event execution updated with execution output, if the execution is * completed/failed with non-transient error the input event execution, if the execution * failed due to transient error */ protected EventExecution execute(EventExecution eventExecution, Action action, Object payload) { try { LOGGER.debug( "Executing action: {} for event: {} with messageId: {} with payload: {}", action.getAction(), eventExecution.getId(), eventExecution.getMessageId(), payload); // TODO: Switch to @Retryable annotation on SimpleActionProcessor.execute() Map output = retryTemplate.execute( context -> actionProcessor.execute( action, payload, eventExecution.getEvent(), eventExecution.getMessageId())); if (output != null) { eventExecution.getOutput().putAll(output); } eventExecution.setStatus(Status.COMPLETED); Monitors.recordEventExecutionSuccess( eventExecution.getEvent(), eventExecution.getName(), eventExecution.getAction().name()); } catch (RuntimeException e) { LOGGER.error( "Error executing action: {} for event: {} with messageId: {}", action.getAction(), eventExecution.getEvent(), eventExecution.getMessageId(), e); if (!isTransientException(e)) { // not a transient error, fail the event execution eventExecution.setStatus(Status.FAILED); eventExecution.getOutput().put("exception", e.getMessage()); Monitors.recordEventExecutionError( eventExecution.getEvent(), eventExecution.getName(), eventExecution.getAction().name(), e.getClass().getSimpleName()); } } return eventExecution; } private Object getPayloadObject(String payload) { Object payloadObject = null; if (payload != null) { try { payloadObject = objectMapper.readValue(payload, Object.class); } catch (Exception e) { payloadObject = payload; } } return payloadObject; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/DefaultEventQueueManager.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.Lifecycle; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.events.queue.DefaultEventQueueProcessor; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel.Status; /** * Manages the event queues registered in the system and sets up listeners for these. * *

Manages the lifecycle of - * *

    *
  • Queues registered with event handlers *
  • Default event queues that Conductor listens on *
* * @see DefaultEventQueueProcessor */ @Component @ConditionalOnProperty( name = "conductor.default-event-processor.enabled", havingValue = "true", matchIfMissing = true) public class DefaultEventQueueManager extends LifecycleAwareComponent implements EventQueueManager { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultEventQueueManager.class); private final EventHandlerDAO eventHandlerDAO; private final EventQueues eventQueues; private final DefaultEventProcessor defaultEventProcessor; private final Map eventToQueueMap = new ConcurrentHashMap<>(); private final Map defaultQueues; public DefaultEventQueueManager( Map defaultQueues, EventHandlerDAO eventHandlerDAO, EventQueues eventQueues, DefaultEventProcessor defaultEventProcessor) { this.defaultQueues = defaultQueues; this.eventHandlerDAO = eventHandlerDAO; this.eventQueues = eventQueues; this.defaultEventProcessor = defaultEventProcessor; } /** * @return Returns a map of queues which are active. Key is event name and value is queue URI */ @Override public Map getQueues() { Map queues = new HashMap<>(); eventToQueueMap.forEach((key, value) -> queues.put(key, value.getName())); return queues; } @Override public Map> getQueueSizes() { Map> queues = new HashMap<>(); eventToQueueMap.forEach( (key, value) -> { Map size = new HashMap<>(); size.put(value.getName(), value.size()); queues.put(key, size); }); return queues; } @Override public void doStart() { eventToQueueMap.forEach( (event, queue) -> { LOGGER.info("Start listening for events: {}", event); queue.start(); }); defaultQueues.forEach( (status, queue) -> { LOGGER.info( "Start listening on default queue {} for status {}", queue.getName(), status); queue.start(); }); } @Override public void doStop() { eventToQueueMap.forEach( (event, queue) -> { LOGGER.info("Stop listening for events: {}", event); queue.stop(); }); defaultQueues.forEach( (status, queue) -> { LOGGER.info( "Stop listening on default queue {} for status {}", status, queue.getName()); queue.stop(); }); } @Scheduled(fixedDelay = 60_000) public void refreshEventQueues() { try { Set events = eventHandlerDAO.getAllEventHandlers().stream() .filter(EventHandler::isActive) .map(EventHandler::getEvent) .collect(Collectors.toSet()); List createdQueues = new LinkedList<>(); events.forEach( event -> eventToQueueMap.computeIfAbsent( event, s -> { ObservableQueue q = eventQueues.getQueue(event); createdQueues.add(q); return q; })); // start listening on all of the created queues createdQueues.stream() .filter(Objects::nonNull) .peek(Lifecycle::start) .forEach(this::listen); Set removed = new HashSet<>(eventToQueueMap.keySet()); removed.removeAll(events); removed.forEach( key -> { ObservableQueue queue = eventToQueueMap.remove(key); try { queue.stop(); } catch (Exception e) { LOGGER.error("Failed to stop queue: " + queue, e); } }); LOGGER.debug("Event queues: {}", eventToQueueMap.keySet()); LOGGER.debug("Stored queue: {}", events); LOGGER.debug("Removed queue: {}", removed); } catch (Exception e) { Monitors.error(getClass().getSimpleName(), "refresh"); LOGGER.error("refresh event queues failed", e); } } private void listen(ObservableQueue queue) { queue.observe().subscribe((Message msg) -> defaultEventProcessor.handle(queue, msg)); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/EventQueueManager.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.Map; public interface EventQueueManager { Map getQueues(); Map> getQueueSizes(); } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/EventQueueProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import org.springframework.lang.NonNull; import com.netflix.conductor.core.events.queue.ObservableQueue; public interface EventQueueProvider { String getQueueType(); /** * Creates or reads the {@link ObservableQueue} for the given queueURI. * * @param queueURI The URI of the queue. * @return The {@link ObservableQueue} implementation for the queueURI. * @throws IllegalArgumentException thrown when an {@link ObservableQueue} can not be created * for the queueURI. */ @NonNull ObservableQueue getQueue(String queueURI) throws IllegalArgumentException; } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/EventQueues.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.utils.ParametersUtils; /** Holders for internal event queues */ @Component public class EventQueues { public static final String EVENT_QUEUE_PROVIDERS_QUALIFIER = "EventQueueProviders"; private static final Logger LOGGER = LoggerFactory.getLogger(EventQueues.class); private final ParametersUtils parametersUtils; private final Map providers; @Autowired public EventQueues( @Qualifier(EVENT_QUEUE_PROVIDERS_QUALIFIER) Map providers, ParametersUtils parametersUtils) { this.providers = providers; this.parametersUtils = parametersUtils; } public List getProviders() { return providers.values().stream() .map(p -> p.getClass().getName()) .collect(Collectors.toList()); } @NonNull public ObservableQueue getQueue(String eventType) { String event = parametersUtils.replace(eventType).toString(); int index = event.indexOf(':'); if (index == -1) { throw new IllegalArgumentException("Illegal event " + event); } String type = event.substring(0, index); String queueURI = event.substring(index + 1); EventQueueProvider provider = providers.get(type); if (provider != null) { return provider.getQueue(queueURI); } else { throw new IllegalArgumentException("Unknown queue type " + type); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/ScriptEvaluator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; public class ScriptEvaluator { private static ScriptEngine engine; private ScriptEvaluator() {} /** * Evaluates the script with the help of input provided but converts the result to a boolean * value. * * @param script Script to be evaluated. * @param input Input parameters. * @throws ScriptException * @return True or False based on the result of the evaluated expression. */ public static Boolean evalBool(String script, Object input) throws ScriptException { return toBoolean(eval(script, input)); } /** * Evaluates the script with the help of input provided. * * @param script Script to be evaluated. * @param input Input parameters. * @throws ScriptException * @return Generic object, the result of the evaluated expression. */ public static Object eval(String script, Object input) throws ScriptException { if (engine == null) { engine = new ScriptEngineManager().getEngineByName("Nashorn"); } if (engine == null) { throw new RuntimeException( "missing nashorn engine. Ensure you are running supported JVM"); } Bindings bindings = engine.createBindings(); bindings.put("$", input); return engine.eval(script, bindings); } /** * Converts a generic object into boolean value. Checks if the Object is of type Boolean and * returns the value of the Boolean object. Checks if the Object is of type Number and returns * True if the value is greater than 0. * * @param input Generic object that will be inspected to return a boolean value. * @return True or False based on the input provided. */ public static Boolean toBoolean(Object input) { if (input instanceof Boolean) { return ((Boolean) input); } else if (input instanceof Number) { return ((Number) input).doubleValue() > 0; } return false; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/SimpleActionProcessor.java ================================================ /* * Copyright 2020 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.events.EventHandler.Action; import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; import com.netflix.conductor.common.metadata.events.EventHandler.TaskDetails; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.JsonUtils; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * Action Processor subscribes to the Event Actions queue and processes the actions (e.g. start * workflow etc) */ @Component public class SimpleActionProcessor implements ActionProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(SimpleActionProcessor.class); private final WorkflowExecutor workflowExecutor; private final ParametersUtils parametersUtils; private final JsonUtils jsonUtils; private final StartWorkflowOperation startWorkflowOperation; public SimpleActionProcessor( WorkflowExecutor workflowExecutor, ParametersUtils parametersUtils, JsonUtils jsonUtils, StartWorkflowOperation startWorkflowOperation) { this.workflowExecutor = workflowExecutor; this.parametersUtils = parametersUtils; this.jsonUtils = jsonUtils; this.startWorkflowOperation = startWorkflowOperation; } public Map execute( Action action, Object payloadObject, String event, String messageId) { LOGGER.debug( "Executing action: {} for event: {} with messageId:{}", action.getAction(), event, messageId); Object jsonObject = payloadObject; if (action.isExpandInlineJSON()) { jsonObject = jsonUtils.expand(payloadObject); } switch (action.getAction()) { case start_workflow: return startWorkflow(action, jsonObject, event, messageId); case complete_task: return completeTask( action, jsonObject, action.getComplete_task(), TaskModel.Status.COMPLETED, event, messageId); case fail_task: return completeTask( action, jsonObject, action.getFail_task(), TaskModel.Status.FAILED, event, messageId); default: break; } throw new UnsupportedOperationException( "Action not supported " + action.getAction() + " for event " + event); } private Map completeTask( Action action, Object payload, TaskDetails taskDetails, TaskModel.Status status, String event, String messageId) { Map input = new HashMap<>(); input.put("workflowId", taskDetails.getWorkflowId()); input.put("taskId", taskDetails.getTaskId()); input.put("taskRefName", taskDetails.getTaskRefName()); input.putAll(taskDetails.getOutput()); Map replaced = parametersUtils.replace(input, payload); String workflowId = (String) replaced.get("workflowId"); String taskId = (String) replaced.get("taskId"); String taskRefName = (String) replaced.get("taskRefName"); TaskModel taskModel = null; if (StringUtils.isNotEmpty(taskId)) { taskModel = workflowExecutor.getTask(taskId); } else if (StringUtils.isNotEmpty(workflowId) && StringUtils.isNotEmpty(taskRefName)) { WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); if (workflow == null) { replaced.put("error", "No workflow found with ID: " + workflowId); return replaced; } taskModel = workflow.getTaskByRefName(taskRefName); // Task can be loopover task.In such case find corresponding task and update List loopOverTaskList = workflow.getTasks().stream() .filter( t -> TaskUtils.removeIterationFromTaskRefName( t.getReferenceTaskName()) .equals(taskRefName)) .collect(Collectors.toList()); if (!loopOverTaskList.isEmpty()) { // Find loopover task with the highest iteration value taskModel = loopOverTaskList.stream() .sorted(Comparator.comparingInt(TaskModel::getIteration).reversed()) .findFirst() .get(); } } if (taskModel == null) { replaced.put( "error", "No task found with taskId: " + taskId + ", reference name: " + taskRefName + ", workflowId: " + workflowId); return replaced; } taskModel.setStatus(status); taskModel.setOutputData(replaced); taskModel.setOutputMessage(taskDetails.getOutputMessage()); taskModel.addOutput("conductor.event.messageId", messageId); taskModel.addOutput("conductor.event.name", event); try { workflowExecutor.updateTask(new TaskResult(taskModel.toTask())); LOGGER.debug( "Updated task: {} in workflow:{} with status: {} for event: {} for message:{}", taskId, workflowId, status, event, messageId); } catch (RuntimeException e) { Monitors.recordEventActionError( action.getAction().name(), taskModel.getTaskType(), event); LOGGER.error( "Error updating task: {} in workflow: {} in action: {} for event: {} for message: {}", taskDetails.getTaskRefName(), taskDetails.getWorkflowId(), action.getAction(), event, messageId, e); replaced.put("error", e.getMessage()); throw e; } return replaced; } private Map startWorkflow( Action action, Object payload, String event, String messageId) { StartWorkflow params = action.getStart_workflow(); Map output = new HashMap<>(); try { Map inputParams = params.getInput(); Map workflowInput = parametersUtils.replace(inputParams, payload); Map paramsMap = new HashMap<>(); Optional.ofNullable(params.getCorrelationId()) .ifPresent(value -> paramsMap.put("correlationId", value)); Map replaced = parametersUtils.replace(paramsMap, payload); workflowInput.put("conductor.event.messageId", messageId); workflowInput.put("conductor.event.name", event); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName(params.getName()); startWorkflowInput.setVersion(params.getVersion()); startWorkflowInput.setCorrelationId( Optional.ofNullable(replaced.get("correlationId")) .map(Object::toString) .orElse(params.getCorrelationId())); startWorkflowInput.setWorkflowInput(workflowInput); startWorkflowInput.setEvent(event); startWorkflowInput.setTaskToDomain(params.getTaskToDomain()); String workflowId = startWorkflowOperation.execute(startWorkflowInput); output.put("workflowId", workflowId); LOGGER.debug( "Started workflow: {}/{}/{} for event: {} for message:{}", params.getName(), params.getVersion(), workflowId, event, messageId); } catch (RuntimeException e) { Monitors.recordEventActionError(action.getAction().name(), params.getName(), event); LOGGER.error( "Error starting workflow: {}, version: {}, for event: {} for message: {}", params.getName(), params.getVersion(), event, messageId, e); output.put("error", e.getMessage()); throw e; } return output; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/queue/ConductorEventQueueProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events.queue; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.EventQueueProvider; import com.netflix.conductor.dao.QueueDAO; import rx.Scheduler; /** * Default provider for {@link com.netflix.conductor.core.events.queue.ObservableQueue} that listens * on the conductor queue prefix. * *

Set conductor.event-queues.default.enabled=false to disable the default queue. * * @see ConductorObservableQueue */ @Component @ConditionalOnProperty( name = "conductor.event-queues.default.enabled", havingValue = "true", matchIfMissing = true) public class ConductorEventQueueProvider implements EventQueueProvider { private static final Logger LOGGER = LoggerFactory.getLogger(ConductorEventQueueProvider.class); private final Map queues = new ConcurrentHashMap<>(); private final QueueDAO queueDAO; private final ConductorProperties properties; private final Scheduler scheduler; public ConductorEventQueueProvider( QueueDAO queueDAO, ConductorProperties properties, Scheduler scheduler) { this.queueDAO = queueDAO; this.properties = properties; this.scheduler = scheduler; } @Override public String getQueueType() { return "conductor"; } @Override @NonNull public ObservableQueue getQueue(String queueURI) { return queues.computeIfAbsent( queueURI, q -> new ConductorObservableQueue(queueURI, queueDAO, properties, scheduler)); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/queue/ConductorObservableQueue.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events.queue; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import rx.Observable; import rx.Observable.OnSubscribe; import rx.Scheduler; /** * An {@link ObservableQueue} implementation using the underlying {@link QueueDAO} implementation. */ public class ConductorObservableQueue implements ObservableQueue { private static final Logger LOGGER = LoggerFactory.getLogger(ConductorObservableQueue.class); private static final String QUEUE_TYPE = "conductor"; private final String queueName; private final QueueDAO queueDAO; private final long pollTimeMS; private final int longPollTimeout; private final int pollCount; private final Scheduler scheduler; private volatile boolean running; ConductorObservableQueue( String queueName, QueueDAO queueDAO, ConductorProperties properties, Scheduler scheduler) { this.queueName = queueName; this.queueDAO = queueDAO; this.pollTimeMS = properties.getEventQueuePollInterval().toMillis(); this.pollCount = properties.getEventQueuePollCount(); this.longPollTimeout = (int) properties.getEventQueueLongPollTimeout().toMillis(); this.scheduler = scheduler; } @Override public Observable observe() { OnSubscribe subscriber = getOnSubscribe(); return Observable.create(subscriber); } @Override public List ack(List messages) { for (Message msg : messages) { queueDAO.ack(queueName, msg.getId()); } return messages.stream().map(Message::getId).collect(Collectors.toList()); } public void setUnackTimeout(Message message, long unackTimeout) { queueDAO.setUnackTimeout(queueName, message.getId(), unackTimeout); } @Override public void publish(List messages) { queueDAO.push(queueName, messages); } @Override public long size() { return queueDAO.getSize(queueName); } @Override public String getType() { return QUEUE_TYPE; } @Override public String getName() { return queueName; } @Override public String getURI() { return queueName; } private List receiveMessages() { try { List messages = queueDAO.pollMessages(queueName, pollCount, longPollTimeout); Monitors.recordEventQueueMessagesProcessed(QUEUE_TYPE, queueName, messages.size()); Monitors.recordEventQueuePollSize(queueName, messages.size()); return messages; } catch (Exception exception) { LOGGER.error("Exception while getting messages from queueDAO", exception); Monitors.recordObservableQMessageReceivedErrors(QUEUE_TYPE); } return new ArrayList<>(); } private OnSubscribe getOnSubscribe() { return subscriber -> { Observable interval = Observable.interval(pollTimeMS, TimeUnit.MILLISECONDS, scheduler); interval.flatMap( (Long x) -> { if (!isRunning()) { LOGGER.debug( "Component stopped, skip listening for messages from Conductor Queue"); return Observable.from(Collections.emptyList()); } List messages = receiveMessages(); return Observable.from(messages); }) .subscribe(subscriber::onNext, subscriber::onError); }; } @Override public void start() { LOGGER.info("Started listening to {}:{}", getClass().getSimpleName(), queueName); running = true; } @Override public void stop() { LOGGER.info("Stopped listening to {}:{}", getClass().getSimpleName(), queueName); running = false; } @Override public boolean isRunning() { return running; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/queue/DefaultEventQueueProcessor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events.queue; import java.util.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; /** * Monitors and processes messages on the default event queues that Conductor listens on. * *

The default event queue type is controlled using the property: * conductor.default-event-queue.type */ @Component @ConditionalOnProperty( name = "conductor.default-event-queue-processor.enabled", havingValue = "true", matchIfMissing = true) public class DefaultEventQueueProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultEventQueueProcessor.class); private final Map queues; private final WorkflowExecutor workflowExecutor; private static final TypeReference> _mapType = new TypeReference<>() {}; private final ObjectMapper objectMapper; public DefaultEventQueueProcessor( Map queues, WorkflowExecutor workflowExecutor, ObjectMapper objectMapper) { this.queues = queues; this.workflowExecutor = workflowExecutor; this.objectMapper = objectMapper; queues.forEach(this::startMonitor); LOGGER.info( "DefaultEventQueueProcessor initialized with {} queues", queues.entrySet().size()); } private void startMonitor(Status status, ObservableQueue queue) { queue.observe() .subscribe( (Message msg) -> { try { LOGGER.debug("Got message {}", msg.getPayload()); String payload = msg.getPayload(); JsonNode payloadJSON = objectMapper.readTree(payload); String externalId = getValue("externalId", payloadJSON); if (externalId == null || "".equals(externalId)) { LOGGER.error("No external Id found in the payload {}", payload); queue.ack(Collections.singletonList(msg)); return; } JsonNode json = objectMapper.readTree(externalId); String workflowId = getValue("workflowId", json); String taskRefName = getValue("taskRefName", json); String taskId = getValue("taskId", json); if (workflowId == null || "".equals(workflowId)) { // This is a bad message, we cannot process it LOGGER.error( "No workflow id found in the message. {}", payload); queue.ack(Collections.singletonList(msg)); return; } WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); Optional optionalTaskModel; if (StringUtils.isNotEmpty(taskId)) { optionalTaskModel = workflow.getTasks().stream() .filter( task -> !task.getStatus().isTerminal() && task.getTaskId() .equals(taskId)) .findFirst(); } else if (StringUtils.isEmpty(taskRefName)) { LOGGER.error( "No taskRefName found in the message. If there is only one WAIT task, will mark it as completed. {}", payload); optionalTaskModel = workflow.getTasks().stream() .filter( task -> !task.getStatus().isTerminal() && task.getTaskType() .equals( TASK_TYPE_WAIT)) .findFirst(); } else { optionalTaskModel = workflow.getTasks().stream() .filter( task -> !task.getStatus().isTerminal() && TaskUtils .removeIterationFromTaskRefName( task .getReferenceTaskName()) .equals( taskRefName)) .findFirst(); } if (optionalTaskModel.isEmpty()) { LOGGER.error( "No matching tasks found to be marked as completed for workflow {}, taskRefName {}, taskId {}", workflowId, taskRefName, taskId); queue.ack(Collections.singletonList(msg)); return; } Task task = optionalTaskModel.get().toTask(); task.setStatus(TaskModel.mapToTaskStatus(status)); task.getOutputData() .putAll(objectMapper.convertValue(payloadJSON, _mapType)); workflowExecutor.updateTask(new TaskResult(task)); List failures = queue.ack(Collections.singletonList(msg)); if (!failures.isEmpty()) { LOGGER.error("Not able to ack the messages {}", failures); } } catch (JsonParseException e) { LOGGER.error("Bad message? : {} ", msg, e); queue.ack(Collections.singletonList(msg)); } catch (NotFoundException nfe) { LOGGER.error( "Workflow ID specified is not valid for this environment"); queue.ack(Collections.singletonList(msg)); } catch (Exception e) { LOGGER.error("Error processing message: {}", msg, e); } }, (Throwable t) -> LOGGER.error(t.getMessage(), t)); LOGGER.info("QueueListener::STARTED...listening for " + queue.getName()); } private String getValue(String fieldName, JsonNode json) { JsonNode node = json.findValue(fieldName); if (node == null) { return null; } return node.textValue(); } public Map size() { Map size = new HashMap<>(); queues.forEach((key, queue) -> size.put(queue.getName(), queue.size())); return size; } public Map queues() { Map size = new HashMap<>(); queues.forEach((key, queue) -> size.put(key, queue.getURI())); return size; } public void updateByTaskRefName( String workflowId, String taskRefName, Map output, Status status) throws Exception { Map externalIdMap = new HashMap<>(); externalIdMap.put("workflowId", workflowId); externalIdMap.put("taskRefName", taskRefName); update(externalIdMap, output, status); } public void updateByTaskId( String workflowId, String taskId, Map output, Status status) throws Exception { Map externalIdMap = new HashMap<>(); externalIdMap.put("workflowId", workflowId); externalIdMap.put("taskId", taskId); update(externalIdMap, output, status); } private void update( Map externalIdMap, Map output, Status status) throws Exception { Map outputMap = new HashMap<>(); outputMap.put("externalId", objectMapper.writeValueAsString(externalIdMap)); outputMap.putAll(output); Message msg = new Message( UUID.randomUUID().toString(), objectMapper.writeValueAsString(outputMap), null); ObservableQueue queue = queues.get(status); if (queue == null) { throw new IllegalArgumentException( "There is no queue for handling " + status.toString() + " status"); } queue.publish(Collections.singletonList(msg)); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/queue/Message.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events.queue; import java.util.Objects; public class Message { private String payload; private String id; private String receipt; private int priority; public Message() {} public Message(String id, String payload, String receipt) { this.payload = payload; this.id = id; this.receipt = receipt; } public Message(String id, String payload, String receipt, int priority) { this.payload = payload; this.id = id; this.receipt = receipt; this.priority = priority; } /** * @return the payload */ public String getPayload() { return payload; } /** * @param payload the payload to set */ public void setPayload(String payload) { this.payload = payload; } /** * @return the id */ public String getId() { return id; } /** * @param id the id to set */ public void setId(String id) { this.id = id; } /** * @return Receipt attached to the message */ public String getReceipt() { return receipt; } /** * @param receipt Receipt attached to the message */ public void setReceipt(String receipt) { this.receipt = receipt; } /** * Gets the message priority * * @return priority of message. */ public int getPriority() { return priority; } /** * Sets the message priority (between 0 and 99). Higher priority message is retrieved ahead of * lower priority ones. * * @param priority the priority of message (between 0 and 99) */ public void setPriority(int priority) { this.priority = priority; } @Override public String toString() { return id; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Message message = (Message) o; return Objects.equals(payload, message.payload) && Objects.equals(id, message.id) && Objects.equals(priority, message.priority) && Objects.equals(receipt, message.receipt); } @Override public int hashCode() { return Objects.hash(payload, id, receipt, priority); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/events/queue/ObservableQueue.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events.queue; import java.util.List; import org.springframework.context.Lifecycle; import rx.Observable; public interface ObservableQueue extends Lifecycle { /** * @return An observable for the given queue */ Observable observe(); /** * @return Type of the queue */ String getType(); /** * @return Name of the queue */ String getName(); /** * @return URI identifier for the queue. */ String getURI(); /** * @param messages to be ack'ed * @return the id of the ones which could not be ack'ed */ List ack(List messages); /** * @param messages to be Nack'ed */ default void nack(List messages) {} /** * @param messages Messages to be published */ void publish(List messages); /** * Used to determine if the queue supports unack/visibility timeout such that the messages will * re-appear on the queue after a specific period and are available to be picked up again and * retried. * * @return - false if the queue message need not be re-published to the queue for retriability - * true if the message must be re-published to the queue for retriability */ default boolean rePublishIfNoAck() { return false; } /** * Extend the lease of the unacknowledged message for longer period. * * @param message Message for which the timeout has to be changed * @param unackTimeout timeout in milliseconds for which the unack lease should be extended. * (replaces the current value with this value) */ void setUnackTimeout(Message message, long unackTimeout); /** * @return Size of the queue - no. messages pending. Note: Depending upon the implementation, * this can be an approximation */ long size(); /** Used to close queue instance prior to remove from queues */ default void close() {} } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/exception/ConflictException.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.exception; public class ConflictException extends RuntimeException { public ConflictException(String message) { super(message); } public ConflictException(String message, Object... args) { super(String.format(message, args)); } public ConflictException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/exception/NonTransientException.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.exception; public class NonTransientException extends RuntimeException { public NonTransientException(String message) { super(message); } public NonTransientException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/exception/NotFoundException.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.exception; public class NotFoundException extends RuntimeException { public NotFoundException(String message) { super(message); } public NotFoundException(String message, Object... args) { super(String.format(message, args)); } public NotFoundException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/exception/TerminateWorkflowException.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.exception; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.model.WorkflowModel.Status.FAILED; public class TerminateWorkflowException extends RuntimeException { private final WorkflowModel.Status workflowStatus; private final TaskModel task; public TerminateWorkflowException(String reason) { this(reason, FAILED); } public TerminateWorkflowException(String reason, WorkflowModel.Status workflowStatus) { this(reason, workflowStatus, null); } public TerminateWorkflowException( String reason, WorkflowModel.Status workflowStatus, TaskModel task) { super(reason); this.workflowStatus = workflowStatus; this.task = task; } public WorkflowModel.Status getWorkflowStatus() { return workflowStatus; } public TaskModel getTask() { return task; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/exception/TransientException.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.exception; public class TransientException extends RuntimeException { public TransientException(String message) { super(message); } public TransientException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/AsyncSystemTaskExecutor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; @Component public class AsyncSystemTaskExecutor { private final ExecutionDAOFacade executionDAOFacade; private final QueueDAO queueDAO; private final MetadataDAO metadataDAO; private final long queueTaskMessagePostponeSecs; private final long systemTaskCallbackTime; private final WorkflowExecutor workflowExecutor; private static final Logger LOGGER = LoggerFactory.getLogger(AsyncSystemTaskExecutor.class); public AsyncSystemTaskExecutor( ExecutionDAOFacade executionDAOFacade, QueueDAO queueDAO, MetadataDAO metadataDAO, ConductorProperties conductorProperties, WorkflowExecutor workflowExecutor) { this.executionDAOFacade = executionDAOFacade; this.queueDAO = queueDAO; this.metadataDAO = metadataDAO; this.workflowExecutor = workflowExecutor; this.systemTaskCallbackTime = conductorProperties.getSystemTaskWorkerCallbackDuration().getSeconds(); this.queueTaskMessagePostponeSecs = conductorProperties.getTaskExecutionPostponeDuration().getSeconds(); } /** * Executes and persists the results of an async {@link WorkflowSystemTask}. * * @param systemTask The {@link WorkflowSystemTask} to be executed. * @param taskId The id of the {@link TaskModel} object. */ public void execute(WorkflowSystemTask systemTask, String taskId) { TaskModel task = loadTaskQuietly(taskId); if (task == null) { LOGGER.error("TaskId: {} could not be found while executing {}", taskId, systemTask); try { LOGGER.debug( "Cleaning up dead task from queue message: taskQueue={}, taskId={}", systemTask.getTaskType(), taskId); queueDAO.remove(systemTask.getTaskType(), taskId); } catch (Exception e) { LOGGER.error( "Failed to remove dead task from queue message: taskQueue={}, taskId={}", systemTask.getTaskType(), taskId); } return; } LOGGER.debug("Task: {} fetched from execution DAO for taskId: {}", task, taskId); String queueName = QueueUtils.getQueueName(task); if (task.getStatus().isTerminal()) { // Tune the SystemTaskWorkerCoordinator's queues - if the queue size is very big this // can happen! LOGGER.info("Task {}/{} was already completed.", task.getTaskType(), task.getTaskId()); queueDAO.remove(queueName, task.getTaskId()); return; } if (task.getStatus().equals(TaskModel.Status.SCHEDULED)) { if (executionDAOFacade.exceedsInProgressLimit(task)) { LOGGER.warn( "Concurrent Execution limited for {}:{}", taskId, task.getTaskDefName()); postponeQuietly(queueName, task); return; } if (task.getRateLimitPerFrequency() > 0 && executionDAOFacade.exceedsRateLimitPerFrequency( task, metadataDAO.getTaskDef(task.getTaskDefName()))) { LOGGER.warn( "RateLimit Execution limited for {}:{}, limit:{}", taskId, task.getTaskDefName(), task.getRateLimitPerFrequency()); postponeQuietly(queueName, task); return; } } boolean hasTaskExecutionCompleted = false; boolean shouldRemoveTaskFromQueue = false; String workflowId = task.getWorkflowInstanceId(); // if we are here the Task object is updated and needs to be persisted regardless of an // exception try { WorkflowModel workflow = executionDAOFacade.getWorkflowModel( workflowId, systemTask.isTaskRetrievalRequired()); if (workflow.getStatus().isTerminal()) { LOGGER.info( "Workflow {} has been completed for {}/{}", workflow.toShortString(), systemTask, task.getTaskId()); if (!task.getStatus().isTerminal()) { task.setStatus(TaskModel.Status.CANCELED); task.setReasonForIncompletion( String.format( "Workflow is in %s state", workflow.getStatus().toString())); } shouldRemoveTaskFromQueue = true; return; } LOGGER.debug( "Executing {}/{} in {} state", task.getTaskType(), task.getTaskId(), task.getStatus()); boolean isTaskAsyncComplete = systemTask.isAsyncComplete(task); if (task.getStatus() == TaskModel.Status.SCHEDULED || !isTaskAsyncComplete) { task.incrementPollCount(); } if (task.getStatus() == TaskModel.Status.SCHEDULED) { task.setStartTime(System.currentTimeMillis()); Monitors.recordQueueWaitTime(task.getTaskType(), task.getQueueWaitTime()); systemTask.start(workflow, task, workflowExecutor); } else if (task.getStatus() == TaskModel.Status.IN_PROGRESS) { systemTask.execute(workflow, task, workflowExecutor); } // Update message in Task queue based on Task status // Remove asyncComplete system tasks from the queue that are not in SCHEDULED state if (isTaskAsyncComplete && task.getStatus() != TaskModel.Status.SCHEDULED) { shouldRemoveTaskFromQueue = true; hasTaskExecutionCompleted = true; } else if (task.getStatus().isTerminal()) { task.setEndTime(System.currentTimeMillis()); shouldRemoveTaskFromQueue = true; hasTaskExecutionCompleted = true; } else { task.setCallbackAfterSeconds(systemTaskCallbackTime); systemTask .getEvaluationOffset(task, systemTaskCallbackTime) .ifPresentOrElse( task::setCallbackAfterSeconds, () -> task.setCallbackAfterSeconds(systemTaskCallbackTime)); queueDAO.postpone( queueName, task.getTaskId(), task.getWorkflowPriority(), task.getCallbackAfterSeconds()); LOGGER.debug("{} postponed in queue: {}", task, queueName); } LOGGER.debug( "Finished execution of {}/{}-{}", systemTask, task.getTaskId(), task.getStatus()); } catch (Exception e) { Monitors.error(AsyncSystemTaskExecutor.class.getSimpleName(), "executeSystemTask"); LOGGER.error("Error executing system task - {}, with id: {}", systemTask, taskId, e); } finally { executionDAOFacade.updateTask(task); if (shouldRemoveTaskFromQueue) { queueDAO.remove(queueName, task.getTaskId()); LOGGER.debug("{} removed from queue: {}", task, queueName); } // if the current task execution has completed, then the workflow needs to be evaluated if (hasTaskExecutionCompleted) { workflowExecutor.decide(workflowId); } } } private void postponeQuietly(String queueName, TaskModel task) { try { queueDAO.postpone( queueName, task.getTaskId(), task.getWorkflowPriority(), queueTaskMessagePostponeSecs); } catch (Exception e) { LOGGER.error("Error postponing task: {} in queue: {}", task.getTaskId(), queueName); } } private TaskModel loadTaskQuietly(String taskId) { try { return executionDAOFacade.getTaskModel(taskId); } catch (Exception e) { return null; } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.time.Duration; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.ExternalPayloadStorage.Operation; import com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.mapper.TaskMapper; import com.netflix.conductor.core.execution.mapper.TaskMapperContext; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TERMINATE; import static com.netflix.conductor.common.metadata.tasks.TaskType.USER_DEFINED; import static com.netflix.conductor.model.TaskModel.Status.*; /** * Decider evaluates the state of the workflow by inspecting the current state along with the * blueprint. The result of the evaluation is either to schedule further tasks, complete/fail the * workflow or do nothing. */ @Service @Trace public class DeciderService { private static final Logger LOGGER = LoggerFactory.getLogger(DeciderService.class); private final IDGenerator idGenerator; private final ParametersUtils parametersUtils; private final ExternalPayloadStorageUtils externalPayloadStorageUtils; private final MetadataDAO metadataDAO; private final SystemTaskRegistry systemTaskRegistry; private final long taskPendingTimeThresholdMins; private final Map taskMappers; public DeciderService( IDGenerator idGenerator, ParametersUtils parametersUtils, MetadataDAO metadataDAO, ExternalPayloadStorageUtils externalPayloadStorageUtils, SystemTaskRegistry systemTaskRegistry, @Qualifier("taskMappersByTaskType") Map taskMappers, @Value("${conductor.app.taskPendingTimeThreshold:60m}") Duration taskPendingTimeThreshold) { this.idGenerator = idGenerator; this.metadataDAO = metadataDAO; this.parametersUtils = parametersUtils; this.taskMappers = taskMappers; this.externalPayloadStorageUtils = externalPayloadStorageUtils; this.taskPendingTimeThresholdMins = taskPendingTimeThreshold.toMinutes(); this.systemTaskRegistry = systemTaskRegistry; } public DeciderOutcome decide(WorkflowModel workflow) throws TerminateWorkflowException { // In case of a new workflow the list of tasks will be empty. final List tasks = workflow.getTasks(); // Filter the list of tasks and include only tasks that are not executed, // not marked to be skipped and not ready for rerun. // For a new workflow, the list of unprocessedTasks will be empty List unprocessedTasks = tasks.stream() .filter(t -> !t.getStatus().equals(SKIPPED) && !t.isExecuted()) .collect(Collectors.toList()); List tasksToBeScheduled = new LinkedList<>(); if (unprocessedTasks.isEmpty()) { // this is the flow that the new workflow will go through tasksToBeScheduled = startWorkflow(workflow); if (tasksToBeScheduled == null) { tasksToBeScheduled = new LinkedList<>(); } } return decide(workflow, tasksToBeScheduled); } private DeciderOutcome decide(final WorkflowModel workflow, List preScheduledTasks) throws TerminateWorkflowException { DeciderOutcome outcome = new DeciderOutcome(); if (workflow.getStatus().isTerminal()) { // you cannot evaluate a terminal workflow LOGGER.debug( "Workflow {} is already finished. Reason: {}", workflow, workflow.getReasonForIncompletion()); return outcome; } checkWorkflowTimeout(workflow); if (workflow.getStatus().equals(WorkflowModel.Status.PAUSED)) { LOGGER.debug("Workflow " + workflow.getWorkflowId() + " is paused"); return outcome; } List pendingTasks = new ArrayList<>(); Set executedTaskRefNames = new HashSet<>(); boolean hasSuccessfulTerminateTask = false; for (TaskModel task : workflow.getTasks()) { // Filter the list of tasks and include only tasks that are not retried, not executed // marked to be skipped and not part of System tasks that is DECISION, FORK, JOIN // This list will be empty for a new workflow being started if (!task.isRetried() && !task.getStatus().equals(SKIPPED) && !task.isExecuted()) { pendingTasks.add(task); } // Get all the tasks that have not completed their lifecycle yet // This list will be empty for a new workflow if (task.isExecuted()) { executedTaskRefNames.add(task.getReferenceTaskName()); } if (TERMINATE.name().equals(task.getTaskType()) && task.getStatus().isTerminal() && task.getStatus().isSuccessful()) { hasSuccessfulTerminateTask = true; outcome.terminateTask = task; } } Map tasksToBeScheduled = new LinkedHashMap<>(); preScheduledTasks.forEach( preScheduledTask -> { tasksToBeScheduled.put( preScheduledTask.getReferenceTaskName(), preScheduledTask); }); // A new workflow does not enter this code branch for (TaskModel pendingTask : pendingTasks) { if (systemTaskRegistry.isSystemTask(pendingTask.getTaskType()) && !pendingTask.getStatus().isTerminal()) { tasksToBeScheduled.putIfAbsent(pendingTask.getReferenceTaskName(), pendingTask); executedTaskRefNames.remove(pendingTask.getReferenceTaskName()); } Optional taskDefinition = pendingTask.getTaskDefinition(); if (taskDefinition.isEmpty()) { taskDefinition = Optional.ofNullable( workflow.getWorkflowDefinition() .getTaskByRefName( pendingTask.getReferenceTaskName())) .map(WorkflowTask::getTaskDefinition); } if (taskDefinition.isPresent()) { checkTaskTimeout(taskDefinition.get(), pendingTask); checkTaskPollTimeout(taskDefinition.get(), pendingTask); // If the task has not been updated for "responseTimeoutSeconds" then mark task as // TIMED_OUT if (isResponseTimedOut(taskDefinition.get(), pendingTask)) { timeoutTask(taskDefinition.get(), pendingTask); } } if (!pendingTask.getStatus().isSuccessful()) { WorkflowTask workflowTask = pendingTask.getWorkflowTask(); if (workflowTask == null) { workflowTask = workflow.getWorkflowDefinition() .getTaskByRefName(pendingTask.getReferenceTaskName()); } Optional retryTask = retry(taskDefinition.orElse(null), workflowTask, pendingTask, workflow); if (retryTask.isPresent()) { tasksToBeScheduled.put(retryTask.get().getReferenceTaskName(), retryTask.get()); executedTaskRefNames.remove(retryTask.get().getReferenceTaskName()); outcome.tasksToBeUpdated.add(pendingTask); } else { pendingTask.setStatus(COMPLETED_WITH_ERRORS); } } if (!pendingTask.isExecuted() && !pendingTask.isRetried() && pendingTask.getStatus().isTerminal()) { pendingTask.setExecuted(true); List nextTasks = getNextTask(workflow, pendingTask); if (pendingTask.isLoopOverTask() && !TaskType.DO_WHILE.name().equals(pendingTask.getTaskType()) && !nextTasks.isEmpty()) { nextTasks = filterNextLoopOverTasks(nextTasks, pendingTask, workflow); } nextTasks.forEach( nextTask -> tasksToBeScheduled.putIfAbsent( nextTask.getReferenceTaskName(), nextTask)); outcome.tasksToBeUpdated.add(pendingTask); LOGGER.debug( "Scheduling Tasks from {}, next = {} for workflowId: {}", pendingTask.getTaskDefName(), nextTasks.stream() .map(TaskModel::getTaskDefName) .collect(Collectors.toList()), workflow.getWorkflowId()); } } // All the tasks that need to scheduled are added to the outcome, in case of List unScheduledTasks = tasksToBeScheduled.values().stream() .filter(task -> !executedTaskRefNames.contains(task.getReferenceTaskName())) .collect(Collectors.toList()); if (!unScheduledTasks.isEmpty()) { LOGGER.debug( "Scheduling Tasks: {} for workflow: {}", unScheduledTasks.stream() .map(TaskModel::getTaskDefName) .collect(Collectors.toList()), workflow.getWorkflowId()); outcome.tasksToBeScheduled.addAll(unScheduledTasks); } if (hasSuccessfulTerminateTask || (outcome.tasksToBeScheduled.isEmpty() && checkForWorkflowCompletion(workflow))) { LOGGER.debug("Marking workflow: {} as complete.", workflow); outcome.isComplete = true; } return outcome; } @VisibleForTesting List filterNextLoopOverTasks( List tasks, TaskModel pendingTask, WorkflowModel workflow) { // Update the task reference name and iteration tasks.forEach( nextTask -> { nextTask.setReferenceTaskName( TaskUtils.appendIteration( nextTask.getReferenceTaskName(), pendingTask.getIteration())); nextTask.setIteration(pendingTask.getIteration()); }); List tasksInWorkflow = workflow.getTasks().stream() .filter( runningTask -> runningTask.getStatus().equals(TaskModel.Status.IN_PROGRESS) || runningTask.getStatus().isTerminal()) .map(TaskModel::getReferenceTaskName) .collect(Collectors.toList()); return tasks.stream() .filter( runningTask -> !tasksInWorkflow.contains(runningTask.getReferenceTaskName())) .collect(Collectors.toList()); } private List startWorkflow(WorkflowModel workflow) throws TerminateWorkflowException { final WorkflowDef workflowDef = workflow.getWorkflowDefinition(); LOGGER.debug("Starting workflow: {}", workflow); // The tasks will be empty in case of new workflow List tasks = workflow.getTasks(); // Check if the workflow is a re-run case or if it is a new workflow execution if (workflow.getReRunFromWorkflowId() == null || tasks.isEmpty()) { if (workflowDef.getTasks().isEmpty()) { throw new TerminateWorkflowException( "No tasks found to be executed", WorkflowModel.Status.COMPLETED); } WorkflowTask taskToSchedule = workflowDef .getTasks() .get(0); // Nothing is running yet - so schedule the first task // Loop until a non-skipped task is found while (isTaskSkipped(taskToSchedule, workflow)) { taskToSchedule = workflowDef.getNextTask(taskToSchedule.getTaskReferenceName()); } // In case of a new workflow, the first non-skippable task will be scheduled return getTasksToBeScheduled(workflow, taskToSchedule, 0); } // Get the first task to schedule TaskModel rerunFromTask = tasks.stream() .findFirst() .map( task -> { task.setStatus(SCHEDULED); task.setRetried(true); task.setRetryCount(0); return task; }) .orElseThrow( () -> { String reason = String.format( "The workflow %s is marked for re-run from %s but could not find the starting task", workflow.getWorkflowId(), workflow.getReRunFromWorkflowId()); return new TerminateWorkflowException(reason); }); return Collections.singletonList(rerunFromTask); } /** * Updates the workflow output. * * @param workflow the workflow instance * @param task if not null, the output of this task will be copied to workflow output if no * output parameters are specified in the workflow definition if null, the output of the * last task in the workflow will be copied to workflow output of no output parameters are * specified in the workflow definition */ void updateWorkflowOutput(final WorkflowModel workflow, TaskModel task) { List allTasks = workflow.getTasks(); if (allTasks.isEmpty()) { return; } Map output = new HashMap<>(); Optional optionalTask = allTasks.stream() .filter( t -> TaskType.TERMINATE.name().equals(t.getTaskType()) && t.getStatus().isTerminal() && t.getStatus().isSuccessful()) .findFirst(); if (optionalTask.isPresent()) { TaskModel terminateTask = optionalTask.get(); if (StringUtils.isNotBlank(terminateTask.getExternalOutputPayloadStoragePath())) { output = externalPayloadStorageUtils.downloadPayload( terminateTask.getExternalOutputPayloadStoragePath()); Monitors.recordExternalPayloadStorageUsage( terminateTask.getTaskDefName(), Operation.READ.toString(), PayloadType.TASK_OUTPUT.toString()); } else if (!terminateTask.getOutputData().isEmpty()) { output = terminateTask.getOutputData(); } } else { TaskModel last = Optional.ofNullable(task).orElse(allTasks.get(allTasks.size() - 1)); WorkflowDef workflowDef = workflow.getWorkflowDefinition(); if (workflowDef.getOutputParameters() != null && !workflowDef.getOutputParameters().isEmpty()) { output = parametersUtils.getTaskInput( workflowDef.getOutputParameters(), workflow, null, null); } else if (StringUtils.isNotBlank(last.getExternalOutputPayloadStoragePath())) { output = externalPayloadStorageUtils.downloadPayload( last.getExternalOutputPayloadStoragePath()); Monitors.recordExternalPayloadStorageUsage( last.getTaskDefName(), Operation.READ.toString(), PayloadType.TASK_OUTPUT.toString()); } else { output = last.getOutputData(); } } workflow.setOutput(output); } public boolean checkForWorkflowCompletion(final WorkflowModel workflow) throws TerminateWorkflowException { Map taskStatusMap = new HashMap<>(); List nonExecutedTasks = new ArrayList<>(); for (TaskModel task : workflow.getTasks()) { taskStatusMap.put(task.getReferenceTaskName(), task.getStatus()); if (!task.getStatus().isTerminal()) { return false; } // If there is a TERMINATE task that has been executed successfuly then the workflow // should be marked as completed. if (TERMINATE.name().equals(task.getTaskType()) && task.getStatus().isTerminal() && task.getStatus().isSuccessful()) { return true; } if (!task.isRetried() || !task.isExecuted()) { nonExecutedTasks.add(task); } } // If there are no tasks executed, then we are not done yet if (taskStatusMap.isEmpty()) { return false; } List workflowTasks = workflow.getWorkflowDefinition().getTasks(); for (WorkflowTask wftask : workflowTasks) { TaskModel.Status status = taskStatusMap.get(wftask.getTaskReferenceName()); if (status == null || !status.isTerminal()) { return false; } // if we reach here, the task has been completed. // Was the task successful in completion? if (!status.isSuccessful()) { return false; } } boolean noPendingSchedule = nonExecutedTasks.stream() .parallel() .noneMatch( wftask -> { String next = getNextTasksToBeScheduled(workflow, wftask); return next != null && !taskStatusMap.containsKey(next); }); return noPendingSchedule; } List getNextTask(WorkflowModel workflow, TaskModel task) { final WorkflowDef workflowDef = workflow.getWorkflowDefinition(); // Get the following task after the last completed task if (systemTaskRegistry.isSystemTask(task.getTaskType()) && (TaskType.TASK_TYPE_DECISION.equals(task.getTaskType()) || TaskType.TASK_TYPE_SWITCH.equals(task.getTaskType()))) { if (task.getInputData().get("hasChildren") != null) { return Collections.emptyList(); } } String taskReferenceName = task.isLoopOverTask() ? TaskUtils.removeIterationFromTaskRefName(task.getReferenceTaskName()) : task.getReferenceTaskName(); WorkflowTask taskToSchedule = workflowDef.getNextTask(taskReferenceName); while (isTaskSkipped(taskToSchedule, workflow)) { taskToSchedule = workflowDef.getNextTask(taskToSchedule.getTaskReferenceName()); } if (taskToSchedule != null && TaskType.DO_WHILE.name().equals(taskToSchedule.getType())) { // check if already has this DO_WHILE task, ignore it if it already exists String nextTaskReferenceName = taskToSchedule.getTaskReferenceName(); if (workflow.getTasks().stream() .anyMatch( runningTask -> runningTask .getReferenceTaskName() .equals(nextTaskReferenceName))) { return Collections.emptyList(); } } if (taskToSchedule != null) { return getTasksToBeScheduled(workflow, taskToSchedule, 0); } return Collections.emptyList(); } private String getNextTasksToBeScheduled(WorkflowModel workflow, TaskModel task) { final WorkflowDef def = workflow.getWorkflowDefinition(); String taskReferenceName = task.getReferenceTaskName(); WorkflowTask taskToSchedule = def.getNextTask(taskReferenceName); while (isTaskSkipped(taskToSchedule, workflow)) { taskToSchedule = def.getNextTask(taskToSchedule.getTaskReferenceName()); } return taskToSchedule == null ? null : taskToSchedule.getTaskReferenceName(); } @VisibleForTesting Optional retry( TaskDef taskDefinition, WorkflowTask workflowTask, TaskModel task, WorkflowModel workflow) throws TerminateWorkflowException { int retryCount = task.getRetryCount(); if (taskDefinition == null) { taskDefinition = metadataDAO.getTaskDef(task.getTaskDefName()); } final int expectedRetryCount = taskDefinition == null ? 0 : Optional.ofNullable(workflowTask) .map(WorkflowTask::getRetryCount) .orElse(taskDefinition.getRetryCount()); if (!task.getStatus().isRetriable() || TaskType.isBuiltIn(task.getTaskType()) || expectedRetryCount <= retryCount) { if (workflowTask != null && workflowTask.isOptional()) { return Optional.empty(); } WorkflowModel.Status status; switch (task.getStatus()) { case CANCELED: status = WorkflowModel.Status.TERMINATED; break; case TIMED_OUT: status = WorkflowModel.Status.TIMED_OUT; break; default: status = WorkflowModel.Status.FAILED; break; } updateWorkflowOutput(workflow, task); final String errMsg = String.format( "Task %s failed with status: %s and reason: '%s'", task.getTaskId(), status, task.getReasonForIncompletion()); throw new TerminateWorkflowException(errMsg, status, task); } // retry... - but not immediately - put a delay... int startDelay = taskDefinition.getRetryDelaySeconds(); switch (taskDefinition.getRetryLogic()) { case FIXED: startDelay = taskDefinition.getRetryDelaySeconds(); break; case LINEAR_BACKOFF: int linearRetryDelaySeconds = taskDefinition.getRetryDelaySeconds() * taskDefinition.getBackoffScaleFactor() * (task.getRetryCount() + 1); // Reset integer overflow to max value startDelay = linearRetryDelaySeconds < 0 ? Integer.MAX_VALUE : linearRetryDelaySeconds; break; case EXPONENTIAL_BACKOFF: int exponentialRetryDelaySeconds = taskDefinition.getRetryDelaySeconds() * (int) Math.pow(2, task.getRetryCount()); // Reset integer overflow to max value startDelay = exponentialRetryDelaySeconds < 0 ? Integer.MAX_VALUE : exponentialRetryDelaySeconds; break; } task.setRetried(true); TaskModel rescheduled = task.copy(); rescheduled.setStartDelayInSeconds(startDelay); rescheduled.setCallbackAfterSeconds(startDelay); rescheduled.setRetryCount(task.getRetryCount() + 1); rescheduled.setRetried(false); rescheduled.setTaskId(idGenerator.generate()); rescheduled.setRetriedTaskId(task.getTaskId()); rescheduled.setStatus(SCHEDULED); rescheduled.setPollCount(0); rescheduled.setInputData(new HashMap<>(task.getInputData())); rescheduled.setReasonForIncompletion(null); rescheduled.setSubWorkflowId(null); rescheduled.setSeq(0); rescheduled.setScheduledTime(0); rescheduled.setStartTime(0); rescheduled.setEndTime(0); rescheduled.setWorkerId(null); if (StringUtils.isNotBlank(task.getExternalInputPayloadStoragePath())) { rescheduled.setExternalInputPayloadStoragePath( task.getExternalInputPayloadStoragePath()); } else { rescheduled.addInput(task.getInputData()); } if (workflowTask != null && workflow.getWorkflowDefinition().getSchemaVersion() > 1) { Map taskInput = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflow, rescheduled.getTaskId(), taskDefinition); rescheduled.addInput(taskInput); } // for the schema version 1, we do not have to recompute the inputs return Optional.of(rescheduled); } @VisibleForTesting void checkWorkflowTimeout(WorkflowModel workflow) { WorkflowDef workflowDef = workflow.getWorkflowDefinition(); if (workflowDef == null) { LOGGER.warn("Missing workflow definition : {}", workflow.getWorkflowId()); return; } if (workflow.getStatus().isTerminal() || workflowDef.getTimeoutSeconds() <= 0) { return; } long timeout = 1000L * workflowDef.getTimeoutSeconds(); long now = System.currentTimeMillis(); long elapsedTime = workflow.getLastRetriedTime() > 0 ? now - workflow.getLastRetriedTime() : now - workflow.getCreateTime(); if (elapsedTime < timeout) { return; } String reason = String.format( "Workflow timed out after %d seconds. Timeout configured as %d seconds. " + "Timeout policy configured to %s", elapsedTime / 1000L, workflowDef.getTimeoutSeconds(), workflowDef.getTimeoutPolicy().name()); switch (workflowDef.getTimeoutPolicy()) { case ALERT_ONLY: LOGGER.info("{} {}", workflow.getWorkflowId(), reason); Monitors.recordWorkflowTermination( workflow.getWorkflowName(), WorkflowModel.Status.TIMED_OUT, workflow.getOwnerApp()); return; case TIME_OUT_WF: throw new TerminateWorkflowException(reason, WorkflowModel.Status.TIMED_OUT); } } @VisibleForTesting void checkTaskTimeout(TaskDef taskDef, TaskModel task) { if (taskDef == null) { LOGGER.warn( "Missing task definition for task:{}/{} in workflow:{}", task.getTaskId(), task.getTaskDefName(), task.getWorkflowInstanceId()); return; } if (task.getStatus().isTerminal() || taskDef.getTimeoutSeconds() <= 0 || task.getStartTime() <= 0) { return; } long timeout = 1000L * taskDef.getTimeoutSeconds(); long now = System.currentTimeMillis(); long elapsedTime = now - (task.getStartTime() + ((long) task.getStartDelayInSeconds() * 1000L)); if (elapsedTime < timeout) { return; } String reason = String.format( "Task timed out after %d seconds. Timeout configured as %d seconds. " + "Timeout policy configured to %s", elapsedTime / 1000L, taskDef.getTimeoutSeconds(), taskDef.getTimeoutPolicy().name()); timeoutTaskWithTimeoutPolicy(reason, taskDef, task); } @VisibleForTesting void checkTaskPollTimeout(TaskDef taskDef, TaskModel task) { if (taskDef == null) { LOGGER.warn( "Missing task definition for task:{}/{} in workflow:{}", task.getTaskId(), task.getTaskDefName(), task.getWorkflowInstanceId()); return; } if (taskDef.getPollTimeoutSeconds() == null || taskDef.getPollTimeoutSeconds() <= 0 || !task.getStatus().equals(SCHEDULED)) { return; } final long pollTimeout = 1000L * taskDef.getPollTimeoutSeconds(); final long adjustedPollTimeout = pollTimeout + task.getCallbackAfterSeconds() * 1000L; final long now = System.currentTimeMillis(); final long pollElapsedTime = now - (task.getScheduledTime() + ((long) task.getStartDelayInSeconds() * 1000L)); if (pollElapsedTime < adjustedPollTimeout) { return; } String reason = String.format( "Task poll timed out after %d seconds. Poll timeout configured as %d seconds. Timeout policy configured to %s", pollElapsedTime / 1000L, pollTimeout / 1000L, taskDef.getTimeoutPolicy().name()); timeoutTaskWithTimeoutPolicy(reason, taskDef, task); } void timeoutTaskWithTimeoutPolicy(String reason, TaskDef taskDef, TaskModel task) { Monitors.recordTaskTimeout(task.getTaskDefName()); switch (taskDef.getTimeoutPolicy()) { case ALERT_ONLY: LOGGER.info(reason); return; case RETRY: task.setStatus(TIMED_OUT); task.setReasonForIncompletion(reason); return; case TIME_OUT_WF: task.setStatus(TIMED_OUT); task.setReasonForIncompletion(reason); throw new TerminateWorkflowException(reason, WorkflowModel.Status.TIMED_OUT, task); } } @VisibleForTesting boolean isResponseTimedOut(TaskDef taskDefinition, TaskModel task) { if (taskDefinition == null) { LOGGER.warn( "missing task type : {}, workflowId= {}", task.getTaskDefName(), task.getWorkflowInstanceId()); return false; } if (task.getStatus().isTerminal() || isAyncCompleteSystemTask(task)) { return false; } // calculate pendingTime long now = System.currentTimeMillis(); long callbackTime = 1000L * task.getCallbackAfterSeconds(); long referenceTime = task.getUpdateTime() > 0 ? task.getUpdateTime() : task.getScheduledTime(); long pendingTime = now - (referenceTime + callbackTime); Monitors.recordTaskPendingTime(task.getTaskType(), task.getWorkflowType(), pendingTime); long thresholdMS = taskPendingTimeThresholdMins * 60 * 1000; if (pendingTime > thresholdMS) { LOGGER.warn( "Task: {} of type: {} in workflow: {}/{} is in pending state for longer than {} ms", task.getTaskId(), task.getTaskType(), task.getWorkflowInstanceId(), task.getWorkflowType(), thresholdMS); } if (!task.getStatus().equals(IN_PROGRESS) || taskDefinition.getResponseTimeoutSeconds() == 0) { return false; } LOGGER.debug( "Evaluating responseTimeOut for Task: {}, with Task Definition: {}", task, taskDefinition); long responseTimeout = 1000L * taskDefinition.getResponseTimeoutSeconds(); long adjustedResponseTimeout = responseTimeout + callbackTime; long noResponseTime = now - task.getUpdateTime(); if (noResponseTime < adjustedResponseTimeout) { LOGGER.debug( "Current responseTime: {} has not exceeded the configured responseTimeout of {} for the Task: {} with Task Definition: {}", pendingTime, responseTimeout, task, taskDefinition); return false; } Monitors.recordTaskResponseTimeout(task.getTaskDefName()); return true; } private void timeoutTask(TaskDef taskDef, TaskModel task) { String reason = "responseTimeout: " + taskDef.getResponseTimeoutSeconds() + " exceeded for the taskId: " + task.getTaskId() + " with Task Definition: " + task.getTaskDefName(); LOGGER.debug(reason); task.setStatus(TIMED_OUT); task.setReasonForIncompletion(reason); } public List getTasksToBeScheduled( WorkflowModel workflow, WorkflowTask taskToSchedule, int retryCount) { return getTasksToBeScheduled(workflow, taskToSchedule, retryCount, null); } public List getTasksToBeScheduled( WorkflowModel workflow, WorkflowTask taskToSchedule, int retryCount, String retriedTaskId) { Map input = parametersUtils.getTaskInput( taskToSchedule.getInputParameters(), workflow, null, null); String type = taskToSchedule.getType(); // get tasks already scheduled (in progress/terminal) for this workflow instance List tasksInWorkflow = workflow.getTasks().stream() .filter( runningTask -> runningTask.getStatus().equals(TaskModel.Status.IN_PROGRESS) || runningTask.getStatus().isTerminal()) .map(TaskModel::getReferenceTaskName) .collect(Collectors.toList()); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(taskToSchedule.getTaskDefinition()) .withWorkflowTask(taskToSchedule) .withTaskInput(input) .withRetryCount(retryCount) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .withDeciderService(this) .build(); // For static forks, each branch of the fork creates a join task upon completion for // dynamic forks, a join task is created with the fork and also with each branch of the // fork. // A new task must only be scheduled if a task, with the same reference name is not already // in this workflow instance return taskMappers .getOrDefault(type, taskMappers.get(USER_DEFINED.name())) .getMappedTasks(taskMapperContext) .stream() .filter(task -> !tasksInWorkflow.contains(task.getReferenceTaskName())) .collect(Collectors.toList()); } private boolean isTaskSkipped(WorkflowTask taskToSchedule, WorkflowModel workflow) { try { boolean isTaskSkipped = false; if (taskToSchedule != null) { TaskModel t = workflow.getTaskByRefName(taskToSchedule.getTaskReferenceName()); if (t == null) { isTaskSkipped = false; } else if (t.getStatus().equals(SKIPPED)) { isTaskSkipped = true; } } return isTaskSkipped; } catch (Exception e) { throw new TerminateWorkflowException(e.getMessage()); } } private boolean isAyncCompleteSystemTask(TaskModel task) { return systemTaskRegistry.isSystemTask(task.getTaskType()) && systemTaskRegistry.get(task.getTaskType()).isAsyncComplete(task); } public static class DeciderOutcome { List tasksToBeScheduled = new LinkedList<>(); List tasksToBeUpdated = new LinkedList<>(); boolean isComplete; TaskModel terminateTask; private DeciderOutcome() {} } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/StartWorkflowInput.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.util.Map; import java.util.Objects; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; public class StartWorkflowInput { private String name; private Integer version; private WorkflowDef workflowDefinition; private Map workflowInput; private String externalInputPayloadStoragePath; private String correlationId; private Integer priority; private String parentWorkflowId; private String parentWorkflowTaskId; private String event; private Map taskToDomain; private String workflowId; private String triggeringWorkflowId; public StartWorkflowInput() {} public StartWorkflowInput(StartWorkflowRequest startWorkflowRequest) { this.name = startWorkflowRequest.getName(); this.version = startWorkflowRequest.getVersion(); this.workflowDefinition = startWorkflowRequest.getWorkflowDef(); this.correlationId = startWorkflowRequest.getCorrelationId(); this.priority = startWorkflowRequest.getPriority(); this.workflowInput = startWorkflowRequest.getInput(); this.externalInputPayloadStoragePath = startWorkflowRequest.getExternalInputPayloadStoragePath(); this.taskToDomain = startWorkflowRequest.getTaskToDomain(); } public String getName() { return name; } public void setName(String name) { this.name = name; } public Integer getVersion() { return version; } public void setVersion(Integer version) { this.version = version; } public WorkflowDef getWorkflowDefinition() { return workflowDefinition; } public void setWorkflowDefinition(WorkflowDef workflowDefinition) { this.workflowDefinition = workflowDefinition; } public Map getWorkflowInput() { return workflowInput; } public void setWorkflowInput(Map workflowInput) { this.workflowInput = workflowInput; } public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } public String getCorrelationId() { return correlationId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public Integer getPriority() { return priority; } public void setPriority(Integer priority) { this.priority = priority; } public String getParentWorkflowId() { return parentWorkflowId; } public void setParentWorkflowId(String parentWorkflowId) { this.parentWorkflowId = parentWorkflowId; } public String getParentWorkflowTaskId() { return parentWorkflowTaskId; } public void setParentWorkflowTaskId(String parentWorkflowTaskId) { this.parentWorkflowTaskId = parentWorkflowTaskId; } public String getEvent() { return event; } public void setEvent(String event) { this.event = event; } public Map getTaskToDomain() { return taskToDomain; } public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } public String getWorkflowId() { return workflowId; } public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } public String getTriggeringWorkflowId() { return triggeringWorkflowId; } public void setTriggeringWorkflowId(String triggeringWorkflowId) { this.triggeringWorkflowId = triggeringWorkflowId; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; StartWorkflowInput that = (StartWorkflowInput) o; return Objects.equals(name, that.name) && Objects.equals(version, that.version) && Objects.equals(workflowDefinition, that.workflowDefinition) && Objects.equals(workflowInput, that.workflowInput) && Objects.equals( externalInputPayloadStoragePath, that.externalInputPayloadStoragePath) && Objects.equals(correlationId, that.correlationId) && Objects.equals(priority, that.priority) && Objects.equals(parentWorkflowId, that.parentWorkflowId) && Objects.equals(parentWorkflowTaskId, that.parentWorkflowTaskId) && Objects.equals(event, that.event) && Objects.equals(taskToDomain, that.taskToDomain) && Objects.equals(triggeringWorkflowId, that.triggeringWorkflowId) && Objects.equals(workflowId, that.workflowId); } @Override public int hashCode() { return Objects.hash( name, version, workflowDefinition, workflowInput, externalInputPayloadStoragePath, correlationId, priority, parentWorkflowId, parentWorkflowTaskId, event, taskToDomain, triggeringWorkflowId, workflowId); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.*; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.event.WorkflowCreationEvent; import com.netflix.conductor.core.event.WorkflowEvaluationEvent; import com.netflix.conductor.core.exception.*; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.Terminate; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.listener.TaskStatusListener; import com.netflix.conductor.core.listener.WorkflowStatusListener; import com.netflix.conductor.core.metadata.MetadataMapperService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; /** Workflow services provider interface */ @Trace @Component public class WorkflowExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowExecutor.class); private static final int EXPEDITED_PRIORITY = 10; private static final String CLASS_NAME = WorkflowExecutor.class.getSimpleName(); private static final Predicate UNSUCCESSFUL_TERMINAL_TASK = task -> !task.getStatus().isSuccessful() && task.getStatus().isTerminal(); private static final Predicate UNSUCCESSFUL_JOIN_TASK = UNSUCCESSFUL_TERMINAL_TASK.and(t -> TaskType.TASK_TYPE_JOIN.equals(t.getTaskType())); private static final Predicate NON_TERMINAL_TASK = task -> !task.getStatus().isTerminal(); private final MetadataDAO metadataDAO; private final QueueDAO queueDAO; private final DeciderService deciderService; private final ConductorProperties properties; private final MetadataMapperService metadataMapperService; private final ExecutionDAOFacade executionDAOFacade; private final ParametersUtils parametersUtils; private final IDGenerator idGenerator; private final WorkflowStatusListener workflowStatusListener; private final TaskStatusListener taskStatusListener; private final SystemTaskRegistry systemTaskRegistry; private final ApplicationEventPublisher eventPublisher; private long activeWorkerLastPollMs; private final ExecutionLockService executionLockService; private final Predicate validateLastPolledTime = pollData -> pollData.getLastPollTime() > System.currentTimeMillis() - activeWorkerLastPollMs; public WorkflowExecutor( DeciderService deciderService, MetadataDAO metadataDAO, QueueDAO queueDAO, MetadataMapperService metadataMapperService, WorkflowStatusListener workflowStatusListener, TaskStatusListener taskStatusListener, ExecutionDAOFacade executionDAOFacade, ConductorProperties properties, ExecutionLockService executionLockService, SystemTaskRegistry systemTaskRegistry, ParametersUtils parametersUtils, IDGenerator idGenerator, ApplicationEventPublisher eventPublisher) { this.deciderService = deciderService; this.metadataDAO = metadataDAO; this.queueDAO = queueDAO; this.properties = properties; this.metadataMapperService = metadataMapperService; this.executionDAOFacade = executionDAOFacade; this.activeWorkerLastPollMs = properties.getActiveWorkerLastPollTimeout().toMillis(); this.workflowStatusListener = workflowStatusListener; this.taskStatusListener = taskStatusListener; this.executionLockService = executionLockService; this.parametersUtils = parametersUtils; this.idGenerator = idGenerator; this.systemTaskRegistry = systemTaskRegistry; this.eventPublisher = eventPublisher; } /** * @param workflowId the id of the workflow for which task callbacks are to be reset * @throws ConflictException if the workflow is in terminal state */ public void resetCallbacksForWorkflow(String workflowId) { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (workflow.getStatus().isTerminal()) { throw new ConflictException( "Workflow is in terminal state. Status = %s", workflow.getStatus()); } // Get SIMPLE tasks in SCHEDULED state that have callbackAfterSeconds > 0 and set the // callbackAfterSeconds to 0 workflow.getTasks().stream() .filter( task -> !systemTaskRegistry.isSystemTask(task.getTaskType()) && SCHEDULED == task.getStatus() && task.getCallbackAfterSeconds() > 0) .forEach( task -> { if (queueDAO.resetOffsetTime( QueueUtils.getQueueName(task), task.getTaskId())) { task.setCallbackAfterSeconds(0); executionDAOFacade.updateTask(task); } }); } public String rerun(RerunWorkflowRequest request) { Utils.checkNotNull(request.getReRunFromWorkflowId(), "reRunFromWorkflowId is missing"); if (!rerunWF( request.getReRunFromWorkflowId(), request.getReRunFromTaskId(), request.getTaskInput(), request.getWorkflowInput(), request.getCorrelationId())) { throw new IllegalArgumentException( "Task " + request.getReRunFromTaskId() + " not found"); } return request.getReRunFromWorkflowId(); } /** * @param workflowId the id of the workflow to be restarted * @param useLatestDefinitions if true, use the latest workflow and task definitions upon * restart * @throws ConflictException Workflow is not in a terminal state. * @throws NotFoundException Workflow definition is not found or Workflow is deemed * non-restartable as per workflow definition. */ public void restart(String workflowId, boolean useLatestDefinitions) { final WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (!workflow.getStatus().isTerminal()) { String errorMsg = String.format( "Workflow: %s is not in terminal state, unable to restart.", workflow); LOGGER.error(errorMsg); throw new ConflictException(errorMsg); } WorkflowDef workflowDef; if (useLatestDefinitions) { workflowDef = metadataDAO .getLatestWorkflowDef(workflow.getWorkflowName()) .orElseThrow( () -> new NotFoundException( "Unable to find latest definition for %s", workflowId)); workflow.setWorkflowDefinition(workflowDef); workflowDef = metadataMapperService.populateTaskDefinitions(workflowDef); } else { workflowDef = Optional.ofNullable(workflow.getWorkflowDefinition()) .orElseGet( () -> metadataDAO .getWorkflowDef( workflow.getWorkflowName(), workflow.getWorkflowVersion()) .orElseThrow( () -> new NotFoundException( "Unable to find definition for %s", workflowId))); } if (!workflowDef.isRestartable() && workflow.getStatus() .equals( WorkflowModel.Status .COMPLETED)) { // Can only restart non-completed workflows // when the configuration is set to false throw new NotFoundException("Workflow: %s is non-restartable", workflow); } // Reset the workflow in the primary datastore and remove from indexer; then re-create it executionDAOFacade.resetWorkflow(workflowId); workflow.getTasks().clear(); workflow.setReasonForIncompletion(null); workflow.setFailedTaskId(null); workflow.setCreateTime(System.currentTimeMillis()); workflow.setEndTime(0); workflow.setLastRetriedTime(0); // Change the status to running workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOutput(null); workflow.setExternalOutputPayloadStoragePath(null); try { executionDAOFacade.createWorkflow(workflow); } catch (Exception e) { Monitors.recordWorkflowStartError( workflowDef.getName(), WorkflowContext.get().getClientApp()); LOGGER.error("Unable to restart workflow: {}", workflowDef.getName(), e); terminateWorkflow(workflowId, "Error when restarting the workflow"); throw e; } metadataMapperService.populateWorkflowWithDefinitions(workflow); decide(workflowId); updateAndPushParents(workflow, "restarted"); } /** * Gets the last instance of each failed task and reschedule each Gets all cancelled tasks and * schedule all of them except JOIN (join should change status to INPROGRESS) Switch workflow * back to RUNNING status and call decider. * * @param workflowId the id of the workflow to be retried */ public void retry(String workflowId, boolean resumeSubworkflowTasks) { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (!workflow.getStatus().isTerminal()) { throw new NotFoundException( "Workflow is still running. status=%s", workflow.getStatus()); } if (workflow.getTasks().isEmpty()) { throw new ConflictException("Workflow has not started yet"); } if (resumeSubworkflowTasks) { Optional taskToRetry = workflow.getTasks().stream().filter(UNSUCCESSFUL_TERMINAL_TASK).findFirst(); if (taskToRetry.isPresent()) { workflow = findLastFailedSubWorkflowIfAny(taskToRetry.get(), workflow); retry(workflow); updateAndPushParents(workflow, "retried"); } } else { retry(workflow); updateAndPushParents(workflow, "retried"); } } private void updateAndPushParents(WorkflowModel workflow, String operation) { String workflowIdentifier = ""; while (workflow.hasParent()) { // update parent's sub workflow task TaskModel subWorkflowTask = executionDAOFacade.getTaskModel(workflow.getParentWorkflowTaskId()); if (subWorkflowTask.getWorkflowTask().isOptional()) { // break out LOGGER.info( "Sub workflow task {} is optional, skip updating parents", subWorkflowTask); break; } subWorkflowTask.setSubworkflowChanged(true); subWorkflowTask.setStatus(IN_PROGRESS); executionDAOFacade.updateTask(subWorkflowTask); // add an execution log String currentWorkflowIdentifier = workflow.toShortString(); workflowIdentifier = !workflowIdentifier.equals("") ? String.format( "%s -> %s", currentWorkflowIdentifier, workflowIdentifier) : currentWorkflowIdentifier; TaskExecLog log = new TaskExecLog( String.format("Sub workflow %s %s.", workflowIdentifier, operation)); log.setTaskId(subWorkflowTask.getTaskId()); executionDAOFacade.addTaskExecLog(Collections.singletonList(log)); LOGGER.info("Task {} updated. {}", log.getTaskId(), log.getLog()); // push the parent workflow to decider queue for asynchronous 'decide' String parentWorkflowId = workflow.getParentWorkflowId(); WorkflowModel parentWorkflow = executionDAOFacade.getWorkflowModel(parentWorkflowId, true); parentWorkflow.setStatus(WorkflowModel.Status.RUNNING); parentWorkflow.setLastRetriedTime(System.currentTimeMillis()); executionDAOFacade.updateWorkflow(parentWorkflow); expediteLazyWorkflowEvaluation(parentWorkflowId); workflow = parentWorkflow; } } private void retry(WorkflowModel workflow) { // Get all FAILED or CANCELED tasks that are not COMPLETED (or reach other terminal states) // on further executions. // // Eg: for Seq of tasks task1.CANCELED, task1.COMPLETED, task1 shouldn't be retried. // Throw an exception if there are no FAILED tasks. // Handle JOIN task CANCELED status as special case. Map retriableMap = new HashMap<>(); for (TaskModel task : workflow.getTasks()) { switch (task.getStatus()) { case FAILED: case FAILED_WITH_TERMINAL_ERROR: case TIMED_OUT: retriableMap.put(task.getReferenceTaskName(), task); break; case CANCELED: if (task.getTaskType().equalsIgnoreCase(TaskType.JOIN.toString()) || task.getTaskType().equalsIgnoreCase(TaskType.DO_WHILE.toString())) { task.setStatus(IN_PROGRESS); addTaskToQueue(task); // Task doesn't have to be updated yet. Will be updated along with other // Workflow tasks downstream. } else { retriableMap.put(task.getReferenceTaskName(), task); } break; default: retriableMap.remove(task.getReferenceTaskName()); break; } } // if workflow TIMED_OUT due to timeoutSeconds configured in the workflow definition, // it may not have any unsuccessful tasks that can be retried if (retriableMap.values().size() == 0 && workflow.getStatus() != WorkflowModel.Status.TIMED_OUT) { throw new ConflictException( "There are no retryable tasks! Use restart if you want to attempt entire workflow execution again."); } // Update Workflow with new status. // This should load Workflow from archive, if archived. workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setLastRetriedTime(System.currentTimeMillis()); String lastReasonForIncompletion = workflow.getReasonForIncompletion(); workflow.setReasonForIncompletion(null); // Add to decider queue queueDAO.push( DECIDER_QUEUE, workflow.getWorkflowId(), workflow.getPriority(), properties.getWorkflowOffsetTimeout().getSeconds()); executionDAOFacade.updateWorkflow(workflow); LOGGER.info( "Workflow {} that failed due to '{}' was retried", workflow.toShortString(), lastReasonForIncompletion); // taskToBeRescheduled would set task `retried` to true, and hence it's important to // updateTasks after obtaining task copy from taskToBeRescheduled. final WorkflowModel finalWorkflow = workflow; List retriableTasks = retriableMap.values().stream() .sorted(Comparator.comparingInt(TaskModel::getSeq)) .map(task -> taskToBeRescheduled(finalWorkflow, task)) .collect(Collectors.toList()); dedupAndAddTasks(workflow, retriableTasks); // Note: updateTasks before updateWorkflow might fail when Workflow is archived and doesn't // exist in primary store. executionDAOFacade.updateTasks(workflow.getTasks()); scheduleTask(workflow, retriableTasks); } private WorkflowModel findLastFailedSubWorkflowIfAny( TaskModel task, WorkflowModel parentWorkflow) { if (TaskType.TASK_TYPE_SUB_WORKFLOW.equals(task.getTaskType()) && UNSUCCESSFUL_TERMINAL_TASK.test(task)) { WorkflowModel subWorkflow = executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true); Optional taskToRetry = subWorkflow.getTasks().stream().filter(UNSUCCESSFUL_TERMINAL_TASK).findFirst(); if (taskToRetry.isPresent()) { return findLastFailedSubWorkflowIfAny(taskToRetry.get(), subWorkflow); } } return parentWorkflow; } /** * Reschedule a task * * @param task failed or cancelled task * @return new instance of a task with "SCHEDULED" status */ private TaskModel taskToBeRescheduled(WorkflowModel workflow, TaskModel task) { TaskModel taskToBeRetried = task.copy(); taskToBeRetried.setTaskId(idGenerator.generate()); taskToBeRetried.setRetriedTaskId(task.getTaskId()); taskToBeRetried.setStatus(SCHEDULED); taskToBeRetried.setRetryCount(task.getRetryCount() + 1); taskToBeRetried.setRetried(false); taskToBeRetried.setPollCount(0); taskToBeRetried.setCallbackAfterSeconds(0); taskToBeRetried.setSubWorkflowId(null); taskToBeRetried.setScheduledTime(0); taskToBeRetried.setStartTime(0); taskToBeRetried.setEndTime(0); taskToBeRetried.setWorkerId(null); taskToBeRetried.setReasonForIncompletion(null); taskToBeRetried.setSeq(0); // perform parameter replacement for retried task Map taskInput = parametersUtils.getTaskInput( taskToBeRetried.getWorkflowTask().getInputParameters(), workflow, taskToBeRetried.getWorkflowTask().getTaskDefinition(), taskToBeRetried.getTaskId()); taskToBeRetried.getInputData().putAll(taskInput); task.setRetried(true); // since this task is being retried and a retry has been computed, task lifecycle is // complete task.setExecuted(true); return taskToBeRetried; } private void endExecution(WorkflowModel workflow, TaskModel terminateTask) { if (terminateTask != null) { String terminationStatus = (String) terminateTask .getInputData() .get(Terminate.getTerminationStatusParameter()); String reason = (String) terminateTask .getInputData() .get(Terminate.getTerminationReasonParameter()); if (StringUtils.isBlank(reason)) { reason = String.format( "Workflow is %s by TERMINATE task: %s", terminationStatus, terminateTask.getTaskId()); } if (WorkflowModel.Status.FAILED.name().equals(terminationStatus)) { workflow.setStatus(WorkflowModel.Status.FAILED); workflow = terminate( workflow, new TerminateWorkflowException( reason, workflow.getStatus(), terminateTask)); } else { workflow.setReasonForIncompletion(reason); workflow = completeWorkflow(workflow); } } else { workflow = completeWorkflow(workflow); } cancelNonTerminalTasks(workflow); } /** * @param workflow the workflow to be completed * @throws ConflictException if workflow is already in terminal state. */ @VisibleForTesting WorkflowModel completeWorkflow(WorkflowModel workflow) { LOGGER.debug("Completing workflow execution for {}", workflow.getWorkflowId()); if (workflow.getStatus().equals(WorkflowModel.Status.COMPLETED)) { queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); // remove from the sweep queue executionDAOFacade.removeFromPendingWorkflow( workflow.getWorkflowName(), workflow.getWorkflowId()); LOGGER.debug("Workflow: {} has already been completed.", workflow.getWorkflowId()); return workflow; } if (workflow.getStatus().isTerminal()) { String msg = "Workflow is already in terminal state. Current status: " + workflow.getStatus(); throw new ConflictException(msg); } deciderService.updateWorkflowOutput(workflow, null); workflow.setStatus(WorkflowModel.Status.COMPLETED); // update the failed reference task names List failedTasks = workflow.getTasks().stream() .filter( t -> FAILED.equals(t.getStatus()) || FAILED_WITH_TERMINAL_ERROR.equals(t.getStatus())) .collect(Collectors.toList()); workflow.getFailedReferenceTaskNames() .addAll( failedTasks.stream() .map(TaskModel::getReferenceTaskName) .collect(Collectors.toSet())); workflow.getFailedTaskNames() .addAll( failedTasks.stream() .map(TaskModel::getTaskDefName) .collect(Collectors.toSet())); executionDAOFacade.updateWorkflow(workflow); LOGGER.debug("Completed workflow execution for {}", workflow.getWorkflowId()); workflowStatusListener.onWorkflowCompletedIfEnabled(workflow); Monitors.recordWorkflowCompletion( workflow.getWorkflowName(), workflow.getEndTime() - workflow.getCreateTime(), workflow.getOwnerApp()); if (workflow.hasParent()) { updateParentWorkflowTask(workflow); LOGGER.info( "{} updated parent {} task {}", workflow.toShortString(), workflow.getParentWorkflowId(), workflow.getParentWorkflowTaskId()); expediteLazyWorkflowEvaluation(workflow.getParentWorkflowId()); } executionLockService.releaseLock(workflow.getWorkflowId()); executionLockService.deleteLock(workflow.getWorkflowId()); return workflow; } public void terminateWorkflow(String workflowId, String reason) { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (WorkflowModel.Status.COMPLETED.equals(workflow.getStatus())) { throw new ConflictException("Cannot terminate a COMPLETED workflow."); } workflow.setStatus(WorkflowModel.Status.TERMINATED); terminateWorkflow(workflow, reason, null); } /** * @param workflow the workflow to be terminated * @param reason the reason for termination * @param failureWorkflow the failure workflow (if any) to be triggered as a result of this * termination */ public WorkflowModel terminateWorkflow( WorkflowModel workflow, String reason, String failureWorkflow) { try { executionLockService.acquireLock(workflow.getWorkflowId(), 60000); if (!workflow.getStatus().isTerminal()) { workflow.setStatus(WorkflowModel.Status.TERMINATED); } try { deciderService.updateWorkflowOutput(workflow, null); } catch (Exception e) { // catch any failure in this step and continue the execution of terminating workflow LOGGER.error( "Failed to update output data for workflow: {}", workflow.getWorkflowId(), e); Monitors.error(CLASS_NAME, "terminateWorkflow"); } // update the failed reference task names List failedTasks = workflow.getTasks().stream() .filter( t -> FAILED.equals(t.getStatus()) || FAILED_WITH_TERMINAL_ERROR.equals( t.getStatus())) .collect(Collectors.toList()); workflow.getFailedReferenceTaskNames() .addAll( failedTasks.stream() .map(TaskModel::getReferenceTaskName) .collect(Collectors.toSet())); workflow.getFailedTaskNames() .addAll( failedTasks.stream() .map(TaskModel::getTaskDefName) .collect(Collectors.toSet())); String workflowId = workflow.getWorkflowId(); workflow.setReasonForIncompletion(reason); executionDAOFacade.updateWorkflow(workflow); workflowStatusListener.onWorkflowTerminatedIfEnabled(workflow); Monitors.recordWorkflowTermination( workflow.getWorkflowName(), workflow.getStatus(), workflow.getOwnerApp()); LOGGER.info("Workflow {} is terminated because of {}", workflowId, reason); List tasks = workflow.getTasks(); try { // Remove from the task queue if they were there tasks.forEach( task -> queueDAO.remove(QueueUtils.getQueueName(task), task.getTaskId())); } catch (Exception e) { LOGGER.warn( "Error removing task(s) from queue during workflow termination : {}", workflowId, e); } if (workflow.hasParent()) { updateParentWorkflowTask(workflow); LOGGER.info( "{} updated parent {} task {}", workflow.toShortString(), workflow.getParentWorkflowId(), workflow.getParentWorkflowTaskId()); expediteLazyWorkflowEvaluation(workflow.getParentWorkflowId()); } if (!StringUtils.isBlank(failureWorkflow)) { Map input = new HashMap<>(workflow.getInput()); input.put("workflowId", workflowId); input.put("reason", reason); input.put("failureStatus", workflow.getStatus().toString()); if (workflow.getFailedTaskId() != null) { input.put("failureTaskId", workflow.getFailedTaskId()); } input.put("failedWorkflow", workflow); try { String failureWFId = idGenerator.generate(); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName(failureWorkflow); startWorkflowInput.setWorkflowInput(input); startWorkflowInput.setCorrelationId(workflow.getCorrelationId()); startWorkflowInput.setTaskToDomain(workflow.getTaskToDomain()); startWorkflowInput.setWorkflowId(failureWFId); startWorkflowInput.setTriggeringWorkflowId(workflowId); eventPublisher.publishEvent(new WorkflowCreationEvent(startWorkflowInput)); workflow.addOutput("conductor.failure_workflow", failureWFId); } catch (Exception e) { LOGGER.error("Failed to start error workflow", e); workflow.getOutput() .put( "conductor.failure_workflow", "Error workflow " + failureWorkflow + " failed to start. reason: " + e.getMessage()); Monitors.recordWorkflowStartError( failureWorkflow, WorkflowContext.get().getClientApp()); } executionDAOFacade.updateWorkflow(workflow); } executionDAOFacade.removeFromPendingWorkflow( workflow.getWorkflowName(), workflow.getWorkflowId()); List erroredTasks = cancelNonTerminalTasks(workflow); if (!erroredTasks.isEmpty()) { throw new NonTransientException( String.format( "Error canceling system tasks: %s", String.join(",", erroredTasks))); } return workflow; } finally { executionLockService.releaseLock(workflow.getWorkflowId()); executionLockService.deleteLock(workflow.getWorkflowId()); } } /** * @param taskResult the task result to be updated. * @throws IllegalArgumentException if the {@link TaskResult} is null. * @throws NotFoundException if the Task is not found. */ public void updateTask(TaskResult taskResult) { if (taskResult == null) { throw new IllegalArgumentException("Task object is null"); } else if (taskResult.isExtendLease()) { extendLease(taskResult); return; } String workflowId = taskResult.getWorkflowInstanceId(); WorkflowModel workflowInstance = executionDAOFacade.getWorkflowModel(workflowId, false); TaskModel task = Optional.ofNullable(executionDAOFacade.getTaskModel(taskResult.getTaskId())) .orElseThrow( () -> new NotFoundException( "No such task found by id: %s", taskResult.getTaskId())); LOGGER.debug("Task: {} belonging to Workflow {} being updated", task, workflowInstance); String taskQueueName = QueueUtils.getQueueName(task); if (task.getStatus().isTerminal()) { // Task was already updated.... queueDAO.remove(taskQueueName, taskResult.getTaskId()); LOGGER.info( "Task: {} has already finished execution with status: {} within workflow: {}. Removed task from queue: {}", task.getTaskId(), task.getStatus(), task.getWorkflowInstanceId(), taskQueueName); Monitors.recordUpdateConflict( task.getTaskType(), workflowInstance.getWorkflowName(), task.getStatus()); return; } if (workflowInstance.getStatus().isTerminal()) { // Workflow is in terminal state queueDAO.remove(taskQueueName, taskResult.getTaskId()); LOGGER.info( "Workflow: {} has already finished execution. Task update for: {} ignored and removed from Queue: {}.", workflowInstance, taskResult.getTaskId(), taskQueueName); Monitors.recordUpdateConflict( task.getTaskType(), workflowInstance.getWorkflowName(), workflowInstance.getStatus()); return; } // for system tasks, setting to SCHEDULED would mean restarting the task which is // undesirable // for worker tasks, set status to SCHEDULED and push to the queue if (!systemTaskRegistry.isSystemTask(task.getTaskType()) && taskResult.getStatus() == TaskResult.Status.IN_PROGRESS) { task.setStatus(SCHEDULED); } else { task.setStatus(TaskModel.Status.valueOf(taskResult.getStatus().name())); } task.setOutputMessage(taskResult.getOutputMessage()); task.setReasonForIncompletion(taskResult.getReasonForIncompletion()); task.setWorkerId(taskResult.getWorkerId()); task.setCallbackAfterSeconds(taskResult.getCallbackAfterSeconds()); task.setOutputData(taskResult.getOutputData()); task.setSubWorkflowId(taskResult.getSubWorkflowId()); if (StringUtils.isNotBlank(taskResult.getExternalOutputPayloadStoragePath())) { task.setExternalOutputPayloadStoragePath( taskResult.getExternalOutputPayloadStoragePath()); } if (task.getStatus().isTerminal()) { task.setEndTime(System.currentTimeMillis()); } // Update message in Task queue based on Task status switch (task.getStatus()) { case COMPLETED: case CANCELED: case FAILED: case FAILED_WITH_TERMINAL_ERROR: case TIMED_OUT: try { queueDAO.remove(taskQueueName, taskResult.getTaskId()); LOGGER.debug( "Task: {} removed from taskQueue: {} since the task status is {}", task, taskQueueName, task.getStatus().name()); } catch (Exception e) { // Ignore exceptions on queue remove as it wouldn't impact task and workflow // execution, and will be cleaned up eventually String errorMsg = String.format( "Error removing the message in queue for task: %s for workflow: %s", task.getTaskId(), workflowId); LOGGER.warn(errorMsg, e); Monitors.recordTaskQueueOpError( task.getTaskType(), workflowInstance.getWorkflowName()); } break; case IN_PROGRESS: case SCHEDULED: try { long callBack = taskResult.getCallbackAfterSeconds(); queueDAO.postpone( taskQueueName, task.getTaskId(), task.getWorkflowPriority(), callBack); LOGGER.debug( "Task: {} postponed in taskQueue: {} since the task status is {} with callbackAfterSeconds: {}", task, taskQueueName, task.getStatus().name(), callBack); } catch (Exception e) { // Throw exceptions on queue postpone, this would impact task execution String errorMsg = String.format( "Error postponing the message in queue for task: %s for workflow: %s", task.getTaskId(), workflowId); LOGGER.error(errorMsg, e); Monitors.recordTaskQueueOpError( task.getTaskType(), workflowInstance.getWorkflowName()); throw new TransientException(errorMsg, e); } break; default: break; } // Throw a TransientException if below operations fail to avoid workflow inconsistencies. try { executionDAOFacade.updateTask(task); } catch (Exception e) { String errorMsg = String.format( "Error updating task: %s for workflow: %s", task.getTaskId(), workflowId); LOGGER.error(errorMsg, e); Monitors.recordTaskUpdateError(task.getTaskType(), workflowInstance.getWorkflowName()); throw new TransientException(errorMsg, e); } try { notifyTaskStatusListener(task); } catch (Exception e) { String errorMsg = String.format( "Error while notifying TaskStatusListener: %s for workflow: %s", task.getTaskId(), workflowId); LOGGER.error(errorMsg, e); } taskResult.getLogs().forEach(taskExecLog -> taskExecLog.setTaskId(task.getTaskId())); executionDAOFacade.addTaskExecLog(taskResult.getLogs()); if (task.getStatus().isTerminal()) { long duration = getTaskDuration(0, task); long lastDuration = task.getEndTime() - task.getStartTime(); Monitors.recordTaskExecutionTime( task.getTaskDefName(), duration, true, task.getStatus()); Monitors.recordTaskExecutionTime( task.getTaskDefName(), lastDuration, false, task.getStatus()); } if (!isLazyEvaluateWorkflow(workflowInstance.getWorkflowDefinition(), task)) { decide(workflowId); } } private void notifyTaskStatusListener(TaskModel task) { switch (task.getStatus()) { case COMPLETED: taskStatusListener.onTaskCompleted(task); break; case CANCELED: taskStatusListener.onTaskCanceled(task); break; case FAILED: taskStatusListener.onTaskFailed(task); break; case FAILED_WITH_TERMINAL_ERROR: taskStatusListener.onTaskFailedWithTerminalError(task); break; case TIMED_OUT: taskStatusListener.onTaskTimedOut(task); break; case IN_PROGRESS: taskStatusListener.onTaskInProgress(task); break; case SCHEDULED: // no-op, already done in addTaskToQueue default: break; } } private void extendLease(TaskResult taskResult) { TaskModel task = Optional.ofNullable(executionDAOFacade.getTaskModel(taskResult.getTaskId())) .orElseThrow( () -> new NotFoundException( "No such task found by id: %s", taskResult.getTaskId())); LOGGER.debug( "Extend lease for Task: {} belonging to Workflow: {}", task, task.getWorkflowInstanceId()); if (!task.getStatus().isTerminal()) { try { executionDAOFacade.extendLease(task); } catch (Exception e) { String errorMsg = String.format( "Error extend lease for Task: %s belonging to Workflow: %s", task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); Monitors.recordTaskExtendLeaseError(task.getTaskType(), task.getWorkflowType()); throw new TransientException(errorMsg, e); } } } /** * Determines if a workflow can be lazily evaluated, if it meets any of these criteria * *

    *
  • The task is NOT a loop task within DO_WHILE *
  • The task is one of the intermediate tasks in a branch within a FORK_JOIN *
  • The task is forked from a FORK_JOIN_DYNAMIC *
* * @param workflowDef The workflow definition of the workflow for which evaluation decision is * to be made * @param task The task which is attempting to trigger the evaluation * @return true if workflow can be lazily evaluated, false otherwise */ @VisibleForTesting boolean isLazyEvaluateWorkflow(WorkflowDef workflowDef, TaskModel task) { if (task.isLoopOverTask()) { return false; } String taskRefName = task.getReferenceTaskName(); List workflowTasks = workflowDef.collectTasks(); List forkTasks = workflowTasks.stream() .filter(t -> t.getType().equals(TaskType.FORK_JOIN.name())) .collect(Collectors.toList()); List joinTasks = workflowTasks.stream() .filter(t -> t.getType().equals(TaskType.JOIN.name())) .collect(Collectors.toList()); if (forkTasks.stream().anyMatch(fork -> fork.has(taskRefName))) { return joinTasks.stream().anyMatch(join -> join.getJoinOn().contains(taskRefName)) && task.getStatus().isSuccessful(); } return workflowTasks.stream().noneMatch(t -> t.getTaskReferenceName().equals(taskRefName)) && task.getStatus().isSuccessful(); } public TaskModel getTask(String taskId) { return Optional.ofNullable(executionDAOFacade.getTaskModel(taskId)) .map( task -> { if (task.getWorkflowTask() != null) { return metadataMapperService.populateTaskWithDefinition(task); } return task; }) .orElse(null); } public List getRunningWorkflows(String workflowName, int version) { return executionDAOFacade.getPendingWorkflowsByName(workflowName, version); } public List getWorkflows(String name, Integer version, Long startTime, Long endTime) { return executionDAOFacade.getWorkflowsByName(name, startTime, endTime).stream() .filter(workflow -> workflow.getWorkflowVersion() == version) .map(Workflow::getWorkflowId) .collect(Collectors.toList()); } public List getRunningWorkflowIds(String workflowName, int version) { return executionDAOFacade.getRunningWorkflowIds(workflowName, version); } @EventListener(WorkflowEvaluationEvent.class) public void handleWorkflowEvaluationEvent(WorkflowEvaluationEvent wee) { decide(wee.getWorkflowModel()); } /** Records a metric for the "decide" process. */ public WorkflowModel decide(String workflowId) { StopWatch watch = new StopWatch(); watch.start(); if (!executionLockService.acquireLock(workflowId)) { return null; } try { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (workflow == null) { // This can happen if the workflowId is incorrect return null; } return decide(workflow); } finally { executionLockService.releaseLock(workflowId); watch.stop(); Monitors.recordWorkflowDecisionTime(watch.getTime()); } } /** * This method overloads the {@link #decide(String)}. It will acquire a lock and evaluate the * state of the workflow. * * @param workflow the workflow to evaluate the state for * @return the workflow */ public WorkflowModel decideWithLock(WorkflowModel workflow) { if (workflow == null) { return null; } StopWatch watch = new StopWatch(); watch.start(); if (!executionLockService.acquireLock(workflow.getWorkflowId())) { return null; } try { return decide(workflow); } finally { executionLockService.releaseLock(workflow.getWorkflowId()); watch.stop(); Monitors.recordWorkflowDecisionTime(watch.getTime()); } } /** * @param workflow the workflow to evaluate the state for * @return true if the workflow has completed (success or failed), false otherwise. Note: This * method does not acquire the lock on the workflow and should ony be called / overridden if * No locking is required or lock is acquired externally */ public WorkflowModel decide(WorkflowModel workflow) { if (workflow.getStatus().isTerminal()) { if (!workflow.getStatus().isSuccessful()) { cancelNonTerminalTasks(workflow); } return workflow; } // we find any sub workflow tasks that have changed // and change the workflow/task state accordingly adjustStateIfSubWorkflowChanged(workflow); try { DeciderService.DeciderOutcome outcome = deciderService.decide(workflow); if (outcome.isComplete) { endExecution(workflow, outcome.terminateTask); return workflow; } List tasksToBeScheduled = outcome.tasksToBeScheduled; setTaskDomains(tasksToBeScheduled, workflow); List tasksToBeUpdated = outcome.tasksToBeUpdated; tasksToBeScheduled = dedupAndAddTasks(workflow, tasksToBeScheduled); boolean stateChanged = scheduleTask(workflow, tasksToBeScheduled); // start for (TaskModel task : outcome.tasksToBeScheduled) { executionDAOFacade.populateTaskData(task); if (systemTaskRegistry.isSystemTask(task.getTaskType()) && NON_TERMINAL_TASK.test(task)) { WorkflowSystemTask workflowSystemTask = systemTaskRegistry.get(task.getTaskType()); if (!workflowSystemTask.isAsync() && workflowSystemTask.execute(workflow, task, this)) { tasksToBeUpdated.add(task); stateChanged = true; } } } if (!outcome.tasksToBeUpdated.isEmpty() || !tasksToBeScheduled.isEmpty()) { executionDAOFacade.updateTasks(tasksToBeUpdated); } if (stateChanged) { return decide(workflow); } if (!outcome.tasksToBeUpdated.isEmpty() || !tasksToBeScheduled.isEmpty()) { executionDAOFacade.updateWorkflow(workflow); } return workflow; } catch (TerminateWorkflowException twe) { LOGGER.info("Execution terminated of workflow: {}", workflow, twe); terminate(workflow, twe); return workflow; } catch (RuntimeException e) { LOGGER.error("Error deciding workflow: {}", workflow.getWorkflowId(), e); throw e; } } private void adjustStateIfSubWorkflowChanged(WorkflowModel workflow) { Optional changedSubWorkflowTask = findChangedSubWorkflowTask(workflow); if (changedSubWorkflowTask.isPresent()) { // reset the flag TaskModel subWorkflowTask = changedSubWorkflowTask.get(); subWorkflowTask.setSubworkflowChanged(false); executionDAOFacade.updateTask(subWorkflowTask); LOGGER.info( "{} reset subworkflowChanged flag for {}", workflow.toShortString(), subWorkflowTask.getTaskId()); // find all terminal and unsuccessful JOIN tasks and set them to IN_PROGRESS if (workflow.getWorkflowDefinition().containsType(TaskType.TASK_TYPE_JOIN) || workflow.getWorkflowDefinition() .containsType(TaskType.TASK_TYPE_FORK_JOIN_DYNAMIC)) { // if we are here, then the SUB_WORKFLOW task could be part of a FORK_JOIN or // FORK_JOIN_DYNAMIC // and the JOIN task(s) needs to be evaluated again, set them to IN_PROGRESS workflow.getTasks().stream() .filter(UNSUCCESSFUL_JOIN_TASK) .peek( task -> { task.setStatus(TaskModel.Status.IN_PROGRESS); addTaskToQueue(task); }) .forEach(executionDAOFacade::updateTask); } } } private Optional findChangedSubWorkflowTask(WorkflowModel workflow) { WorkflowDef workflowDef = Optional.ofNullable(workflow.getWorkflowDefinition()) .orElseGet( () -> metadataDAO .getWorkflowDef( workflow.getWorkflowName(), workflow.getWorkflowVersion()) .orElseThrow( () -> new TransientException( "Workflow Definition is not found"))); if (workflowDef.containsType(TaskType.TASK_TYPE_SUB_WORKFLOW) || workflow.getWorkflowDefinition() .containsType(TaskType.TASK_TYPE_FORK_JOIN_DYNAMIC)) { return workflow.getTasks().stream() .filter( t -> t.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) && t.isSubworkflowChanged() && !t.isRetried()) .findFirst(); } return Optional.empty(); } @VisibleForTesting List cancelNonTerminalTasks(WorkflowModel workflow) { List erroredTasks = new ArrayList<>(); // Update non-terminal tasks' status to CANCELED for (TaskModel task : workflow.getTasks()) { if (!task.getStatus().isTerminal()) { // Cancel the ones which are not completed yet.... task.setStatus(CANCELED); if (systemTaskRegistry.isSystemTask(task.getTaskType())) { WorkflowSystemTask workflowSystemTask = systemTaskRegistry.get(task.getTaskType()); try { workflowSystemTask.cancel(workflow, task, this); } catch (Exception e) { erroredTasks.add(task.getReferenceTaskName()); LOGGER.error( "Error canceling system task:{}/{} in workflow: {}", workflowSystemTask.getTaskType(), task.getTaskId(), workflow.getWorkflowId(), e); } } executionDAOFacade.updateTask(task); } } if (erroredTasks.isEmpty()) { try { workflowStatusListener.onWorkflowFinalizedIfEnabled(workflow); queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); } catch (Exception e) { LOGGER.error( "Error removing workflow: {} from decider queue", workflow.getWorkflowId(), e); } } return erroredTasks; } @VisibleForTesting List dedupAndAddTasks(WorkflowModel workflow, List tasks) { Set tasksInWorkflow = workflow.getTasks().stream() .map(task -> task.getReferenceTaskName() + "_" + task.getRetryCount()) .collect(Collectors.toSet()); List dedupedTasks = tasks.stream() .filter( task -> !tasksInWorkflow.contains( task.getReferenceTaskName() + "_" + task.getRetryCount())) .collect(Collectors.toList()); workflow.getTasks().addAll(dedupedTasks); return dedupedTasks; } /** * @throws ConflictException if the workflow is in terminal state. */ public void pauseWorkflow(String workflowId) { try { executionLockService.acquireLock(workflowId, 60000); WorkflowModel.Status status = WorkflowModel.Status.PAUSED; WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, false); if (workflow.getStatus().isTerminal()) { throw new ConflictException( "Workflow %s has ended, status cannot be updated.", workflow.toShortString()); } if (workflow.getStatus().equals(status)) { return; // Already paused! } workflow.setStatus(status); executionDAOFacade.updateWorkflow(workflow); } finally { executionLockService.releaseLock(workflowId); } // remove from the sweep queue // any exceptions can be ignored, as this is not critical to the pause operation try { queueDAO.remove(DECIDER_QUEUE, workflowId); } catch (Exception e) { LOGGER.info( "[pauseWorkflow] Error removing workflow: {} from decider queue", workflowId, e); } } /** * @param workflowId the workflow to be resumed * @throws IllegalStateException if the workflow is not in PAUSED state */ public void resumeWorkflow(String workflowId) { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, false); if (!workflow.getStatus().equals(WorkflowModel.Status.PAUSED)) { throw new IllegalStateException( "The workflow " + workflowId + " is not PAUSED so cannot resume. " + "Current status is " + workflow.getStatus().name()); } workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setLastRetriedTime(System.currentTimeMillis()); // Add to decider queue queueDAO.push( DECIDER_QUEUE, workflow.getWorkflowId(), workflow.getPriority(), properties.getWorkflowOffsetTimeout().getSeconds()); executionDAOFacade.updateWorkflow(workflow); decide(workflowId); } /** * @param workflowId the id of the workflow * @param taskReferenceName the referenceName of the task to be skipped * @param skipTaskRequest the {@link SkipTaskRequest} object * @throws IllegalStateException */ public void skipTaskFromWorkflow( String workflowId, String taskReferenceName, SkipTaskRequest skipTaskRequest) { WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); // If the workflow is not running then cannot skip any task if (!workflow.getStatus().equals(WorkflowModel.Status.RUNNING)) { String errorMsg = String.format( "The workflow %s is not running so the task referenced by %s cannot be skipped", workflowId, taskReferenceName); throw new IllegalStateException(errorMsg); } // Check if the reference name is as per the workflowdef WorkflowTask workflowTask = workflow.getWorkflowDefinition().getTaskByRefName(taskReferenceName); if (workflowTask == null) { String errorMsg = String.format( "The task referenced by %s does not exist in the WorkflowDefinition %s", taskReferenceName, workflow.getWorkflowName()); throw new IllegalStateException(errorMsg); } // If the task is already started the again it cannot be skipped workflow.getTasks() .forEach( task -> { if (task.getReferenceTaskName().equals(taskReferenceName)) { String errorMsg = String.format( "The task referenced %s has already been processed, cannot be skipped", taskReferenceName); throw new IllegalStateException(errorMsg); } }); // Now create a "SKIPPED" task for this workflow TaskModel taskToBeSkipped = new TaskModel(); taskToBeSkipped.setTaskId(idGenerator.generate()); taskToBeSkipped.setReferenceTaskName(taskReferenceName); taskToBeSkipped.setWorkflowInstanceId(workflowId); taskToBeSkipped.setWorkflowPriority(workflow.getPriority()); taskToBeSkipped.setStatus(SKIPPED); taskToBeSkipped.setEndTime(System.currentTimeMillis()); taskToBeSkipped.setTaskType(workflowTask.getName()); taskToBeSkipped.setCorrelationId(workflow.getCorrelationId()); if (skipTaskRequest != null) { taskToBeSkipped.setInputData(skipTaskRequest.getTaskInput()); taskToBeSkipped.setOutputData(skipTaskRequest.getTaskOutput()); taskToBeSkipped.setInputMessage(skipTaskRequest.getTaskInputMessage()); taskToBeSkipped.setOutputMessage(skipTaskRequest.getTaskOutputMessage()); } executionDAOFacade.createTasks(Collections.singletonList(taskToBeSkipped)); decide(workflow.getWorkflowId()); } public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) { return executionDAOFacade.getWorkflowModel(workflowId, includeTasks); } public void addTaskToQueue(TaskModel task) { // put in queue String taskQueueName = QueueUtils.getQueueName(task); if (task.getCallbackAfterSeconds() > 0) { queueDAO.push( taskQueueName, task.getTaskId(), task.getWorkflowPriority(), task.getCallbackAfterSeconds()); } else { queueDAO.push(taskQueueName, task.getTaskId(), task.getWorkflowPriority(), 0); } LOGGER.debug( "Added task {} with priority {} to queue {} with call back seconds {}", task, task.getWorkflowPriority(), taskQueueName, task.getCallbackAfterSeconds()); } @VisibleForTesting void setTaskDomains(List tasks, WorkflowModel workflow) { Map taskToDomain = workflow.getTaskToDomain(); if (taskToDomain != null) { // Step 1: Apply * mapping to all tasks, if present. String domainstr = taskToDomain.get("*"); if (StringUtils.isNotBlank(domainstr)) { String[] domains = domainstr.split(","); tasks.forEach( task -> { // Filter out SystemTask if (!systemTaskRegistry.isSystemTask(task.getTaskType())) { // Check which domain worker is polling // Set the task domain task.setDomain(getActiveDomain(task.getTaskType(), domains)); } }); } // Step 2: Override additional mappings. tasks.forEach( task -> { if (!systemTaskRegistry.isSystemTask(task.getTaskType())) { String taskDomainstr = taskToDomain.get(task.getTaskType()); if (taskDomainstr != null) { task.setDomain( getActiveDomain( task.getTaskType(), taskDomainstr.split(","))); } } }); } } /** * Gets the active domain from the list of domains where the task is to be queued. The domain * list must be ordered. In sequence, check if any worker has polled for last * `activeWorkerLastPollMs`, if so that is the Active domain. When no active domains are found: *
  • If NO_DOMAIN token is provided, return null. *
  • Else, return last domain from list. * * @param taskType the taskType of the task for which active domain is to be found * @param domains the array of domains for the task. (Must contain atleast one element). * @return the active domain where the task will be queued */ @VisibleForTesting String getActiveDomain(String taskType, String[] domains) { if (domains == null || domains.length == 0) { return null; } return Arrays.stream(domains) .filter(domain -> !domain.equalsIgnoreCase("NO_DOMAIN")) .map(domain -> executionDAOFacade.getTaskPollDataByDomain(taskType, domain.trim())) .filter(Objects::nonNull) .filter(validateLastPolledTime) .findFirst() .map(PollData::getDomain) .orElse( domains[domains.length - 1].trim().equalsIgnoreCase("NO_DOMAIN") ? null : domains[domains.length - 1].trim()); } private long getTaskDuration(long s, TaskModel task) { long duration = task.getEndTime() - task.getStartTime(); s += duration; if (task.getRetriedTaskId() == null) { return s; } return s + getTaskDuration(s, executionDAOFacade.getTaskModel(task.getRetriedTaskId())); } @VisibleForTesting boolean scheduleTask(WorkflowModel workflow, List tasks) { List tasksToBeQueued; boolean startedSystemTasks = false; try { if (tasks == null || tasks.isEmpty()) { return false; } // Get the highest seq number int count = workflow.getTasks().stream().mapToInt(TaskModel::getSeq).max().orElse(0); for (TaskModel task : tasks) { if (task.getSeq() == 0) { // Set only if the seq was not set task.setSeq(++count); } } // metric to track the distribution of number of tasks within a workflow Monitors.recordNumTasksInWorkflow( workflow.getTasks().size() + tasks.size(), workflow.getWorkflowName(), String.valueOf(workflow.getWorkflowVersion())); // Save the tasks in the DAO executionDAOFacade.createTasks(tasks); List systemTasks = tasks.stream() .filter(task -> systemTaskRegistry.isSystemTask(task.getTaskType())) .collect(Collectors.toList()); tasksToBeQueued = tasks.stream() .filter(task -> !systemTaskRegistry.isSystemTask(task.getTaskType())) .collect(Collectors.toList()); // Traverse through all the system tasks, start the sync tasks, in case of async queue // the tasks for (TaskModel task : systemTasks) { WorkflowSystemTask workflowSystemTask = systemTaskRegistry.get(task.getTaskType()); if (workflowSystemTask == null) { throw new NotFoundException( "No system task found by name %s", task.getTaskType()); } if (task.getStatus() != null && !task.getStatus().isTerminal() && task.getStartTime() == 0) { task.setStartTime(System.currentTimeMillis()); } if (!workflowSystemTask.isAsync()) { try { // start execution of synchronous system tasks workflowSystemTask.start(workflow, task, this); } catch (Exception e) { String errorMsg = String.format( "Unable to start system task: %s, {id: %s, name: %s}", task.getTaskType(), task.getTaskId(), task.getTaskDefName()); throw new NonTransientException(errorMsg, e); } startedSystemTasks = true; executionDAOFacade.updateTask(task); } else { tasksToBeQueued.add(task); } } } catch (Exception e) { List taskIds = tasks.stream().map(TaskModel::getTaskId).collect(Collectors.toList()); String errorMsg = String.format( "Error scheduling tasks: %s, for workflow: %s", taskIds, workflow.getWorkflowId()); LOGGER.error(errorMsg, e); Monitors.error(CLASS_NAME, "scheduleTask"); throw new TerminateWorkflowException(errorMsg); } // On addTaskToQueue failures, ignore the exceptions and let WorkflowRepairService take care // of republishing the messages to the queue. try { addTaskToQueue(tasksToBeQueued); } catch (Exception e) { List taskIds = tasksToBeQueued.stream().map(TaskModel::getTaskId).collect(Collectors.toList()); String errorMsg = String.format( "Error pushing tasks to the queue: %s, for workflow: %s", taskIds, workflow.getWorkflowId()); LOGGER.warn(errorMsg, e); Monitors.error(CLASS_NAME, "scheduleTask"); } return startedSystemTasks; } private void addTaskToQueue(final List tasks) { for (TaskModel task : tasks) { addTaskToQueue(task); // notify TaskStatusListener try { taskStatusListener.onTaskScheduled(task); } catch (Exception e) { String errorMsg = String.format( "Error while notifying TaskStatusListener: %s for workflow: %s", task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); } } } private WorkflowModel terminate( final WorkflowModel workflow, TerminateWorkflowException terminateWorkflowException) { if (!workflow.getStatus().isTerminal()) { workflow.setStatus(terminateWorkflowException.getWorkflowStatus()); } if (terminateWorkflowException.getTask() != null && workflow.getFailedTaskId() == null) { workflow.setFailedTaskId(terminateWorkflowException.getTask().getTaskId()); } String failureWorkflow = workflow.getWorkflowDefinition().getFailureWorkflow(); if (failureWorkflow != null) { if (failureWorkflow.startsWith("$")) { String[] paramPathComponents = failureWorkflow.split("\\."); String name = paramPathComponents[2]; // name of the input parameter failureWorkflow = (String) workflow.getInput().get(name); } } if (terminateWorkflowException.getTask() != null) { executionDAOFacade.updateTask(terminateWorkflowException.getTask()); } return terminateWorkflow( workflow, terminateWorkflowException.getMessage(), failureWorkflow); } private boolean rerunWF( String workflowId, String taskId, Map taskInput, Map workflowInput, String correlationId) { // Get the workflow WorkflowModel workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (!workflow.getStatus().isTerminal()) { String errorMsg = String.format( "Workflow: %s is not in terminal state, unable to rerun.", workflow); LOGGER.error(errorMsg); throw new ConflictException(errorMsg); } updateAndPushParents(workflow, "reran"); // If the task Id is null it implies that the entire workflow has to be rerun if (taskId == null) { // remove all tasks workflow.getTasks().forEach(task -> executionDAOFacade.removeTask(task.getTaskId())); workflow.setTasks(new ArrayList<>()); // Set workflow as RUNNING workflow.setStatus(WorkflowModel.Status.RUNNING); // Reset failure reason from previous run to default workflow.setReasonForIncompletion(null); workflow.setFailedTaskId(null); workflow.setFailedReferenceTaskNames(new HashSet<>()); workflow.setFailedTaskNames(new HashSet<>()); if (correlationId != null) { workflow.setCorrelationId(correlationId); } if (workflowInput != null) { workflow.setInput(workflowInput); } queueDAO.push( DECIDER_QUEUE, workflow.getWorkflowId(), workflow.getPriority(), properties.getWorkflowOffsetTimeout().getSeconds()); executionDAOFacade.updateWorkflow(workflow); decide(workflowId); return true; } // Now iterate through the tasks and find the "specific" task TaskModel rerunFromTask = null; for (TaskModel task : workflow.getTasks()) { if (task.getTaskId().equals(taskId)) { rerunFromTask = task; break; } } // If not found look into sub workflows if (rerunFromTask == null) { for (TaskModel task : workflow.getTasks()) { if (task.getTaskType().equalsIgnoreCase(TaskType.TASK_TYPE_SUB_WORKFLOW)) { String subWorkflowId = task.getSubWorkflowId(); if (rerunWF(subWorkflowId, taskId, taskInput, null, null)) { rerunFromTask = task; break; } } } } if (rerunFromTask != null) { // set workflow as RUNNING workflow.setStatus(WorkflowModel.Status.RUNNING); // Reset failure reason from previous run to default workflow.setReasonForIncompletion(null); workflow.setFailedTaskId(null); workflow.setFailedReferenceTaskNames(new HashSet<>()); workflow.setFailedTaskNames(new HashSet<>()); if (correlationId != null) { workflow.setCorrelationId(correlationId); } if (workflowInput != null) { workflow.setInput(workflowInput); } // Add to decider queue queueDAO.push( DECIDER_QUEUE, workflow.getWorkflowId(), workflow.getPriority(), properties.getWorkflowOffsetTimeout().getSeconds()); executionDAOFacade.updateWorkflow(workflow); // update tasks in datastore to update workflow-tasks relationship for archived // workflows executionDAOFacade.updateTasks(workflow.getTasks()); // Remove all tasks after the "rerunFromTask" List filteredTasks = new ArrayList<>(); for (TaskModel task : workflow.getTasks()) { if (task.getSeq() > rerunFromTask.getSeq()) { executionDAOFacade.removeTask(task.getTaskId()); } else { filteredTasks.add(task); } } workflow.setTasks(filteredTasks); // reset fields before restarting the task rerunFromTask.setScheduledTime(System.currentTimeMillis()); rerunFromTask.setStartTime(0); rerunFromTask.setUpdateTime(0); rerunFromTask.setEndTime(0); rerunFromTask.clearOutput(); rerunFromTask.setRetried(false); rerunFromTask.setExecuted(false); if (rerunFromTask.getTaskType().equalsIgnoreCase(TaskType.TASK_TYPE_SUB_WORKFLOW)) { // if task is sub workflow set task as IN_PROGRESS and reset start time rerunFromTask.setStatus(IN_PROGRESS); rerunFromTask.setStartTime(System.currentTimeMillis()); } else { if (taskInput != null) { rerunFromTask.setInputData(taskInput); } if (systemTaskRegistry.isSystemTask(rerunFromTask.getTaskType()) && !systemTaskRegistry.get(rerunFromTask.getTaskType()).isAsync()) { // Start the synchronous system task directly systemTaskRegistry .get(rerunFromTask.getTaskType()) .start(workflow, rerunFromTask, this); } else { // Set the task to rerun as SCHEDULED rerunFromTask.setStatus(SCHEDULED); addTaskToQueue(rerunFromTask); } } executionDAOFacade.updateTask(rerunFromTask); decide(workflow.getWorkflowId()); return true; } return false; } public void scheduleNextIteration(TaskModel loopTask, WorkflowModel workflow) { // Schedule only first loop over task. Rest will be taken care in Decider Service when this // task will get completed. List scheduledLoopOverTasks = deciderService.getTasksToBeScheduled( workflow, loopTask.getWorkflowTask().getLoopOver().get(0), loopTask.getRetryCount(), null); setTaskDomains(scheduledLoopOverTasks, workflow); scheduledLoopOverTasks.forEach( t -> { t.setReferenceTaskName( TaskUtils.appendIteration( t.getReferenceTaskName(), loopTask.getIteration())); t.setIteration(loopTask.getIteration()); }); scheduleTask(workflow, scheduledLoopOverTasks); workflow.getTasks().addAll(scheduledLoopOverTasks); } public TaskDef getTaskDefinition(TaskModel task) { return task.getTaskDefinition() .orElseGet( () -> Optional.ofNullable( metadataDAO.getTaskDef( task.getWorkflowTask().getName())) .orElseThrow( () -> { String reason = String.format( "Invalid task specified. Cannot find task by name %s in the task definitions", task.getWorkflowTask() .getName()); return new TerminateWorkflowException(reason); })); } @VisibleForTesting void updateParentWorkflowTask(WorkflowModel subWorkflow) { TaskModel subWorkflowTask = executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId()); executeSubworkflowTaskAndSyncData(subWorkflow, subWorkflowTask); executionDAOFacade.updateTask(subWorkflowTask); } private void executeSubworkflowTaskAndSyncData( WorkflowModel subWorkflow, TaskModel subWorkflowTask) { WorkflowSystemTask subWorkflowSystemTask = systemTaskRegistry.get(TaskType.TASK_TYPE_SUB_WORKFLOW); subWorkflowSystemTask.execute(subWorkflow, subWorkflowTask, this); } /** * Pushes workflow id into the decider queue with a higher priority to expedite evaluation. * * @param workflowId The workflow to be evaluated at higher priority */ private void expediteLazyWorkflowEvaluation(String workflowId) { if (queueDAO.containsMessage(DECIDER_QUEUE, workflowId)) { queueDAO.postpone(DECIDER_QUEUE, workflowId, EXPEDITED_PRIORITY, 0); } else { queueDAO.push(DECIDER_QUEUE, workflowId, EXPEDITED_PRIORITY, 0); } LOGGER.info("Pushed workflow {} to {} for expedited evaluation", workflowId, DECIDER_QUEUE); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/evaluators/Evaluator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.evaluators; public interface Evaluator { /** * Evaluate the expression using the inputs provided, if required. Evaluation of the expression * depends on the type of the evaluator. * * @param expression Expression to be evaluated. * @param input Input object to the evaluator to help evaluate the expression. * @return Return the evaluation result. */ Object evaluate(String expression, Object input); } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/evaluators/JavascriptEvaluator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.evaluators; import javax.script.ScriptException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.events.ScriptEvaluator; import com.netflix.conductor.core.exception.TerminateWorkflowException; @Component(JavascriptEvaluator.NAME) public class JavascriptEvaluator implements Evaluator { public static final String NAME = "javascript"; private static final Logger LOGGER = LoggerFactory.getLogger(JavascriptEvaluator.class); @Override public Object evaluate(String expression, Object input) { LOGGER.debug("Javascript evaluator -- expression: {}", expression); try { // Evaluate the expression by using the Javascript evaluation engine. Object result = ScriptEvaluator.eval(expression, input); LOGGER.debug("Javascript evaluator -- result: {}", result); return result; } catch (ScriptException e) { LOGGER.error("Error while evaluating script: {}", expression, e); throw new TerminateWorkflowException(e.getMessage()); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/evaluators/ValueParamEvaluator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.evaluators; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.exception.TerminateWorkflowException; @Component(ValueParamEvaluator.NAME) public class ValueParamEvaluator implements Evaluator { public static final String NAME = "value-param"; private static final Logger LOGGER = LoggerFactory.getLogger(ValueParamEvaluator.class); @SuppressWarnings("unchecked") @Override public Object evaluate(String expression, Object input) { LOGGER.debug("ValueParam evaluator -- evaluating: {}", expression); if (input instanceof Map) { Object result = ((Map) input).get(expression); LOGGER.debug("ValueParam evaluator -- result: {}", result); return result; } else { String errorMsg = String.format("Input has to be a JSON object: %s", input.getClass()); LOGGER.error(errorMsg); throw new TerminateWorkflowException(errorMsg); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.script.ScriptException; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.events.ScriptEvaluator; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DECISION} to a List {@link TaskModel} starting with Task of type {@link * TaskType#DECISION} which is marked as IN_PROGRESS, followed by the list of {@link TaskModel} * based on the case expression evaluation in the Decision task. * * @deprecated {@link com.netflix.conductor.core.execution.tasks.Decision} is also deprecated. Use * {@link com.netflix.conductor.core.execution.tasks.Switch} and so ${@link SwitchTaskMapper} * will be used as a result. */ @Deprecated @Component public class DecisionTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(DecisionTaskMapper.class); @Override public String getTaskType() { return TaskType.DECISION.name(); } /** * This method gets the list of tasks that need to scheduled when the task to scheduled is of * type {@link TaskType#DECISION}. * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return List of tasks in the following order: *

      *
    • {@link TaskType#DECISION} with {@link TaskModel.Status#IN_PROGRESS} *
    • List of task based on the evaluation of {@link WorkflowTask#getCaseExpression()} * are scheduled. *
    • In case of no matching result after the evaluation of the {@link * WorkflowTask#getCaseExpression()}, the {@link WorkflowTask#getDefaultCase()} Tasks * are scheduled. *
    */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in DecisionTaskMapper", taskMapperContext); List tasksToBeScheduled = new LinkedList<>(); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); Map taskInput = taskMapperContext.getTaskInput(); int retryCount = taskMapperContext.getRetryCount(); // get the expression to be evaluated String caseValue = getEvaluatedCaseValue(workflowTask, taskInput); // QQ why is the case value and the caseValue passed and caseOutput passes as the same ?? TaskModel decisionTask = taskMapperContext.createTaskModel(); decisionTask.setTaskType(TaskType.TASK_TYPE_DECISION); decisionTask.setTaskDefName(TaskType.TASK_TYPE_DECISION); decisionTask.addInput("case", caseValue); decisionTask.addOutput("caseOutput", Collections.singletonList(caseValue)); decisionTask.setStartTime(System.currentTimeMillis()); decisionTask.setStatus(TaskModel.Status.IN_PROGRESS); tasksToBeScheduled.add(decisionTask); // get the list of tasks based on the decision List selectedTasks = workflowTask.getDecisionCases().get(caseValue); // if the tasks returned are empty based on evaluated case value, then get the default case // if there is one if (selectedTasks == null || selectedTasks.isEmpty()) { selectedTasks = workflowTask.getDefaultCase(); } // once there are selected tasks that need to proceeded as part of the decision, get the // next task to be scheduled by using the decider service if (selectedTasks != null && !selectedTasks.isEmpty()) { WorkflowTask selectedTask = selectedTasks.get(0); // Schedule the first task to be executed... // TODO break out this recursive call using function composition of what needs to be // done and then walk back the condition tree List caseTasks = taskMapperContext .getDeciderService() .getTasksToBeScheduled( workflowModel, selectedTask, retryCount, taskMapperContext.getRetryTaskId()); tasksToBeScheduled.addAll(caseTasks); decisionTask.addInput("hasChildren", "true"); } return tasksToBeScheduled; } /** * This method evaluates the case expression of a decision task and returns a string * representation of the evaluated result. * * @param workflowTask: The decision task that has the case expression to be evaluated. * @param taskInput: the input which has the values that will be used in evaluating the case * expression. * @return A String representation of the evaluated result */ @VisibleForTesting String getEvaluatedCaseValue(WorkflowTask workflowTask, Map taskInput) { String expression = workflowTask.getCaseExpression(); String caseValue; if (StringUtils.isNotBlank(expression)) { LOGGER.debug("Case being evaluated using decision expression: {}", expression); try { // Evaluate the expression by using the Nashhorn based script evaluator Object returnValue = ScriptEvaluator.eval(expression, taskInput); caseValue = (returnValue == null) ? "null" : returnValue.toString(); } catch (ScriptException e) { String errorMsg = String.format("Error while evaluating script: %s", expression); LOGGER.error(errorMsg, e); throw new TerminateWorkflowException(errorMsg); } } else { // In case of no case expression, get the caseValueParam and treat it as a string // representation of caseValue LOGGER.debug( "No Expression available on the decision task, case value being assigned as param name"); String paramName = workflowTask.getCaseValueParam(); caseValue = "" + taskInput.get(paramName); } return caseValue; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/DoWhileTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DO_WHILE} to a {@link TaskModel} of type {@link TaskType#DO_WHILE} */ @Component public class DoWhileTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(DoWhileTaskMapper.class); private final MetadataDAO metadataDAO; private final ParametersUtils parametersUtils; @Autowired public DoWhileTaskMapper(MetadataDAO metadataDAO, ParametersUtils parametersUtils) { this.metadataDAO = metadataDAO; this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.DO_WHILE.name(); } /** * This method maps {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DO_WHILE} to a {@link TaskModel} of type {@link TaskType#DO_WHILE} with a status of * {@link TaskModel.Status#IN_PROGRESS} * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return: A {@link TaskModel} of type {@link TaskType#DO_WHILE} in a List */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in DoWhileTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); TaskModel task = workflowModel.getTaskByRefName(workflowTask.getTaskReferenceName()); if (task != null && task.getStatus().isTerminal()) { // Since loopTask is already completed no need to schedule task again. return List.of(); } TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet( () -> Optional.ofNullable( metadataDAO.getTaskDef( workflowTask.getName())) .orElseGet(TaskDef::new)); TaskModel doWhileTask = taskMapperContext.createTaskModel(); doWhileTask.setTaskType(TaskType.TASK_TYPE_DO_WHILE); doWhileTask.setStatus(TaskModel.Status.IN_PROGRESS); doWhileTask.setStartTime(System.currentTimeMillis()); doWhileTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); doWhileTask.setRateLimitFrequencyInSeconds(taskDefinition.getRateLimitFrequencyInSeconds()); doWhileTask.setRetryCount(taskMapperContext.getRetryCount()); Map taskInput = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, doWhileTask.getTaskId(), taskDefinition); doWhileTask.setInputData(taskInput); return List.of(doWhileTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DYNAMIC} to a {@link TaskModel} based on definition derived from the dynamic task name * defined in {@link WorkflowTask#getInputParameters()} */ @Component public class DynamicTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(DynamicTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; @Autowired public DynamicTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.DYNAMIC.name(); } /** * This method maps a dynamic task to a {@link TaskModel} based on the input params * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return A {@link List} that contains a single {@link TaskModel} with a {@link * TaskModel.Status#SCHEDULED} */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in DynamicTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); Map taskInput = taskMapperContext.getTaskInput(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); int retryCount = taskMapperContext.getRetryCount(); String retriedTaskId = taskMapperContext.getRetryTaskId(); String taskNameParam = workflowTask.getDynamicTaskNameParam(); String taskName = getDynamicTaskName(taskInput, taskNameParam); workflowTask.setName(taskName); TaskDef taskDefinition = getDynamicTaskDefinition(workflowTask); workflowTask.setTaskDefinition(taskDefinition); Map input = parametersUtils.getTaskInput( workflowTask.getInputParameters(), workflowModel, taskDefinition, taskMapperContext.getTaskId()); // IMPORTANT: The WorkflowTask that is inside TaskMapperContext is changed above // createTaskModel() must be called here so the changes are reflected in the created // TaskModel TaskModel dynamicTask = taskMapperContext.createTaskModel(); dynamicTask.setStartDelayInSeconds(workflowTask.getStartDelay()); dynamicTask.setInputData(input); dynamicTask.setStatus(TaskModel.Status.SCHEDULED); dynamicTask.setRetryCount(retryCount); dynamicTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); dynamicTask.setResponseTimeoutSeconds(taskDefinition.getResponseTimeoutSeconds()); dynamicTask.setTaskType(taskName); dynamicTask.setRetriedTaskId(retriedTaskId); dynamicTask.setWorkflowPriority(workflowModel.getPriority()); return Collections.singletonList(dynamicTask); } /** * Helper method that looks into the input params and returns the dynamic task name * * @param taskInput: a map which contains different input parameters and also contains the * mapping between the dynamic task name param and the actual name representing the dynamic * task * @param taskNameParam: the key that is used to look up the dynamic task name. * @return The name of the dynamic task * @throws TerminateWorkflowException : In case is there is no value dynamic task name in the * input parameters. */ @VisibleForTesting String getDynamicTaskName(Map taskInput, String taskNameParam) throws TerminateWorkflowException { return Optional.ofNullable(taskInput.get(taskNameParam)) .map(String::valueOf) .orElseThrow( () -> { String reason = String.format( "Cannot map a dynamic task based on the parameter and input. " + "Parameter= %s, input= %s", taskNameParam, taskInput); return new TerminateWorkflowException(reason); }); } /** * This method gets the TaskDefinition for a specific {@link WorkflowTask} * * @param workflowTask: An instance of {@link WorkflowTask} which has the name of the using * which the {@link TaskDef} can be retrieved. * @return An instance of TaskDefinition * @throws TerminateWorkflowException : in case of no workflow definition available */ @VisibleForTesting TaskDef getDynamicTaskDefinition(WorkflowTask workflowTask) throws TerminateWorkflowException { // TODO this is a common pattern in code base can // be moved to DAO return Optional.ofNullable(workflowTask.getTaskDefinition()) .orElseGet( () -> Optional.ofNullable(metadataDAO.getTaskDef(workflowTask.getName())) .orElseThrow( () -> { String reason = String.format( "Invalid task specified. Cannot find task by name %s in the task definitions", workflowTask.getName()); return new TerminateWorkflowException(reason); })); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/EventTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_EVENT; @Component public class EventTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(EventTaskMapper.class); private final ParametersUtils parametersUtils; @Autowired public EventTaskMapper(ParametersUtils parametersUtils) { this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.EVENT.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in EventTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); workflowTask.getInputParameters().put("sink", workflowTask.getSink()); workflowTask.getInputParameters().put("asyncComplete", workflowTask.isAsyncComplete()); Map eventTaskInput = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, taskId, null); String sink = (String) eventTaskInput.get("sink"); Boolean asynComplete = (Boolean) eventTaskInput.get("asyncComplete"); TaskModel eventTask = taskMapperContext.createTaskModel(); eventTask.setTaskType(TASK_TYPE_EVENT); eventTask.setStatus(TaskModel.Status.SCHEDULED); eventTask.setInputData(eventTaskInput); eventTask.getInputData().put("sink", sink); eventTask.getInputData().put("asyncComplete", asynComplete); return List.of(eventTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/ExclusiveJoinTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.model.TaskModel; @Component public class ExclusiveJoinTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(ExclusiveJoinTaskMapper.class); @Override public String getTaskType() { return TaskType.EXCLUSIVE_JOIN.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in ExclusiveJoinTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); Map joinInput = new HashMap<>(); joinInput.put("joinOn", workflowTask.getJoinOn()); if (workflowTask.getDefaultExclusiveJoinTask() != null) { joinInput.put("defaultExclusiveJoinTask", workflowTask.getDefaultExclusiveJoinTask()); } TaskModel joinTask = taskMapperContext.createTaskModel(); joinTask.setTaskType(TaskType.TASK_TYPE_EXCLUSIVE_JOIN); joinTask.setTaskDefName(TaskType.TASK_TYPE_EXCLUSIVE_JOIN); joinTask.setStartTime(System.currentTimeMillis()); joinTask.setInputData(joinInput); joinTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(joinTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.DynamicForkJoinTaskList; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#FORK_JOIN_DYNAMIC} to a LinkedList of {@link TaskModel} beginning with a {@link * TaskType#TASK_TYPE_FORK}, followed by the user defined dynamic tasks and a {@link TaskType#JOIN} * at the end */ @Component public class ForkJoinDynamicTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(ForkJoinDynamicTaskMapper.class); private final IDGenerator idGenerator; private final ParametersUtils parametersUtils; private final ObjectMapper objectMapper; private final MetadataDAO metadataDAO; private static final TypeReference> ListOfWorkflowTasks = new TypeReference<>() {}; @Autowired public ForkJoinDynamicTaskMapper( IDGenerator idGenerator, ParametersUtils parametersUtils, ObjectMapper objectMapper, MetadataDAO metadataDAO) { this.idGenerator = idGenerator; this.parametersUtils = parametersUtils; this.objectMapper = objectMapper; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.FORK_JOIN_DYNAMIC.name(); } /** * This method gets the list of tasks that need to scheduled when the task to scheduled is of * type {@link TaskType#FORK_JOIN_DYNAMIC}. Creates a Fork Task, followed by the Dynamic tasks * and a final JOIN task. * *

    The definitions of the dynamic forks that need to be scheduled are available in the {@link * WorkflowTask#getInputParameters()} which are accessed using the {@link * TaskMapperContext#getWorkflowTask()}. The dynamic fork task definitions are referred by a key * value either by {@link WorkflowTask#getDynamicForkTasksParam()} or by {@link * WorkflowTask#getDynamicForkJoinTasksParam()} When creating the list of tasks to be scheduled * a set of preconditions are validated: * *

      *
    • If the input parameter representing the Dynamic fork tasks is available as part of * {@link WorkflowTask#getDynamicForkTasksParam()} then the input for the dynamic task is * validated to be a map by using {@link WorkflowTask#getDynamicForkTasksInputParamName()} *
    • If the input parameter representing the Dynamic fork tasks is available as part of * {@link WorkflowTask#getDynamicForkJoinTasksParam()} then the input for the dynamic * tasks is available in the payload of the tasks definition. *
    • A check is performed that the next following task in the {@link WorkflowDef} is a * {@link TaskType#JOIN} *
    * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return List of tasks in the following order: *
      *
    • {@link TaskType#TASK_TYPE_FORK} with {@link TaskModel.Status#COMPLETED} *
    • Might be any kind of task, but this is most cases is a UserDefinedTask with {@link * TaskModel.Status#SCHEDULED} *
    • {@link TaskType#JOIN} with {@link TaskModel.Status#IN_PROGRESS} *
    * * @throws TerminateWorkflowException In case of: *
      *
    • When the task after {@link TaskType#FORK_JOIN_DYNAMIC} is not a {@link * TaskType#JOIN} *
    • When the input parameters for the dynamic tasks are not of type {@link Map} *
    */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in ForkJoinDynamicTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); int retryCount = taskMapperContext.getRetryCount(); List mappedTasks = new LinkedList<>(); // Get the list of dynamic tasks and the input for the tasks Pair, Map>> workflowTasksAndInputPair = Optional.ofNullable(workflowTask.getDynamicForkTasksParam()) .map( dynamicForkTaskParam -> getDynamicForkTasksAndInput( workflowTask, workflowModel, dynamicForkTaskParam)) .orElseGet( () -> getDynamicForkJoinTasksAndInput(workflowTask, workflowModel)); List dynForkTasks = workflowTasksAndInputPair.getLeft(); Map> tasksInput = workflowTasksAndInputPair.getRight(); // Create Fork Task which needs to be followed by the dynamic tasks TaskModel forkDynamicTask = createDynamicForkTask(taskMapperContext, dynForkTasks); mappedTasks.add(forkDynamicTask); List joinOnTaskRefs = new LinkedList<>(); // Add each dynamic task to the mapped tasks and also get the last dynamic task in the list, // which indicates that the following task after that needs to be a join task for (WorkflowTask dynForkTask : dynForkTasks) { // TODO this is a cyclic dependency, break it out using function // composition List forkedTasks = taskMapperContext .getDeciderService() .getTasksToBeScheduled(workflowModel, dynForkTask, retryCount); // It's an error state if no forkedTasks can be decided upon. In the cases where we've // seen // this happen is when a dynamic task is attempting to be created here, but a task with // the // same reference name has already been created in the Workflow. if (forkedTasks == null || forkedTasks.isEmpty()) { Optional existingTaskRefName = workflowModel.getTasks().stream() .filter( runningTask -> runningTask .getStatus() .equals( TaskModel.Status .IN_PROGRESS) || runningTask.getStatus().isTerminal()) .map(TaskModel::getReferenceTaskName) .filter( refTaskName -> refTaskName.equals( dynForkTask.getTaskReferenceName())) .findAny(); // Construct an informative error message String terminateMessage = "No dynamic tasks could be created for the Workflow: " + workflowModel.toShortString() + ", Dynamic Fork Task: " + dynForkTask; if (existingTaskRefName.isPresent()) { terminateMessage += "Attempted to create a duplicate task reference name: " + existingTaskRefName.get(); } throw new TerminateWorkflowException(terminateMessage); } for (TaskModel forkedTask : forkedTasks) { try { Map forkedTaskInput = tasksInput.get(forkedTask.getReferenceTaskName()); forkedTask.addInput(forkedTaskInput); } catch (Exception e) { String reason = String.format( "Tasks could not be dynamically forked due to invalid input: %s", e.getMessage()); throw new TerminateWorkflowException(reason); } } mappedTasks.addAll(forkedTasks); // Get the last of the dynamic tasks so that the join can be performed once this task is // done TaskModel last = forkedTasks.get(forkedTasks.size() - 1); joinOnTaskRefs.add(last.getReferenceTaskName()); } // From the workflow definition get the next task and make sure that it is a JOIN task. // The dynamic fork tasks need to be followed by a join task WorkflowTask joinWorkflowTask = workflowModel .getWorkflowDefinition() .getNextTask(workflowTask.getTaskReferenceName()); if (joinWorkflowTask == null || !joinWorkflowTask.getType().equals(TaskType.JOIN.name())) { throw new TerminateWorkflowException( "Dynamic join definition is not followed by a join task. Check the workflow definition."); } // Create Join task HashMap joinInput = new HashMap<>(); joinInput.put("joinOn", joinOnTaskRefs); TaskModel joinTask = createJoinTask(workflowModel, joinWorkflowTask, joinInput); mappedTasks.add(joinTask); return mappedTasks; } /** * This method creates a FORK task and adds the list of dynamic fork tasks keyed by * "forkedTaskDefs" and their names keyed by "forkedTasks" into {@link TaskModel#getInputData()} * * @param taskMapperContext: The {@link TaskMapperContext} which wraps workflowTask, workflowDef * and workflowModel * @param dynForkTasks: The list of dynamic forked tasks, the reference names of these tasks * will be added to the forkDynamicTask * @return A new instance of {@link TaskModel} representing a {@link TaskType#TASK_TYPE_FORK} */ @VisibleForTesting TaskModel createDynamicForkTask( TaskMapperContext taskMapperContext, List dynForkTasks) { TaskModel forkDynamicTask = taskMapperContext.createTaskModel(); forkDynamicTask.setTaskType(TaskType.TASK_TYPE_FORK); forkDynamicTask.setTaskDefName(TaskType.TASK_TYPE_FORK); forkDynamicTask.setStartTime(System.currentTimeMillis()); forkDynamicTask.setEndTime(System.currentTimeMillis()); List forkedTaskNames = dynForkTasks.stream() .map(WorkflowTask::getTaskReferenceName) .collect(Collectors.toList()); forkDynamicTask.getInputData().put("forkedTasks", forkedTaskNames); forkDynamicTask .getInputData() .put( "forkedTaskDefs", dynForkTasks); // TODO: Remove this parameter in the later releases forkDynamicTask.setStatus(TaskModel.Status.COMPLETED); return forkDynamicTask; } /** * This method creates a JOIN task that is used in the {@link * this#getMappedTasks(TaskMapperContext)} at the end to add a join task to be scheduled after * all the fork tasks * * @param workflowModel: A instance of the {@link WorkflowModel} which represents the workflow * being executed. * @param joinWorkflowTask: A instance of {@link WorkflowTask} which is of type {@link * TaskType#JOIN} * @param joinInput: The input which is set in the {@link TaskModel#setInputData(Map)} * @return a new instance of {@link TaskModel} representing a {@link TaskType#JOIN} */ @VisibleForTesting TaskModel createJoinTask( WorkflowModel workflowModel, WorkflowTask joinWorkflowTask, HashMap joinInput) { TaskModel joinTask = new TaskModel(); joinTask.setTaskType(TaskType.TASK_TYPE_JOIN); joinTask.setTaskDefName(TaskType.TASK_TYPE_JOIN); joinTask.setReferenceTaskName(joinWorkflowTask.getTaskReferenceName()); joinTask.setWorkflowInstanceId(workflowModel.getWorkflowId()); joinTask.setWorkflowType(workflowModel.getWorkflowName()); joinTask.setCorrelationId(workflowModel.getCorrelationId()); joinTask.setScheduledTime(System.currentTimeMillis()); joinTask.setStartTime(System.currentTimeMillis()); joinTask.setInputData(joinInput); joinTask.setTaskId(idGenerator.generate()); joinTask.setStatus(TaskModel.Status.IN_PROGRESS); joinTask.setWorkflowTask(joinWorkflowTask); joinTask.setWorkflowPriority(workflowModel.getPriority()); return joinTask; } /** * This method is used to get the List of dynamic workflow tasks and their input based on the * {@link WorkflowTask#getDynamicForkTasksParam()} * * @param workflowTask: The Task of type FORK_JOIN_DYNAMIC that needs to scheduled, which has * the input parameters * @param workflowModel: The instance of the {@link WorkflowModel} which represents the workflow * being executed. * @param dynamicForkTaskParam: The key representing the dynamic fork join json payload which is * available in {@link WorkflowTask#getInputParameters()} * @return a {@link Pair} representing the list of dynamic fork tasks in {@link Pair#getLeft()} * and the input for the dynamic fork tasks in {@link Pair#getRight()} * @throws TerminateWorkflowException : In case of input parameters of the dynamic fork tasks * not represented as {@link Map} */ @SuppressWarnings("unchecked") @VisibleForTesting Pair, Map>> getDynamicForkTasksAndInput( WorkflowTask workflowTask, WorkflowModel workflowModel, String dynamicForkTaskParam) throws TerminateWorkflowException { Map input = parametersUtils.getTaskInput( workflowTask.getInputParameters(), workflowModel, null, null); Object dynamicForkTasksJson = input.get(dynamicForkTaskParam); List dynamicForkWorkflowTasks = objectMapper.convertValue(dynamicForkTasksJson, ListOfWorkflowTasks); if (dynamicForkWorkflowTasks == null) { dynamicForkWorkflowTasks = new ArrayList<>(); } for (WorkflowTask dynamicForkWorkflowTask : dynamicForkWorkflowTasks) { if ((dynamicForkWorkflowTask.getTaskDefinition() == null) && StringUtils.isNotBlank(dynamicForkWorkflowTask.getName())) { dynamicForkWorkflowTask.setTaskDefinition( metadataDAO.getTaskDef(dynamicForkWorkflowTask.getName())); } } Object dynamicForkTasksInput = input.get(workflowTask.getDynamicForkTasksInputParamName()); if (!(dynamicForkTasksInput instanceof Map)) { throw new TerminateWorkflowException( "Input to the dynamically forked tasks is not a map -> expecting a map of K,V but found " + dynamicForkTasksInput); } return new ImmutablePair<>( dynamicForkWorkflowTasks, (Map>) dynamicForkTasksInput); } /** * This method is used to get the List of dynamic workflow tasks and their input based on the * {@link WorkflowTask#getDynamicForkJoinTasksParam()} * *

    NOTE: This method is kept for legacy reasons, new workflows should use the {@link * #getDynamicForkTasksAndInput} * * @param workflowTask: The Task of type FORK_JOIN_DYNAMIC that needs to scheduled, which has * the input parameters * @param workflowModel: The instance of the {@link WorkflowModel} which represents the workflow * being executed. * @return {@link Pair} representing the list of dynamic fork tasks in {@link Pair#getLeft()} * and the input for the dynamic fork tasks in {@link Pair#getRight()} * @throws TerminateWorkflowException : In case of the {@link WorkflowTask#getInputParameters()} * does not have a payload that contains the list of the dynamic tasks */ @VisibleForTesting Pair, Map>> getDynamicForkJoinTasksAndInput( WorkflowTask workflowTask, WorkflowModel workflowModel) throws TerminateWorkflowException { String dynamicForkJoinTaskParam = workflowTask.getDynamicForkJoinTasksParam(); Map input = parametersUtils.getTaskInput( workflowTask.getInputParameters(), workflowModel, null, null); Object paramValue = input.get(dynamicForkJoinTaskParam); DynamicForkJoinTaskList dynamicForkJoinTaskList = objectMapper.convertValue(paramValue, DynamicForkJoinTaskList.class); if (dynamicForkJoinTaskList == null) { String reason = String.format( "Dynamic tasks could not be created. The value of %s from task's input %s has no dynamic tasks to be scheduled", dynamicForkJoinTaskParam, input); LOGGER.error(reason); throw new TerminateWorkflowException(reason); } Map> dynamicForkJoinTasksInput = new HashMap<>(); List dynamicForkJoinWorkflowTasks = dynamicForkJoinTaskList.getDynamicTasks().stream() .peek( dynamicForkJoinTask -> dynamicForkJoinTasksInput.put( dynamicForkJoinTask.getReferenceName(), dynamicForkJoinTask .getInput())) // TODO create a custom pair // collector .map( dynamicForkJoinTask -> { WorkflowTask dynamicForkJoinWorkflowTask = new WorkflowTask(); dynamicForkJoinWorkflowTask.setTaskReferenceName( dynamicForkJoinTask.getReferenceName()); dynamicForkJoinWorkflowTask.setName( dynamicForkJoinTask.getTaskName()); dynamicForkJoinWorkflowTask.setType( dynamicForkJoinTask.getType()); if (dynamicForkJoinWorkflowTask.getTaskDefinition() == null && StringUtils.isNotBlank( dynamicForkJoinWorkflowTask.getName())) { dynamicForkJoinWorkflowTask.setTaskDefinition( metadataDAO.getTaskDef( dynamicForkJoinTask.getTaskName())); } return dynamicForkJoinWorkflowTask; }) .collect(Collectors.toCollection(LinkedList::new)); return new ImmutablePair<>(dynamicForkJoinWorkflowTasks, dynamicForkJoinTasksInput); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#FORK_JOIN} to a LinkedList of {@link TaskModel} beginning with a completed {@link * TaskType#TASK_TYPE_FORK}, followed by the user defined fork tasks */ @Component public class ForkJoinTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(ForkJoinTaskMapper.class); @Override public String getTaskType() { return TaskType.FORK_JOIN.name(); } /** * This method gets the list of tasks that need to scheduled when the task to scheduled is of * type {@link TaskType#FORK_JOIN}. * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return List of tasks in the following order: * *

      *
    • {@link TaskType#TASK_TYPE_FORK} with {@link TaskModel.Status#COMPLETED} *
    • Might be any kind of task, but in most cases is a UserDefinedTask with {@link * TaskModel.Status#SCHEDULED} *
    * * @throws TerminateWorkflowException When the task after {@link TaskType#FORK_JOIN} is not a * {@link TaskType#JOIN} */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in ForkJoinTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); Map taskInput = taskMapperContext.getTaskInput(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); int retryCount = taskMapperContext.getRetryCount(); List tasksToBeScheduled = new LinkedList<>(); TaskModel forkTask = taskMapperContext.createTaskModel(); forkTask.setTaskType(TaskType.TASK_TYPE_FORK); forkTask.setTaskDefName(TaskType.TASK_TYPE_FORK); long epochMillis = System.currentTimeMillis(); forkTask.setStartTime(epochMillis); forkTask.setEndTime(epochMillis); forkTask.setInputData(taskInput); forkTask.setStatus(TaskModel.Status.COMPLETED); tasksToBeScheduled.add(forkTask); List> forkTasks = workflowTask.getForkTasks(); for (List wfts : forkTasks) { WorkflowTask wft = wfts.get(0); List tasks2 = taskMapperContext .getDeciderService() .getTasksToBeScheduled(workflowModel, wft, retryCount); tasksToBeScheduled.addAll(tasks2); } WorkflowTask joinWorkflowTask = workflowModel .getWorkflowDefinition() .getNextTask(workflowTask.getTaskReferenceName()); if (joinWorkflowTask == null || !joinWorkflowTask.getType().equals(TaskType.JOIN.name())) { throw new TerminateWorkflowException( "Fork task definition is not followed by a join task. Check the blueprint"); } List joinTask = taskMapperContext .getDeciderService() .getTasksToBeScheduled(workflowModel, joinWorkflowTask, retryCount); tasksToBeScheduled.addAll(joinTask); return tasksToBeScheduled; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/HTTPTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#HTTP} to a {@link TaskModel} of type {@link TaskType#HTTP} with {@link * TaskModel.Status#SCHEDULED} */ @Component public class HTTPTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(HTTPTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; @Autowired public HTTPTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.HTTP.name(); } /** * This method maps a {@link WorkflowTask} of type {@link TaskType#HTTP} to a {@link TaskModel} * in a {@link TaskModel.Status#SCHEDULED} state * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return a List with just one HTTP task * @throws TerminateWorkflowException In case if the task definition does not exist */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in HTTPTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); workflowTask.getInputParameters().put("asyncComplete", workflowTask.isAsyncComplete()); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); int retryCount = taskMapperContext.getRetryCount(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); Map input = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, taskId, taskDefinition); Boolean asynComplete = (Boolean) input.get("asyncComplete"); TaskModel httpTask = taskMapperContext.createTaskModel(); httpTask.setInputData(input); httpTask.getInputData().put("asyncComplete", asynComplete); httpTask.setStatus(TaskModel.Status.SCHEDULED); httpTask.setRetryCount(retryCount); httpTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); if (Objects.nonNull(taskDefinition)) { httpTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); httpTask.setRateLimitFrequencyInSeconds( taskDefinition.getRateLimitFrequencyInSeconds()); httpTask.setIsolationGroupId(taskDefinition.getIsolationGroupId()); httpTask.setExecutionNameSpace(taskDefinition.getExecutionNameSpace()); } return List.of(httpTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/HumanTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.tasks.Human; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HUMAN; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#HUMAN} to a {@link TaskModel} of type {@link Human} with {@link * TaskModel.Status#IN_PROGRESS} */ @Component public class HumanTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(HumanTaskMapper.class); private final ParametersUtils parametersUtils; public HumanTaskMapper(ParametersUtils parametersUtils) { this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.HUMAN.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); Map humanTaskInput = parametersUtils.getTaskInputV2( taskMapperContext.getWorkflowTask().getInputParameters(), workflowModel, taskId, null); TaskModel humanTask = taskMapperContext.createTaskModel(); humanTask.setTaskType(TASK_TYPE_HUMAN); humanTask.setInputData(humanTaskInput); humanTask.setStartTime(System.currentTimeMillis()); humanTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(humanTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/InlineTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#INLINE} to a List {@link TaskModel} starting with Task of type {@link TaskType#INLINE} * which is marked as IN_PROGRESS, followed by the list of {@link TaskModel} based on the case * expression evaluation in the Inline task. */ @Component public class InlineTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(InlineTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; public InlineTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.INLINE.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in InlineTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); Map taskInput = parametersUtils.getTaskInputV2( taskMapperContext.getWorkflowTask().getInputParameters(), workflowModel, taskId, taskDefinition); TaskModel inlineTask = taskMapperContext.createTaskModel(); inlineTask.setTaskType(TaskType.TASK_TYPE_INLINE); inlineTask.setStartTime(System.currentTimeMillis()); inlineTask.setInputData(taskInput); inlineTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(inlineTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/JoinTaskMapper.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#JOIN} to a {@link TaskModel} of type {@link TaskType#JOIN} */ @Component public class JoinTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(JoinTaskMapper.class); @Override public String getTaskType() { return TaskType.JOIN.name(); } /** * This method maps {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#JOIN} to a {@link TaskModel} of type {@link TaskType#JOIN} with a status of {@link * TaskModel.Status#IN_PROGRESS} * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return A {@link TaskModel} of type {@link TaskType#JOIN} in a List */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in JoinTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); Map joinInput = new HashMap<>(); joinInput.put("joinOn", workflowTask.getJoinOn()); TaskModel joinTask = taskMapperContext.createTaskModel(); joinTask.setTaskType(TaskType.TASK_TYPE_JOIN); joinTask.setTaskDefName(TaskType.TASK_TYPE_JOIN); joinTask.setStartTime(System.currentTimeMillis()); joinTask.setInputData(joinInput); joinTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(joinTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/JsonJQTransformTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; @Component public class JsonJQTransformTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(JsonJQTransformTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; public JsonJQTransformTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.JSON_JQ_TRANSFORM.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in JsonJQTransformTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); Map taskInput = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, taskId, taskDefinition); TaskModel jsonJQTransformTask = taskMapperContext.createTaskModel(); jsonJQTransformTask.setStartTime(System.currentTimeMillis()); jsonJQTransformTask.setInputData(taskInput); jsonJQTransformTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(jsonJQTransformTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/KafkaPublishTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; @Component public class KafkaPublishTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(KafkaPublishTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; @Autowired public KafkaPublishTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.KAFKA_PUBLISH.name(); } /** * This method maps a {@link WorkflowTask} of type {@link TaskType#KAFKA_PUBLISH} to a {@link * TaskModel} in a {@link TaskModel.Status#SCHEDULED} state * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return a List with just one Kafka task * @throws TerminateWorkflowException In case if the task definition does not exist */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in KafkaPublishTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); int retryCount = taskMapperContext.getRetryCount(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); Map input = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, taskId, taskDefinition); TaskModel kafkaPublishTask = taskMapperContext.createTaskModel(); kafkaPublishTask.setInputData(input); kafkaPublishTask.setStatus(TaskModel.Status.SCHEDULED); kafkaPublishTask.setRetryCount(retryCount); kafkaPublishTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); if (Objects.nonNull(taskDefinition)) { kafkaPublishTask.setExecutionNameSpace(taskDefinition.getExecutionNameSpace()); kafkaPublishTask.setIsolationGroupId(taskDefinition.getIsolationGroupId()); kafkaPublishTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); kafkaPublishTask.setRateLimitFrequencyInSeconds( taskDefinition.getRateLimitFrequencyInSeconds()); } return Collections.singletonList(kafkaPublishTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/LambdaTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * @author x-ultra * @deprecated {@link com.netflix.conductor.core.execution.tasks.Lambda} is also deprecated. Use * {@link com.netflix.conductor.core.execution.tasks.Inline} and so ${@link InlineTaskMapper} * will be used as a result. */ @Deprecated @Component public class LambdaTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(LambdaTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; public LambdaTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.LAMBDA.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in LambdaTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); Map taskInput = parametersUtils.getTaskInputV2( taskMapperContext.getWorkflowTask().getInputParameters(), workflowModel, taskId, taskDefinition); TaskModel lambdaTask = taskMapperContext.createTaskModel(); lambdaTask.setTaskType(TaskType.TASK_TYPE_LAMBDA); lambdaTask.setStartTime(System.currentTimeMillis()); lambdaTask.setInputData(taskInput); lambdaTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(lambdaTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/NoopTaskMapper.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.model.TaskModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.*; @Component public class NoopTaskMapper implements TaskMapper { public static final Logger logger = LoggerFactory.getLogger(NoopTaskMapper.class); @Override public String getTaskType() { return TaskType.NOOP.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { logger.debug("TaskMapperContext {} in NoopTaskMapper", taskMapperContext); TaskModel task = taskMapperContext.createTaskModel(); task.setTaskType(TASK_TYPE_NOOP); task.setStartTime(System.currentTimeMillis()); task.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(task); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/SetVariableTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; @Component public class SetVariableTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(SetVariableTaskMapper.class); @Override public String getTaskType() { return TaskType.SET_VARIABLE.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in SetVariableMapper", taskMapperContext); TaskModel varTask = taskMapperContext.createTaskModel(); varTask.setStartTime(System.currentTimeMillis()); varTask.setInputData(taskMapperContext.getTaskInput()); varTask.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(varTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/SimpleTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#SIMPLE} to a {@link TaskModel} with status {@link TaskModel.Status#SCHEDULED}. * NOTE: There is not type defined for simples task. */ @Component public class SimpleTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(SimpleTaskMapper.class); private final ParametersUtils parametersUtils; public SimpleTaskMapper(ParametersUtils parametersUtils) { this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.SIMPLE.name(); } /** * This method maps a {@link WorkflowTask} of type {@link TaskType#SIMPLE} to a {@link * TaskModel} * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @throws TerminateWorkflowException In case if the task definition does not exist * @return a List with just one simple task */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in SimpleTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); int retryCount = taskMapperContext.getRetryCount(); String retriedTaskId = taskMapperContext.getRetryTaskId(); TaskDef taskDefinition = Optional.ofNullable(workflowTask.getTaskDefinition()) .orElseThrow( () -> { String reason = String.format( "Invalid task. Task %s does not have a definition", workflowTask.getName()); return new TerminateWorkflowException(reason); }); Map input = parametersUtils.getTaskInput( workflowTask.getInputParameters(), workflowModel, taskDefinition, taskMapperContext.getTaskId()); TaskModel simpleTask = taskMapperContext.createTaskModel(); simpleTask.setTaskType(workflowTask.getName()); simpleTask.setStartDelayInSeconds(workflowTask.getStartDelay()); simpleTask.setInputData(input); simpleTask.setStatus(TaskModel.Status.SCHEDULED); simpleTask.setRetryCount(retryCount); simpleTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); simpleTask.setResponseTimeoutSeconds(taskDefinition.getResponseTimeoutSeconds()); simpleTask.setRetriedTaskId(retriedTaskId); simpleTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); simpleTask.setRateLimitFrequencyInSeconds(taskDefinition.getRateLimitFrequencyInSeconds()); return List.of(simpleTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/StartWorkflowTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.START_WORKFLOW; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_START_WORKFLOW; @Component public class StartWorkflowTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(StartWorkflowTaskMapper.class); @Override public String getTaskType() { return START_WORKFLOW.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); TaskModel startWorkflowTask = taskMapperContext.createTaskModel(); startWorkflowTask.setTaskType(TASK_TYPE_START_WORKFLOW); startWorkflowTask.addInput(taskMapperContext.getTaskInput()); startWorkflowTask.setStatus(TaskModel.Status.SCHEDULED); startWorkflowTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); LOGGER.debug("{} created", startWorkflowTask); return List.of(startWorkflowTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; @Component public class SubWorkflowTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(SubWorkflowTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; public SubWorkflowTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.SUB_WORKFLOW.name(); } @SuppressWarnings("rawtypes") @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in SubWorkflowTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); // Check if there are sub workflow parameters, if not throw an exception, cannot initiate a // sub-workflow without workflow params SubWorkflowParams subWorkflowParams = getSubWorkflowParams(workflowTask); Map resolvedParams = getSubWorkflowInputParameters(workflowModel, subWorkflowParams); String subWorkflowName = resolvedParams.get("name").toString(); Integer subWorkflowVersion = getSubWorkflowVersion(resolvedParams, subWorkflowName); Object subWorkflowDefinition = resolvedParams.get("workflowDefinition"); Map subWorkflowTaskToDomain = null; Object uncheckedTaskToDomain = resolvedParams.get("taskToDomain"); if (uncheckedTaskToDomain instanceof Map) { subWorkflowTaskToDomain = (Map) uncheckedTaskToDomain; } TaskModel subWorkflowTask = taskMapperContext.createTaskModel(); subWorkflowTask.setTaskType(TASK_TYPE_SUB_WORKFLOW); subWorkflowTask.addInput("subWorkflowName", subWorkflowName); subWorkflowTask.addInput("subWorkflowVersion", subWorkflowVersion); subWorkflowTask.addInput("subWorkflowTaskToDomain", subWorkflowTaskToDomain); subWorkflowTask.addInput("subWorkflowDefinition", subWorkflowDefinition); subWorkflowTask.addInput("workflowInput", taskMapperContext.getTaskInput()); subWorkflowTask.setStatus(TaskModel.Status.SCHEDULED); subWorkflowTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); LOGGER.debug("SubWorkflowTask {} created to be Scheduled", subWorkflowTask); return List.of(subWorkflowTask); } @VisibleForTesting SubWorkflowParams getSubWorkflowParams(WorkflowTask workflowTask) { return Optional.ofNullable(workflowTask.getSubWorkflowParam()) .orElseThrow( () -> { String reason = String.format( "Task %s is defined as sub-workflow and is missing subWorkflowParams. " + "Please check the workflow definition", workflowTask.getName()); LOGGER.error(reason); return new TerminateWorkflowException(reason); }); } private Map getSubWorkflowInputParameters( WorkflowModel workflowModel, SubWorkflowParams subWorkflowParams) { Map params = new HashMap<>(); params.put("name", subWorkflowParams.getName()); Integer version = subWorkflowParams.getVersion(); if (version != null) { params.put("version", version); } Map taskToDomain = subWorkflowParams.getTaskToDomain(); if (taskToDomain != null) { params.put("taskToDomain", taskToDomain); } params = parametersUtils.getTaskInputV2(params, workflowModel, null, null); // do not resolve params inside subworkflow definition Object subWorkflowDefinition = subWorkflowParams.getWorkflowDefinition(); if (subWorkflowDefinition != null) { params.put("workflowDefinition", subWorkflowDefinition); } return params; } private Integer getSubWorkflowVersion( Map resolvedParams, String subWorkflowName) { return Optional.ofNullable(resolvedParams.get("version")) .map(Object::toString) .map(Integer::parseInt) .orElseGet( () -> metadataDAO .getLatestWorkflowDef(subWorkflowName) .map(WorkflowDef::getVersion) .orElseThrow( () -> { String reason = String.format( "The Task %s defined as a sub-workflow has no workflow definition available ", subWorkflowName); LOGGER.error(reason); return new TerminateWorkflowException(reason); })); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/SwitchTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#SWITCH} to a List {@link TaskModel} starting with Task of type {@link TaskType#SWITCH} * which is marked as IN_PROGRESS, followed by the list of {@link TaskModel} based on the case * expression evaluation in the Switch task. */ @Component public class SwitchTaskMapper implements TaskMapper { private static final Logger LOGGER = LoggerFactory.getLogger(SwitchTaskMapper.class); private final Map evaluators; @Autowired public SwitchTaskMapper(Map evaluators) { this.evaluators = evaluators; } @Override public String getTaskType() { return TaskType.SWITCH.name(); } /** * This method gets the list of tasks that need to scheduled when the task to scheduled is of * type {@link TaskType#SWITCH}. * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return List of tasks in the following order: *

      *
    • {@link TaskType#SWITCH} with {@link TaskModel.Status#IN_PROGRESS} *
    • List of tasks based on the evaluation of {@link WorkflowTask#getEvaluatorType()} * and {@link WorkflowTask#getExpression()} are scheduled. *
    • In the case of no matching {@link WorkflowTask#getEvaluatorType()}, workflow will * be terminated with error message. In case of no matching result after the * evaluation of the {@link WorkflowTask#getExpression()}, the {@link * WorkflowTask#getDefaultCase()} Tasks are scheduled. *
    */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in SwitchTaskMapper", taskMapperContext); List tasksToBeScheduled = new LinkedList<>(); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); Map taskInput = taskMapperContext.getTaskInput(); int retryCount = taskMapperContext.getRetryCount(); // get the expression to be evaluated String evaluatorType = workflowTask.getEvaluatorType(); Evaluator evaluator = evaluators.get(evaluatorType); if (evaluator == null) { String errorMsg = String.format("No evaluator registered for type: %s", evaluatorType); LOGGER.error(errorMsg); throw new TerminateWorkflowException(errorMsg); } String evalResult = ""; try { evalResult = "" + evaluator.evaluate(workflowTask.getExpression(), taskInput); } catch (Exception exception) { TaskModel switchTask = taskMapperContext.createTaskModel(); switchTask.setTaskType(TaskType.TASK_TYPE_SWITCH); switchTask.setTaskDefName(TaskType.TASK_TYPE_SWITCH); switchTask.getInputData().putAll(taskInput); switchTask.setStartTime(System.currentTimeMillis()); switchTask.setStatus(TaskModel.Status.FAILED); switchTask.setReasonForIncompletion(exception.getMessage()); tasksToBeScheduled.add(switchTask); return tasksToBeScheduled; } // QQ why is the case value and the caseValue passed and caseOutput passes as the same ?? TaskModel switchTask = taskMapperContext.createTaskModel(); switchTask.setTaskType(TaskType.TASK_TYPE_SWITCH); switchTask.setTaskDefName(TaskType.TASK_TYPE_SWITCH); switchTask.getInputData().putAll(taskInput); switchTask.getInputData().put("case", evalResult); switchTask.addOutput("evaluationResult", List.of(evalResult)); switchTask.addOutput("selectedCase", evalResult); switchTask.setStartTime(System.currentTimeMillis()); switchTask.setStatus(TaskModel.Status.IN_PROGRESS); tasksToBeScheduled.add(switchTask); // get the list of tasks based on the evaluated expression List selectedTasks = workflowTask.getDecisionCases().get(evalResult); // if the tasks returned are empty based on evaluated result, then get the default case if // there is one if (selectedTasks == null || selectedTasks.isEmpty()) { selectedTasks = workflowTask.getDefaultCase(); } // once there are selected tasks that need to proceeded as part of the switch, get the next // task to be scheduled by using the decider service if (selectedTasks != null && !selectedTasks.isEmpty()) { WorkflowTask selectedTask = selectedTasks.get(0); // Schedule the first task to be executed... // TODO break out this recursive call using function composition of what needs to be // done and then walk back the condition tree List caseTasks = taskMapperContext .getDeciderService() .getTasksToBeScheduled( workflowModel, selectedTask, retryCount, taskMapperContext.getRetryTaskId()); tasksToBeScheduled.addAll(caseTasks); switchTask.getInputData().put("hasChildren", "true"); } return tasksToBeScheduled; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/TaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; public interface TaskMapper { String getTaskType(); List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException; } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/TaskMapperContext.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Map; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** Business Object class used for interaction between the DeciderService and Different Mappers */ public class TaskMapperContext { private final WorkflowModel workflowModel; private final TaskDef taskDefinition; private final WorkflowTask workflowTask; private final Map taskInput; private final int retryCount; private final String retryTaskId; private final String taskId; private final DeciderService deciderService; private TaskMapperContext(Builder builder) { workflowModel = builder.workflowModel; taskDefinition = builder.taskDefinition; workflowTask = builder.workflowTask; taskInput = builder.taskInput; retryCount = builder.retryCount; retryTaskId = builder.retryTaskId; taskId = builder.taskId; deciderService = builder.deciderService; } public static Builder newBuilder() { return new Builder(); } public static Builder newBuilder(TaskMapperContext copy) { Builder builder = new Builder(); builder.workflowModel = copy.getWorkflowModel(); builder.taskDefinition = copy.getTaskDefinition(); builder.workflowTask = copy.getWorkflowTask(); builder.taskInput = copy.getTaskInput(); builder.retryCount = copy.getRetryCount(); builder.retryTaskId = copy.getRetryTaskId(); builder.taskId = copy.getTaskId(); builder.deciderService = copy.getDeciderService(); return builder; } public WorkflowDef getWorkflowDefinition() { return workflowModel.getWorkflowDefinition(); } public WorkflowModel getWorkflowModel() { return workflowModel; } public TaskDef getTaskDefinition() { return taskDefinition; } public WorkflowTask getWorkflowTask() { return workflowTask; } public int getRetryCount() { return retryCount; } public String getRetryTaskId() { return retryTaskId; } public String getTaskId() { return taskId; } public Map getTaskInput() { return taskInput; } public DeciderService getDeciderService() { return deciderService; } public TaskModel createTaskModel() { TaskModel taskModel = new TaskModel(); taskModel.setReferenceTaskName(workflowTask.getTaskReferenceName()); taskModel.setWorkflowInstanceId(workflowModel.getWorkflowId()); taskModel.setWorkflowType(workflowModel.getWorkflowName()); taskModel.setCorrelationId(workflowModel.getCorrelationId()); taskModel.setScheduledTime(System.currentTimeMillis()); taskModel.setTaskId(taskId); taskModel.setWorkflowTask(workflowTask); taskModel.setWorkflowPriority(workflowModel.getPriority()); // the following properties are overridden by some TaskMapper implementations taskModel.setTaskType(workflowTask.getType()); taskModel.setTaskDefName(workflowTask.getName()); return taskModel; } @Override public String toString() { return "TaskMapperContext{" + "workflowDefinition=" + getWorkflowDefinition() + ", workflowModel=" + workflowModel + ", workflowTask=" + workflowTask + ", taskInput=" + taskInput + ", retryCount=" + retryCount + ", retryTaskId='" + retryTaskId + '\'' + ", taskId='" + taskId + '\'' + '}'; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof TaskMapperContext)) { return false; } TaskMapperContext that = (TaskMapperContext) o; if (getRetryCount() != that.getRetryCount()) { return false; } if (!getWorkflowDefinition().equals(that.getWorkflowDefinition())) { return false; } if (!getWorkflowModel().equals(that.getWorkflowModel())) { return false; } if (!getWorkflowTask().equals(that.getWorkflowTask())) { return false; } if (!getTaskInput().equals(that.getTaskInput())) { return false; } if (getRetryTaskId() != null ? !getRetryTaskId().equals(that.getRetryTaskId()) : that.getRetryTaskId() != null) { return false; } return getTaskId().equals(that.getTaskId()); } @Override public int hashCode() { int result = getWorkflowDefinition().hashCode(); result = 31 * result + getWorkflowModel().hashCode(); result = 31 * result + getWorkflowTask().hashCode(); result = 31 * result + getTaskInput().hashCode(); result = 31 * result + getRetryCount(); result = 31 * result + (getRetryTaskId() != null ? getRetryTaskId().hashCode() : 0); result = 31 * result + getTaskId().hashCode(); return result; } /** {@code TaskMapperContext} builder static inner class. */ public static final class Builder { private WorkflowModel workflowModel; private TaskDef taskDefinition; private WorkflowTask workflowTask; private Map taskInput; private int retryCount; private String retryTaskId; private String taskId; private DeciderService deciderService; private Builder() {} /** * Sets the {@code workflowModel} and returns a reference to this Builder so that the * methods can be chained together. * * @param val the {@code workflowModel} to set * @return a reference to this Builder */ public Builder withWorkflowModel(WorkflowModel val) { workflowModel = val; return this; } /** * Sets the {@code taskDefinition} and returns a reference to this Builder so that the * methods can be chained together. * * @param val the {@code taskDefinition} to set * @return a reference to this Builder */ public Builder withTaskDefinition(TaskDef val) { taskDefinition = val; return this; } /** * Sets the {@code workflowTask} and returns a reference to this Builder so that the methods * can be chained together. * * @param val the {@code workflowTask} to set * @return a reference to this Builder */ public Builder withWorkflowTask(WorkflowTask val) { workflowTask = val; return this; } /** * Sets the {@code taskInput} and returns a reference to this Builder so that the methods * can be chained together. * * @param val the {@code taskInput} to set * @return a reference to this Builder */ public Builder withTaskInput(Map val) { taskInput = val; return this; } /** * Sets the {@code retryCount} and returns a reference to this Builder so that the methods * can be chained together. * * @param val the {@code retryCount} to set * @return a reference to this Builder */ public Builder withRetryCount(int val) { retryCount = val; return this; } /** * Sets the {@code retryTaskId} and returns a reference to this Builder so that the methods * can be chained together. * * @param val the {@code retryTaskId} to set * @return a reference to this Builder */ public Builder withRetryTaskId(String val) { retryTaskId = val; return this; } /** * Sets the {@code taskId} and returns a reference to this Builder so that the methods can * be chained together. * * @param val the {@code taskId} to set * @return a reference to this Builder */ public Builder withTaskId(String val) { taskId = val; return this; } /** * Sets the {@code deciderService} and returns a reference to this Builder so that the * methods can be chained together. * * @param val the {@code deciderService} to set * @return a reference to this Builder */ public Builder withDeciderService(DeciderService val) { deciderService = val; return this; } /** * Returns a {@code TaskMapperContext} built from the parameters previously set. * * @return a {@code TaskMapperContext} built with parameters of this {@code * TaskMapperContext.Builder} */ public TaskMapperContext build() { return new TaskMapperContext(this); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/TerminateTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_TERMINATE; @Component public class TerminateTaskMapper implements TaskMapper { public static final Logger logger = LoggerFactory.getLogger(TerminateTaskMapper.class); private final ParametersUtils parametersUtils; public TerminateTaskMapper(ParametersUtils parametersUtils) { this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.TERMINATE.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { logger.debug("TaskMapperContext {} in TerminateTaskMapper", taskMapperContext); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); Map taskInput = parametersUtils.getTaskInputV2( taskMapperContext.getWorkflowTask().getInputParameters(), workflowModel, taskId, null); TaskModel task = taskMapperContext.createTaskModel(); task.setTaskType(TASK_TYPE_TERMINATE); task.setStartTime(System.currentTimeMillis()); task.setInputData(taskInput); task.setStatus(TaskModel.Status.IN_PROGRESS); return List.of(task); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/UserDefinedTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#USER_DEFINED} to a {@link TaskModel} of type {@link TaskType#USER_DEFINED} with {@link * TaskModel.Status#SCHEDULED} */ @Component public class UserDefinedTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(UserDefinedTaskMapper.class); private final ParametersUtils parametersUtils; private final MetadataDAO metadataDAO; public UserDefinedTaskMapper(ParametersUtils parametersUtils, MetadataDAO metadataDAO) { this.parametersUtils = parametersUtils; this.metadataDAO = metadataDAO; } @Override public String getTaskType() { return TaskType.USER_DEFINED.name(); } /** * This method maps a {@link WorkflowTask} of type {@link TaskType#USER_DEFINED} to a {@link * TaskModel} in a {@link TaskModel.Status#SCHEDULED} state * * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId * @return a List with just one User defined task * @throws TerminateWorkflowException In case if the task definition does not exist */ @Override public List getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException { LOGGER.debug("TaskMapperContext {} in UserDefinedTaskMapper", taskMapperContext); WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); int retryCount = taskMapperContext.getRetryCount(); TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()) .orElseGet( () -> Optional.ofNullable( metadataDAO.getTaskDef( workflowTask.getName())) .orElseThrow( () -> { String reason = String.format( "Invalid task specified. Cannot find task by name %s in the task definitions", workflowTask.getName()); return new TerminateWorkflowException( reason); })); Map input = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflowModel, taskId, taskDefinition); TaskModel userDefinedTask = taskMapperContext.createTaskModel(); userDefinedTask.setInputData(input); userDefinedTask.setStatus(TaskModel.Status.SCHEDULED); userDefinedTask.setRetryCount(retryCount); userDefinedTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); userDefinedTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); userDefinedTask.setRateLimitFrequencyInSeconds( taskDefinition.getRateLimitFrequencyInSeconds()); return List.of(userDefinedTask); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/mapper/WaitTaskMapper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.text.ParseException; import java.time.Duration; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.tasks.Wait; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; import static com.netflix.conductor.core.execution.tasks.Wait.DURATION_INPUT; import static com.netflix.conductor.core.execution.tasks.Wait.UNTIL_INPUT; import static com.netflix.conductor.core.utils.DateTimeUtils.parseDate; import static com.netflix.conductor.core.utils.DateTimeUtils.parseDuration; import static com.netflix.conductor.model.TaskModel.Status.FAILED_WITH_TERMINAL_ERROR; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#WAIT} to a {@link TaskModel} of type {@link Wait} with {@link * TaskModel.Status#IN_PROGRESS} */ @Component public class WaitTaskMapper implements TaskMapper { public static final Logger LOGGER = LoggerFactory.getLogger(WaitTaskMapper.class); private final ParametersUtils parametersUtils; public WaitTaskMapper(ParametersUtils parametersUtils) { this.parametersUtils = parametersUtils; } @Override public String getTaskType() { return TaskType.WAIT.name(); } @Override public List getMappedTasks(TaskMapperContext taskMapperContext) { LOGGER.debug("TaskMapperContext {} in WaitTaskMapper", taskMapperContext); WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); String taskId = taskMapperContext.getTaskId(); Map waitTaskInput = parametersUtils.getTaskInputV2( taskMapperContext.getWorkflowTask().getInputParameters(), workflowModel, taskId, null); TaskModel waitTask = taskMapperContext.createTaskModel(); waitTask.setTaskType(TASK_TYPE_WAIT); waitTask.setInputData(waitTaskInput); waitTask.setStartTime(System.currentTimeMillis()); waitTask.setStatus(TaskModel.Status.IN_PROGRESS); setCallbackAfter(waitTask); return List.of(waitTask); } void setCallbackAfter(TaskModel task) { String duration = Optional.ofNullable(task.getInputData().get(DURATION_INPUT)).orElse("").toString(); String until = Optional.ofNullable(task.getInputData().get(UNTIL_INPUT)).orElse("").toString(); if (StringUtils.isNotBlank(duration) && StringUtils.isNotBlank(until)) { task.setReasonForIncompletion( "Both 'duration' and 'until' specified. Please provide only one input"); task.setStatus(FAILED_WITH_TERMINAL_ERROR); return; } if (StringUtils.isNotBlank(duration)) { Duration timeDuration = parseDuration(duration); long waitTimeout = System.currentTimeMillis() + (timeDuration.getSeconds() * 1000); task.setWaitTimeout(waitTimeout); long seconds = timeDuration.getSeconds(); task.setCallbackAfterSeconds(seconds); } else if (StringUtils.isNotBlank(until)) { try { Date expiryDate = parseDate(until); long timeInMS = expiryDate.getTime(); long now = System.currentTimeMillis(); long seconds = ((timeInMS - now) / 1000); if (seconds < 0) { seconds = 0; } task.setCallbackAfterSeconds(seconds); task.setWaitTimeout(timeInMS); } catch (ParseException parseException) { task.setReasonForIncompletion( "Invalid/Unsupported Wait Until format. Provided: " + until); task.setStatus(FAILED_WITH_TERMINAL_ERROR); } } else { // If there is no time duration specified then the WAIT task should wait forever task.setCallbackAfterSeconds(Integer.MAX_VALUE); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Decision.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DECISION; /** * @deprecated {@link Decision} is deprecated. Use {@link Switch} task for condition evaluation * using the extensible evaluation framework. Also see ${@link * com.netflix.conductor.common.metadata.workflow.WorkflowTask}). */ @Deprecated @Component(TASK_TYPE_DECISION) public class Decision extends WorkflowSystemTask { public Decision() { super(TASK_TYPE_DECISION); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(TaskModel.Status.COMPLETED); return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/DoWhile.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.*; import java.util.stream.Collectors; import javax.script.ScriptException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.events.ScriptEvaluator; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DO_WHILE; @Component(TASK_TYPE_DO_WHILE) public class DoWhile extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(DoWhile.class); private final ParametersUtils parametersUtils; public DoWhile(ParametersUtils parametersUtils) { super(TASK_TYPE_DO_WHILE); this.parametersUtils = parametersUtils; } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { task.setStatus(TaskModel.Status.CANCELED); } @Override public boolean execute( WorkflowModel workflow, TaskModel doWhileTaskModel, WorkflowExecutor workflowExecutor) { boolean hasFailures = false; StringBuilder failureReason = new StringBuilder(); Map output = new HashMap<>(); /* * Get the latest set of tasks (the ones that have the highest retry count). We don't want to evaluate any tasks * that have already failed if there is a more current one (a later retry count). */ Map relevantTasks = new LinkedHashMap<>(); TaskModel relevantTask; for (TaskModel t : workflow.getTasks()) { if (doWhileTaskModel .getWorkflowTask() .has(TaskUtils.removeIterationFromTaskRefName(t.getReferenceTaskName())) && !doWhileTaskModel.getReferenceTaskName().equals(t.getReferenceTaskName()) && doWhileTaskModel.getIteration() == t.getIteration()) { relevantTask = relevantTasks.get(t.getReferenceTaskName()); if (relevantTask == null || t.getRetryCount() > relevantTask.getRetryCount()) { relevantTasks.put(t.getReferenceTaskName(), t); } } } Collection loopOverTasks = relevantTasks.values(); if (LOGGER.isDebugEnabled()) { LOGGER.debug( "Workflow {} waiting for tasks {} to complete iteration {}", workflow.getWorkflowId(), loopOverTasks.stream() .map(TaskModel::getReferenceTaskName) .collect(Collectors.toList()), doWhileTaskModel.getIteration()); } // if the loopOverTasks collection is empty, no tasks inside the loop have been scheduled. // so schedule it and exit the method. if (loopOverTasks.isEmpty()) { doWhileTaskModel.setIteration(1); doWhileTaskModel.addOutput("iteration", doWhileTaskModel.getIteration()); return scheduleNextIteration(doWhileTaskModel, workflow, workflowExecutor); } for (TaskModel loopOverTask : loopOverTasks) { TaskModel.Status taskStatus = loopOverTask.getStatus(); hasFailures = !taskStatus.isSuccessful(); if (hasFailures) { failureReason.append(loopOverTask.getReasonForIncompletion()).append(" "); } output.put( TaskUtils.removeIterationFromTaskRefName(loopOverTask.getReferenceTaskName()), loopOverTask.getOutputData()); if (hasFailures) { break; } } doWhileTaskModel.addOutput(String.valueOf(doWhileTaskModel.getIteration()), output); if (hasFailures) { LOGGER.debug( "Task {} failed in {} iteration", doWhileTaskModel.getTaskId(), doWhileTaskModel.getIteration() + 1); return markTaskFailure( doWhileTaskModel, TaskModel.Status.FAILED, failureReason.toString()); } if (!isIterationComplete(doWhileTaskModel, relevantTasks)) { // current iteration is not complete (all tasks inside the loop are not terminal) return false; } // if we are here, the iteration is complete, and we need to check if there is a next // iteration by evaluating the loopCondition boolean shouldContinue; try { shouldContinue = evaluateCondition(workflow, doWhileTaskModel); LOGGER.debug( "Task {} condition evaluated to {}", doWhileTaskModel.getTaskId(), shouldContinue); if (shouldContinue) { doWhileTaskModel.setIteration(doWhileTaskModel.getIteration() + 1); doWhileTaskModel.addOutput("iteration", doWhileTaskModel.getIteration()); return scheduleNextIteration(doWhileTaskModel, workflow, workflowExecutor); } else { LOGGER.debug( "Task {} took {} iterations to complete", doWhileTaskModel.getTaskId(), doWhileTaskModel.getIteration() + 1); return markTaskSuccess(doWhileTaskModel); } } catch (ScriptException e) { String message = String.format( "Unable to evaluate condition %s, exception %s", doWhileTaskModel.getWorkflowTask().getLoopCondition(), e.getMessage()); LOGGER.error(message); return markTaskFailure( doWhileTaskModel, TaskModel.Status.FAILED_WITH_TERMINAL_ERROR, message); } } /** * Check if all tasks in the current iteration have reached terminal state. * * @param doWhileTaskModel The {@link TaskModel} of DO_WHILE. * @param referenceNameToModel Map of taskReferenceName to {@link TaskModel}. * @return true if all tasks in DO_WHILE.loopOver are in referenceNameToModel and * reached terminal state. */ private boolean isIterationComplete( TaskModel doWhileTaskModel, Map referenceNameToModel) { List workflowTasksInsideDoWhile = doWhileTaskModel.getWorkflowTask().getLoopOver(); int iteration = doWhileTaskModel.getIteration(); boolean allTasksTerminal = true; for (WorkflowTask workflowTaskInsideDoWhile : workflowTasksInsideDoWhile) { String taskReferenceName = TaskUtils.appendIteration( workflowTaskInsideDoWhile.getTaskReferenceName(), iteration); if (referenceNameToModel.containsKey(taskReferenceName)) { TaskModel taskModel = referenceNameToModel.get(taskReferenceName); if (!taskModel.getStatus().isTerminal()) { allTasksTerminal = false; break; } } else { allTasksTerminal = false; break; } } if (!allTasksTerminal) { // Cases where tasks directly inside loop over are not completed. // loopOver -> [task1 -> COMPLETED, task2 -> IN_PROGRESS] return false; } // Check all the tasks in referenceNameToModel are completed or not. These are set of tasks // which are not directly inside loopOver tasks, but they are under hierarchy // loopOver -> [decisionTask -> COMPLETED [ task1 -> COMPLETED, task2 -> IN_PROGRESS]] return referenceNameToModel.values().stream() .noneMatch(taskModel -> !taskModel.getStatus().isTerminal()); } boolean scheduleNextIteration( TaskModel doWhileTaskModel, WorkflowModel workflow, WorkflowExecutor workflowExecutor) { LOGGER.debug( "Scheduling loop tasks for task {} as condition {} evaluated to true", doWhileTaskModel.getTaskId(), doWhileTaskModel.getWorkflowTask().getLoopCondition()); workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflow); return true; // Return true even though status not changed. Iteration has to be updated in // execution DAO. } boolean markTaskFailure(TaskModel taskModel, TaskModel.Status status, String failureReason) { LOGGER.error("Marking task {} failed with error.", taskModel.getTaskId()); taskModel.setReasonForIncompletion(failureReason); taskModel.setStatus(status); return true; } boolean markTaskSuccess(TaskModel taskModel) { LOGGER.debug( "Task {} took {} iterations to complete", taskModel.getTaskId(), taskModel.getIteration() + 1); taskModel.setStatus(TaskModel.Status.COMPLETED); return true; } @VisibleForTesting boolean evaluateCondition(WorkflowModel workflow, TaskModel task) throws ScriptException { TaskDef taskDefinition = task.getTaskDefinition().orElse(null); // Use paramUtils to compute the task input Map conditionInput = parametersUtils.getTaskInputV2( task.getWorkflowTask().getInputParameters(), workflow, task.getTaskId(), taskDefinition); conditionInput.put(task.getReferenceTaskName(), task.getOutputData()); List loopOver = workflow.getTasks().stream() .filter( t -> (task.getWorkflowTask() .has( TaskUtils .removeIterationFromTaskRefName( t .getReferenceTaskName())) && !task.getReferenceTaskName() .equals(t.getReferenceTaskName()))) .collect(Collectors.toList()); for (TaskModel loopOverTask : loopOver) { conditionInput.put( TaskUtils.removeIterationFromTaskRefName(loopOverTask.getReferenceTaskName()), loopOverTask.getOutputData()); } String condition = task.getWorkflowTask().getLoopCondition(); boolean result = false; if (condition != null) { LOGGER.debug("Condition: {} is being evaluated", condition); // Evaluate the expression by using the Nashorn based script evaluator result = ScriptEvaluator.evalBool(condition, conditionInput); } return result; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Event.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.events.EventQueues; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_EVENT; @Component(TASK_TYPE_EVENT) public class Event extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(Event.class); public static final String NAME = "EVENT"; private static final String EVENT_PRODUCED = "event_produced"; private final ObjectMapper objectMapper; private final ParametersUtils parametersUtils; private final EventQueues eventQueues; public Event( EventQueues eventQueues, ParametersUtils parametersUtils, ObjectMapper objectMapper) { super(TASK_TYPE_EVENT); this.parametersUtils = parametersUtils; this.eventQueues = eventQueues; this.objectMapper = objectMapper; } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { Map payload = new HashMap<>(task.getInputData()); payload.put("workflowInstanceId", workflow.getWorkflowId()); payload.put("workflowType", workflow.getWorkflowName()); payload.put("workflowVersion", workflow.getWorkflowVersion()); payload.put("correlationId", workflow.getCorrelationId()); task.setStatus(TaskModel.Status.IN_PROGRESS); task.addOutput(payload); try { task.addOutput(EVENT_PRODUCED, computeQueueName(workflow, task)); } catch (Exception e) { task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion(e.getMessage()); LOGGER.error( "Error executing task: {}, workflow: {}", task.getTaskId(), workflow.getWorkflowId(), e); } } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { try { String queueName = (String) task.getOutputData().get(EVENT_PRODUCED); ObservableQueue queue = getQueue(queueName, task.getTaskId()); Message message = getPopulatedMessage(task); queue.publish(List.of(message)); LOGGER.debug("Published message:{} to queue:{}", message.getId(), queue.getName()); if (!isAsyncComplete(task)) { task.setStatus(TaskModel.Status.COMPLETED); return true; } } catch (JsonProcessingException jpe) { task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion("Error serializing JSON payload: " + jpe.getMessage()); LOGGER.error( "Error serializing JSON payload for task: {}, workflow: {}", task.getTaskId(), workflow.getWorkflowId()); } catch (Exception e) { task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion(e.getMessage()); LOGGER.error( "Error executing task: {}, workflow: {}", task.getTaskId(), workflow.getWorkflowId(), e); } return false; } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { Message message = new Message(task.getTaskId(), null, task.getTaskId()); String queueName = computeQueueName(workflow, task); ObservableQueue queue = getQueue(queueName, task.getTaskId()); queue.ack(List.of(message)); } @VisibleForTesting String computeQueueName(WorkflowModel workflow, TaskModel task) { String sinkValueRaw = (String) task.getInputData().get("sink"); Map input = new HashMap<>(); input.put("sink", sinkValueRaw); Map replaced = parametersUtils.getTaskInputV2(input, workflow, task.getTaskId(), null); String sinkValue = (String) replaced.get("sink"); String queueName = sinkValue; if (sinkValue.startsWith("conductor")) { if ("conductor".equals(sinkValue)) { queueName = sinkValue + ":" + workflow.getWorkflowName() + ":" + task.getReferenceTaskName(); } else if (sinkValue.startsWith("conductor:")) { queueName = "conductor:" + workflow.getWorkflowName() + ":" + sinkValue.replaceAll("conductor:", ""); } else { throw new IllegalStateException( "Invalid / Unsupported sink specified: " + sinkValue); } } return queueName; } @VisibleForTesting ObservableQueue getQueue(String queueName, String taskId) { try { return eventQueues.getQueue(queueName); } catch (IllegalArgumentException e) { throw new IllegalStateException( "Error loading queue:" + queueName + ", for task:" + taskId + ", error: " + e.getMessage()); } catch (Exception e) { throw new NonTransientException("Unable to find queue name for task " + taskId); } } Message getPopulatedMessage(TaskModel task) throws JsonProcessingException { String payloadJson = objectMapper.writeValueAsString(task.getOutputData()); return new Message(task.getTaskId(), payloadJson, task.getTaskId()); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/ExclusiveJoin.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.List; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_EXCLUSIVE_JOIN; @Component(TASK_TYPE_EXCLUSIVE_JOIN) public class ExclusiveJoin extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(ExclusiveJoin.class); private static final String DEFAULT_EXCLUSIVE_JOIN_TASKS = "defaultExclusiveJoinTask"; public ExclusiveJoin() { super(TASK_TYPE_EXCLUSIVE_JOIN); } @Override @SuppressWarnings("unchecked") public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { boolean foundExlusiveJoinOnTask = false; boolean hasFailures = false; StringBuilder failureReason = new StringBuilder(); TaskModel.Status taskStatus; List joinOn = (List) task.getInputData().get("joinOn"); if (task.isLoopOverTask()) { // If exclusive join is part of loop over task, wait for specific iteration to get // complete joinOn = joinOn.stream() .map(name -> TaskUtils.appendIteration(name, task.getIteration())) .collect(Collectors.toList()); } TaskModel exclusiveTask = null; for (String joinOnRef : joinOn) { LOGGER.debug("Exclusive Join On Task {} ", joinOnRef); exclusiveTask = workflow.getTaskByRefName(joinOnRef); if (exclusiveTask == null || exclusiveTask.getStatus() == TaskModel.Status.SKIPPED) { LOGGER.debug("The task {} is either not scheduled or skipped.", joinOnRef); continue; } taskStatus = exclusiveTask.getStatus(); foundExlusiveJoinOnTask = taskStatus.isTerminal(); hasFailures = !taskStatus.isSuccessful(); if (hasFailures) { failureReason.append(exclusiveTask.getReasonForIncompletion()).append(" "); } break; } if (!foundExlusiveJoinOnTask) { List defaultExclusiveJoinTasks = (List) task.getInputData().get(DEFAULT_EXCLUSIVE_JOIN_TASKS); LOGGER.info( "Could not perform exclusive on Join Task(s). Performing now on default exclusive join task(s) {}, workflow: {}", defaultExclusiveJoinTasks, workflow.getWorkflowId()); if (defaultExclusiveJoinTasks != null && !defaultExclusiveJoinTasks.isEmpty()) { for (String defaultExclusiveJoinTask : defaultExclusiveJoinTasks) { // Pick the first task that we should join on and break. exclusiveTask = workflow.getTaskByRefName(defaultExclusiveJoinTask); if (exclusiveTask == null || exclusiveTask.getStatus() == TaskModel.Status.SKIPPED) { LOGGER.debug( "The task {} is either not scheduled or skipped.", defaultExclusiveJoinTask); continue; } taskStatus = exclusiveTask.getStatus(); foundExlusiveJoinOnTask = taskStatus.isTerminal(); hasFailures = !taskStatus.isSuccessful(); if (hasFailures) { failureReason.append(exclusiveTask.getReasonForIncompletion()).append(" "); } break; } } else { LOGGER.debug( "Could not evaluate last tasks output. Verify the task configuration in the workflow definition."); } } LOGGER.debug( "Status of flags: foundExlusiveJoinOnTask: {}, hasFailures {}", foundExlusiveJoinOnTask, hasFailures); if (foundExlusiveJoinOnTask || hasFailures) { if (hasFailures) { task.setReasonForIncompletion(failureReason.toString()); task.setStatus(TaskModel.Status.FAILED); } else { task.setOutputData(exclusiveTask.getOutputData()); task.setStatus(TaskModel.Status.COMPLETED); } LOGGER.debug("Task: {} status is: {}", task.getTaskId(), task.getStatus()); return true; } return false; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/ExecutionConfig.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import com.netflix.conductor.core.utils.SemaphoreUtil; class ExecutionConfig { private final ExecutorService executorService; private final SemaphoreUtil semaphoreUtil; ExecutionConfig(int threadCount, String threadNameFormat) { this.executorService = Executors.newFixedThreadPool( threadCount, new BasicThreadFactory.Builder().namingPattern(threadNameFormat).build()); this.semaphoreUtil = new SemaphoreUtil(threadCount); } public ExecutorService getExecutorService() { return executorService; } public SemaphoreUtil getSemaphoreUtil() { return semaphoreUtil; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Fork.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK; @Component(TASK_TYPE_FORK) public class Fork extends WorkflowSystemTask { public Fork() { super(TASK_TYPE_FORK); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Human.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HUMAN; import static com.netflix.conductor.model.TaskModel.Status.IN_PROGRESS; @Component(TASK_TYPE_HUMAN) public class Human extends WorkflowSystemTask { public Human() { super(TASK_TYPE_HUMAN); } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(IN_PROGRESS); } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(TaskModel.Status.CANCELED); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Inline.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_INLINE; /** * @author X-Ultra *

    Task that enables execute inline script at workflow execution. For example, * 

     * ...
 * {
 *  "tasks": [
 *      {
 *          "name": "INLINE",
 *          "taskReferenceName": "inline_test",
 *          "type": "INLINE",
 *          "inputParameters": {
 *              "input": "${workflow.input}",
 *              "evaluatorType": "javascript"
 *              "expression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false} }"
 *          }
 *      }
 *  ]
 * }
 * ...
 *
    * then to use task output, e.g. script_test.output.testvalue {@link Inline} is a * replacement for deprecated {@link Lambda} */ @Component(TASK_TYPE_INLINE) public class Inline extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(Inline.class); private static final String QUERY_EVALUATOR_TYPE = "evaluatorType"; private static final String QUERY_EXPRESSION_PARAMETER = "expression"; public static final String NAME = "INLINE"; private final Map evaluators; public Inline(Map evaluators) { super(TASK_TYPE_INLINE); this.evaluators = evaluators; } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { Map taskInput = task.getInputData(); String evaluatorType = (String) taskInput.get(QUERY_EVALUATOR_TYPE); String expression = (String) taskInput.get(QUERY_EXPRESSION_PARAMETER); try { checkEvaluatorType(evaluatorType); checkExpression(expression); Evaluator evaluator = evaluators.get(evaluatorType); Object evalResult = evaluator.evaluate(expression, taskInput); task.addOutput("result", evalResult); task.setStatus(TaskModel.Status.COMPLETED); } catch (Exception e) { String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); LOGGER.error( "Failed to execute Inline Task: {} in workflow: {}", task.getTaskId(), workflow.getWorkflowId(), e); // TerminateWorkflowException is thrown when the script evaluation fails // Retry will result in the same error, so FAILED_WITH_TERMINAL_ERROR status is used. task.setStatus( e instanceof TerminateWorkflowException ? TaskModel.Status.FAILED_WITH_TERMINAL_ERROR : TaskModel.Status.FAILED); task.setReasonForIncompletion(errorMessage); task.addOutput("error", errorMessage); } return true; } private void checkEvaluatorType(String evaluatorType) { if (StringUtils.isBlank(evaluatorType)) { LOGGER.error("Empty {} in INLINE task. ", QUERY_EVALUATOR_TYPE); throw new TerminateWorkflowException( "Empty '" + QUERY_EVALUATOR_TYPE + "' in INLINE task's input parameters. A non-empty String value must be provided."); } if (evaluators.get(evaluatorType) == null) { LOGGER.error("Evaluator {} for INLINE task not registered", evaluatorType); throw new TerminateWorkflowException( "Unknown evaluator '" + evaluatorType + "' in INLINE task."); } } private void checkExpression(String expression) { if (StringUtils.isBlank(expression)) { LOGGER.error("Empty {} in INLINE task. ", QUERY_EXPRESSION_PARAMETER); throw new TerminateWorkflowException( "Empty '" + QUERY_EXPRESSION_PARAMETER + "' in Inline task's input parameters. A non-empty String value must be provided."); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducer.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.time.Duration; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.service.MetadataService; import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; @Component @ConditionalOnProperty( name = "conductor.system-task-workers.enabled", havingValue = "true", matchIfMissing = true) public class IsolatedTaskQueueProducer { private static final Logger LOGGER = LoggerFactory.getLogger(IsolatedTaskQueueProducer.class); private final MetadataService metadataService; private final Set asyncSystemTasks; private final SystemTaskWorker systemTaskWorker; private final Set listeningQueues = new HashSet<>(); public IsolatedTaskQueueProducer( MetadataService metadataService, @Qualifier(ASYNC_SYSTEM_TASKS_QUALIFIER) Set asyncSystemTasks, SystemTaskWorker systemTaskWorker, @Value("${conductor.app.isolatedSystemTaskEnabled:false}") boolean isolatedSystemTaskEnabled, @Value("${conductor.app.isolatedSystemTaskQueuePollInterval:10s}") Duration isolatedSystemTaskQueuePollInterval) { this.metadataService = metadataService; this.asyncSystemTasks = asyncSystemTasks; this.systemTaskWorker = systemTaskWorker; if (isolatedSystemTaskEnabled) { LOGGER.info("Listening for isolation groups"); Executors.newSingleThreadScheduledExecutor() .scheduleWithFixedDelay( this::addTaskQueues, 1000, isolatedSystemTaskQueuePollInterval.toMillis(), TimeUnit.MILLISECONDS); } else { LOGGER.info("Isolated System Task Worker DISABLED"); } } private Set getIsolationExecutionNameSpaces() { Set isolationExecutionNameSpaces = Collections.emptySet(); try { List taskDefs = metadataService.getTaskDefs(); isolationExecutionNameSpaces = taskDefs.stream() .filter( taskDef -> StringUtils.isNotBlank(taskDef.getIsolationGroupId()) || StringUtils.isNotBlank( taskDef.getExecutionNameSpace())) .collect(Collectors.toSet()); } catch (RuntimeException e) { LOGGER.error( "Unknown exception received in getting isolation groups, sleeping and retrying", e); } return isolationExecutionNameSpaces; } @VisibleForTesting void addTaskQueues() { Set isolationTaskDefs = getIsolationExecutionNameSpaces(); LOGGER.debug("Retrieved queues {}", isolationTaskDefs); for (TaskDef isolatedTaskDef : isolationTaskDefs) { for (WorkflowSystemTask systemTask : this.asyncSystemTasks) { String taskQueue = QueueUtils.getQueueName( systemTask.getTaskType(), null, isolatedTaskDef.getIsolationGroupId(), isolatedTaskDef.getExecutionNameSpace()); LOGGER.debug("Adding taskQueue:'{}' to system task worker coordinator", taskQueue); if (!listeningQueues.contains(taskQueue)) { systemTaskWorker.startPolling(systemTask, taskQueue); listeningQueues.add(taskQueue); } } } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Join.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.springframework.stereotype.Component; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN; @Component(TASK_TYPE_JOIN) public class Join extends WorkflowSystemTask { public Join() { super(TASK_TYPE_JOIN); } @Override @SuppressWarnings("unchecked") public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { boolean allDone = true; boolean hasFailures = false; StringBuilder failureReason = new StringBuilder(); StringBuilder optionalTaskFailures = new StringBuilder(); List joinOn = (List) task.getInputData().get("joinOn"); if (task.isLoopOverTask()) { // If join is part of loop over task, wait for specific iteration to get complete joinOn = joinOn.stream() .map(name -> TaskUtils.appendIteration(name, task.getIteration())) .collect(Collectors.toList()); } for (String joinOnRef : joinOn) { TaskModel forkedTask = workflow.getTaskByRefName(joinOnRef); if (forkedTask == null) { // Task is not even scheduled yet allDone = false; break; } TaskModel.Status taskStatus = forkedTask.getStatus(); hasFailures = !taskStatus.isSuccessful() && !forkedTask.getWorkflowTask().isOptional(); if (hasFailures) { failureReason.append(forkedTask.getReasonForIncompletion()).append(" "); } // Only add to task output if it's not empty if (!forkedTask.getOutputData().isEmpty()) { task.addOutput(joinOnRef, forkedTask.getOutputData()); } if (!taskStatus.isTerminal()) { allDone = false; } if (hasFailures) { break; } // check for optional task failures if (forkedTask.getWorkflowTask().isOptional() && taskStatus == TaskModel.Status.COMPLETED_WITH_ERRORS) { optionalTaskFailures .append( String.format( "%s/%s", forkedTask.getTaskDefName(), forkedTask.getTaskId())) .append(" "); } } if (allDone || hasFailures || optionalTaskFailures.length() > 0) { if (hasFailures) { task.setReasonForIncompletion(failureReason.toString()); task.setStatus(TaskModel.Status.FAILED); } else if (optionalTaskFailures.length() > 0) { task.setStatus(TaskModel.Status.COMPLETED_WITH_ERRORS); optionalTaskFailures.append("completed with errors"); task.setReasonForIncompletion(optionalTaskFailures.toString()); } else { task.setStatus(TaskModel.Status.COMPLETED); } return true; } return false; } @Override public Optional getEvaluationOffset(TaskModel taskModel, long defaultOffset) { int index = taskModel.getPollCount() > 0 ? taskModel.getPollCount() - 1 : 0; if (index == 0) { return Optional.of(0L); } return Optional.of(Math.min((long) Math.pow(2, index), defaultOffset)); } public boolean isAsync() { return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Lambda.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.events.ScriptEvaluator; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_LAMBDA; /** * @author X-Ultra *

    Task that enables execute Lambda script at workflow execution, For example, * 

     * ...
 * {
 *  "tasks": [
 *      {
 *          "name": "LAMBDA",
 *          "taskReferenceName": "lambda_test",
 *          "type": "LAMBDA",
 *          "inputParameters": {
 *              "input": "${workflow.input}",
 *              "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false} }"
 *          }
 *      }
 *  ]
 * }
 * ...
 *
    * then to use task output, e.g. script_test.output.testvalue * @deprecated {@link Lambda} is deprecated. Use {@link Inline} task for inline expression * evaluation. Also see ${@link com.netflix.conductor.common.metadata.workflow.WorkflowTask}) */ @Deprecated @Component(TASK_TYPE_LAMBDA) public class Lambda extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(Lambda.class); private static final String QUERY_EXPRESSION_PARAMETER = "scriptExpression"; public static final String NAME = "LAMBDA"; public Lambda() { super(TASK_TYPE_LAMBDA); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { Map taskInput = task.getInputData(); String scriptExpression; try { scriptExpression = (String) taskInput.get(QUERY_EXPRESSION_PARAMETER); if (StringUtils.isNotBlank(scriptExpression)) { String scriptExpressionBuilder = "function scriptFun(){" + scriptExpression + "} scriptFun();"; LOGGER.debug( "scriptExpressionBuilder: {}, task: {}", scriptExpressionBuilder, task.getTaskId()); Object returnValue = ScriptEvaluator.eval(scriptExpressionBuilder, taskInput); task.addOutput("result", returnValue); task.setStatus(TaskModel.Status.COMPLETED); } else { LOGGER.error("Empty {} in Lambda task. ", QUERY_EXPRESSION_PARAMETER); task.setReasonForIncompletion( "Empty '" + QUERY_EXPRESSION_PARAMETER + "' in Lambda task's input parameters. A non-empty String value must be provided."); task.setStatus(TaskModel.Status.FAILED); } } catch (Exception e) { LOGGER.error( "Failed to execute Lambda Task: {} in workflow: {}", task.getTaskId(), workflow.getWorkflowId(), e); task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion(e.getMessage()); task.addOutput( "error", e.getCause() != null ? e.getCause().getMessage() : e.getMessage()); } return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Noop.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_NOOP; @Component(TASK_TYPE_NOOP) public class Noop extends WorkflowSystemTask { public Noop() { super(TASK_TYPE_NOOP); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(TaskModel.Status.COMPLETED); return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/SetVariable.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SET_VARIABLE; @Component(TASK_TYPE_SET_VARIABLE) public class SetVariable extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(SetVariable.class); private final ConductorProperties properties; private final ObjectMapper objectMapper; private final ExecutionDAOFacade executionDAOFacade; public SetVariable( ConductorProperties properties, ObjectMapper objectMapper, ExecutionDAOFacade executionDAOFacade) { super(TASK_TYPE_SET_VARIABLE); this.properties = properties; this.objectMapper = objectMapper; this.executionDAOFacade = executionDAOFacade; } private boolean validateVariablesSize( WorkflowModel workflow, TaskModel task, Map variables) { String workflowId = workflow.getWorkflowId(); long maxThreshold = properties.getMaxWorkflowVariablesPayloadSizeThreshold().toKilobytes(); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { this.objectMapper.writeValue(byteArrayOutputStream, variables); byte[] payloadBytes = byteArrayOutputStream.toByteArray(); long payloadSize = payloadBytes.length; if (payloadSize > maxThreshold * 1024) { String errorMsg = String.format( "The variables payload size: %d of workflow: %s is greater than the permissible limit: %d bytes", payloadSize, workflowId, maxThreshold); LOGGER.error(errorMsg); task.setReasonForIncompletion(errorMsg); return false; } return true; } catch (IOException e) { LOGGER.error( "Unable to validate variables payload size of workflow: {}", workflowId, e); throw new NonTransientException( "Unable to validate variables payload size of workflow: " + workflowId, e); } } @Override public boolean execute(WorkflowModel workflow, TaskModel task, WorkflowExecutor provider) { Map variables = workflow.getVariables(); Map input = task.getInputData(); String taskId = task.getTaskId(); ArrayList newKeys; Map previousValues; if (input != null && input.size() > 0) { newKeys = new ArrayList<>(); previousValues = new HashMap<>(); input.keySet() .forEach( key -> { if (variables.containsKey(key)) { previousValues.put(key, variables.get(key)); } else { newKeys.add(key); } variables.put(key, input.get(key)); LOGGER.debug( "Task: {} setting value for variable: {}", taskId, key); }); if (!validateVariablesSize(workflow, task, variables)) { // restore previous variables previousValues .keySet() .forEach( key -> { variables.put(key, previousValues.get(key)); }); newKeys.forEach(variables::remove); task.setStatus(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); return true; } } task.setStatus(TaskModel.Status.COMPLETED); executionDAOFacade.updateWorkflow(workflow); return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/StartWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import javax.validation.Validator; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_START_WORKFLOW; import static com.netflix.conductor.model.TaskModel.Status.COMPLETED; import static com.netflix.conductor.model.TaskModel.Status.FAILED; @Component(TASK_TYPE_START_WORKFLOW) public class StartWorkflow extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(StartWorkflow.class); private static final String WORKFLOW_ID = "workflowId"; private static final String START_WORKFLOW_PARAMETER = "startWorkflow"; private final ObjectMapper objectMapper; private final Validator validator; private final StartWorkflowOperation startWorkflowOperation; public StartWorkflow( ObjectMapper objectMapper, Validator validator, StartWorkflowOperation startWorkflowOperation) { super(TASK_TYPE_START_WORKFLOW); this.objectMapper = objectMapper; this.validator = validator; this.startWorkflowOperation = startWorkflowOperation; } @Override public void start( WorkflowModel workflow, TaskModel taskModel, WorkflowExecutor workflowExecutor) { StartWorkflowRequest request = getRequest(taskModel); if (request == null) { return; } if (request.getTaskToDomain() == null || request.getTaskToDomain().isEmpty()) { Map workflowTaskToDomainMap = workflow.getTaskToDomain(); if (workflowTaskToDomainMap != null) { request.setTaskToDomain(new HashMap<>(workflowTaskToDomainMap)); } } // set the correlation id of starter workflow, if its empty in the StartWorkflowRequest request.setCorrelationId( StringUtils.defaultIfBlank( request.getCorrelationId(), workflow.getCorrelationId())); try { String workflowId = startWorkflow(request, workflow.getWorkflowId()); taskModel.addOutput(WORKFLOW_ID, workflowId); taskModel.setStatus(COMPLETED); } catch (TransientException te) { LOGGER.info( "A transient backend error happened when task {} in {} tried to start workflow {}.", taskModel.getTaskId(), workflow.toShortString(), request.getName()); } catch (Exception ae) { taskModel.setStatus(FAILED); taskModel.setReasonForIncompletion(ae.getMessage()); LOGGER.error( "Error starting workflow: {} from workflow: {}", request.getName(), workflow.toShortString(), ae); } } private StartWorkflowRequest getRequest(TaskModel taskModel) { Map taskInput = taskModel.getInputData(); StartWorkflowRequest startWorkflowRequest = null; if (taskInput.get(START_WORKFLOW_PARAMETER) == null) { taskModel.setStatus(FAILED); taskModel.setReasonForIncompletion( "Missing '" + START_WORKFLOW_PARAMETER + "' in input data."); } else { try { startWorkflowRequest = objectMapper.convertValue( taskInput.get(START_WORKFLOW_PARAMETER), StartWorkflowRequest.class); var violations = validator.validate(startWorkflowRequest); if (!violations.isEmpty()) { StringBuilder reasonForIncompletion = new StringBuilder(START_WORKFLOW_PARAMETER) .append(" validation failed. "); for (var violation : violations) { reasonForIncompletion .append("'") .append(violation.getPropertyPath().toString()) .append("' -> ") .append(violation.getMessage()) .append(". "); } taskModel.setStatus(FAILED); taskModel.setReasonForIncompletion(reasonForIncompletion.toString()); startWorkflowRequest = null; } } catch (IllegalArgumentException e) { LOGGER.error("Error reading StartWorkflowRequest for {}", taskModel, e); taskModel.setStatus(FAILED); taskModel.setReasonForIncompletion( "Error reading StartWorkflowRequest. " + e.getMessage()); } } return startWorkflowRequest; } private String startWorkflow(StartWorkflowRequest request, String workflowId) { StartWorkflowInput input = new StartWorkflowInput(request); input.setTriggeringWorkflowId(workflowId); return startWorkflowOperation.execute(input); } @Override public boolean isAsync() { return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/SubWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; @Component(TASK_TYPE_SUB_WORKFLOW) public class SubWorkflow extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(SubWorkflow.class); private static final String SUB_WORKFLOW_ID = "subWorkflowId"; private final ObjectMapper objectMapper; private final StartWorkflowOperation startWorkflowOperation; public SubWorkflow(ObjectMapper objectMapper, StartWorkflowOperation startWorkflowOperation) { super(TASK_TYPE_SUB_WORKFLOW); this.objectMapper = objectMapper; this.startWorkflowOperation = startWorkflowOperation; } @SuppressWarnings("unchecked") @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { Map input = task.getInputData(); String name = input.get("subWorkflowName").toString(); int version = (int) input.get("subWorkflowVersion"); WorkflowDef workflowDefinition = null; if (input.get("subWorkflowDefinition") != null) { // convert the value back to workflow definition object workflowDefinition = objectMapper.convertValue( input.get("subWorkflowDefinition"), WorkflowDef.class); name = workflowDefinition.getName(); } Map taskToDomain = workflow.getTaskToDomain(); if (input.get("subWorkflowTaskToDomain") instanceof Map) { taskToDomain = (Map) input.get("subWorkflowTaskToDomain"); } var wfInput = (Map) input.get("workflowInput"); if (wfInput == null || wfInput.isEmpty()) { wfInput = input; } String correlationId = workflow.getCorrelationId(); try { StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setWorkflowDefinition(workflowDefinition); startWorkflowInput.setName(name); startWorkflowInput.setVersion(version); startWorkflowInput.setWorkflowInput(wfInput); startWorkflowInput.setCorrelationId(correlationId); startWorkflowInput.setParentWorkflowId(workflow.getWorkflowId()); startWorkflowInput.setParentWorkflowTaskId(task.getTaskId()); startWorkflowInput.setTaskToDomain(taskToDomain); String subWorkflowId = startWorkflowOperation.execute(startWorkflowInput); task.setSubWorkflowId(subWorkflowId); // For backwards compatibility task.addOutput(SUB_WORKFLOW_ID, subWorkflowId); // Set task status based on current sub-workflow status, as the status can change in // recursion by the time we update here. WorkflowModel subWorkflow = workflowExecutor.getWorkflow(subWorkflowId, false); updateTaskStatus(subWorkflow, task); } catch (TransientException te) { LOGGER.info( "A transient backend error happened when task {} in {} tried to start sub workflow {}.", task.getTaskId(), workflow.toShortString(), name); } catch (Exception ae) { task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion(ae.getMessage()); LOGGER.error( "Error starting sub workflow: {} from workflow: {}", name, workflow.toShortString(), ae); } } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { String workflowId = task.getSubWorkflowId(); if (StringUtils.isEmpty(workflowId)) { return false; } WorkflowModel subWorkflow = workflowExecutor.getWorkflow(workflowId, false); WorkflowModel.Status subWorkflowStatus = subWorkflow.getStatus(); if (!subWorkflowStatus.isTerminal()) { return false; } updateTaskStatus(subWorkflow, task); return true; } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { String workflowId = task.getSubWorkflowId(); if (StringUtils.isEmpty(workflowId)) { return; } WorkflowModel subWorkflow = workflowExecutor.getWorkflow(workflowId, true); subWorkflow.setStatus(WorkflowModel.Status.TERMINATED); String reason = StringUtils.isEmpty(workflow.getReasonForIncompletion()) ? "Parent workflow has been terminated with status " + workflow.getStatus() : "Parent workflow has been terminated with reason: " + workflow.getReasonForIncompletion(); workflowExecutor.terminateWorkflow(subWorkflow, reason, null); } @Override public boolean isAsync() { return true; } /** * Keep Subworkflow task asyncComplete. The Subworkflow task will be executed once * asynchronously to move to IN_PROGRESS state, and will move to termination by Subworkflow's * completeWorkflow logic, there by avoiding periodic polling. * * @param task * @return */ @Override public boolean isAsyncComplete(TaskModel task) { return true; } private void updateTaskStatus(WorkflowModel subworkflow, TaskModel task) { WorkflowModel.Status status = subworkflow.getStatus(); switch (status) { case RUNNING: case PAUSED: task.setStatus(TaskModel.Status.IN_PROGRESS); break; case COMPLETED: task.setStatus(TaskModel.Status.COMPLETED); break; case FAILED: task.setStatus(TaskModel.Status.FAILED); break; case TERMINATED: task.setStatus(TaskModel.Status.CANCELED); break; case TIMED_OUT: task.setStatus(TaskModel.Status.TIMED_OUT); break; default: throw new NonTransientException( "Subworkflow status does not conform to relevant task status."); } if (status.isTerminal()) { if (subworkflow.getExternalOutputPayloadStoragePath() != null) { task.setExternalOutputPayloadStoragePath( subworkflow.getExternalOutputPayloadStoragePath()); } else { task.addOutput(subworkflow.getOutput()); } if (!status.isSuccessful()) { task.setReasonForIncompletion( String.format( "Sub workflow %s failure reason: %s", subworkflow.toShortString(), subworkflow.getReasonForIncompletion())); } } } /** * We don't need the tasks when retrieving the workflow data. * * @return false */ @Override public boolean isTaskRetrievalRequired() { return false; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Switch.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SWITCH; /** {@link Switch} task is a replacement for now deprecated {@link Decision} task. */ @Component(TASK_TYPE_SWITCH) public class Switch extends WorkflowSystemTask { public Switch() { super(TASK_TYPE_SWITCH); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(TaskModel.Status.COMPLETED); return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskRegistry.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.springframework.stereotype.Component; /** * A container class that holds a mapping of system task types {@link * com.netflix.conductor.common.metadata.tasks.TaskType} to {@link WorkflowSystemTask} instances. */ @Component public class SystemTaskRegistry { public static final String ASYNC_SYSTEM_TASKS_QUALIFIER = "asyncSystemTasks"; private final Map registry; public SystemTaskRegistry(Set tasks) { this.registry = tasks.stream() .collect( Collectors.toMap( WorkflowSystemTask::getTaskType, Function.identity())); } public WorkflowSystemTask get(String taskType) { return Optional.ofNullable(registry.get(taskType)) .orElseThrow( () -> new IllegalStateException( taskType + "not found in " + getClass().getSimpleName())); } public boolean isSystemTask(String taskType) { return registry.containsKey(taskType); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorker.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.core.utils.SemaphoreUtil; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.service.ExecutionService; /** The worker that polls and executes an async system task. */ @Component @ConditionalOnProperty( name = "conductor.system-task-workers.enabled", havingValue = "true", matchIfMissing = true) public class SystemTaskWorker extends LifecycleAwareComponent { private static final Logger LOGGER = LoggerFactory.getLogger(SystemTaskWorker.class); private final long pollInterval; private final QueueDAO queueDAO; ExecutionConfig defaultExecutionConfig; private final AsyncSystemTaskExecutor asyncSystemTaskExecutor; private final ConductorProperties properties; private final ExecutionService executionService; ConcurrentHashMap queueExecutionConfigMap = new ConcurrentHashMap<>(); public SystemTaskWorker( QueueDAO queueDAO, AsyncSystemTaskExecutor asyncSystemTaskExecutor, ConductorProperties properties, ExecutionService executionService) { this.properties = properties; int threadCount = properties.getSystemTaskWorkerThreadCount(); this.defaultExecutionConfig = new ExecutionConfig(threadCount, "system-task-worker-%d"); this.asyncSystemTaskExecutor = asyncSystemTaskExecutor; this.queueDAO = queueDAO; this.pollInterval = properties.getSystemTaskWorkerPollInterval().toMillis(); this.executionService = executionService; LOGGER.info("SystemTaskWorker initialized with {} threads", threadCount); } public void startPolling(WorkflowSystemTask systemTask) { startPolling(systemTask, systemTask.getTaskType()); } public void startPolling(WorkflowSystemTask systemTask, String queueName) { Executors.newSingleThreadScheduledExecutor() .scheduleWithFixedDelay( () -> this.pollAndExecute(systemTask, queueName), 1000, pollInterval, TimeUnit.MILLISECONDS); LOGGER.info("Started listening for task: {} in queue: {}", systemTask, queueName); } void pollAndExecute(WorkflowSystemTask systemTask, String queueName) { if (!isRunning()) { LOGGER.debug( "{} stopped. Not polling for task: {}", getClass().getSimpleName(), systemTask); return; } ExecutionConfig executionConfig = getExecutionConfig(queueName); SemaphoreUtil semaphoreUtil = executionConfig.getSemaphoreUtil(); ExecutorService executorService = executionConfig.getExecutorService(); String taskName = QueueUtils.getTaskType(queueName); int messagesToAcquire = semaphoreUtil.availableSlots(); try { if (messagesToAcquire <= 0 || !semaphoreUtil.acquireSlots(messagesToAcquire)) { // no available slots, do not poll Monitors.recordSystemTaskWorkerPollingLimited(queueName); return; } LOGGER.debug("Polling queue: {} with {} slots acquired", queueName, messagesToAcquire); List polledTaskIds = queueDAO.pop(queueName, messagesToAcquire, 200); Monitors.recordTaskPoll(queueName); LOGGER.debug("Polling queue:{}, got {} tasks", queueName, polledTaskIds.size()); if (polledTaskIds.size() > 0) { // Immediately release unused slots when number of messages acquired is less than // acquired slots if (polledTaskIds.size() < messagesToAcquire) { semaphoreUtil.completeProcessing(messagesToAcquire - polledTaskIds.size()); } for (String taskId : polledTaskIds) { if (StringUtils.isNotBlank(taskId)) { LOGGER.debug( "Task: {} from queue: {} being sent to the workflow executor", taskId, queueName); Monitors.recordTaskPollCount(queueName, 1); executionService.ackTaskReceived(taskId); CompletableFuture taskCompletableFuture = CompletableFuture.runAsync( () -> asyncSystemTaskExecutor.execute(systemTask, taskId), executorService); // release permit after processing is complete taskCompletableFuture.whenComplete( (r, e) -> semaphoreUtil.completeProcessing(1)); } else { semaphoreUtil.completeProcessing(1); } } } else { // no task polled, release permit semaphoreUtil.completeProcessing(messagesToAcquire); } } catch (Exception e) { // release the permit if exception is thrown during polling, because the thread would // not be busy semaphoreUtil.completeProcessing(messagesToAcquire); Monitors.recordTaskPollError(taskName, e.getClass().getSimpleName()); LOGGER.error("Error polling system task in queue:{}", queueName, e); } } @VisibleForTesting ExecutionConfig getExecutionConfig(String taskQueue) { if (!QueueUtils.isIsolatedQueue(taskQueue)) { return this.defaultExecutionConfig; } return queueExecutionConfigMap.computeIfAbsent( taskQueue, __ -> this.createExecutionConfig()); } private ExecutionConfig createExecutionConfig() { int threadCount = properties.getIsolatedSystemTaskWorkerThreadCount(); String threadNameFormat = "isolated-system-task-worker-%d"; return new ExecutionConfig(threadCount, threadNameFormat); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorkerCoordinator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.utils.QueueUtils; import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; @Component @ConditionalOnProperty( name = "conductor.system-task-workers.enabled", havingValue = "true", matchIfMissing = true) public class SystemTaskWorkerCoordinator { private static final Logger LOGGER = LoggerFactory.getLogger(SystemTaskWorkerCoordinator.class); private final SystemTaskWorker systemTaskWorker; private final String executionNameSpace; private final Set asyncSystemTasks; public SystemTaskWorkerCoordinator( SystemTaskWorker systemTaskWorker, ConductorProperties properties, @Qualifier(ASYNC_SYSTEM_TASKS_QUALIFIER) Set asyncSystemTasks) { this.systemTaskWorker = systemTaskWorker; this.asyncSystemTasks = asyncSystemTasks; this.executionNameSpace = properties.getSystemTaskWorkerExecutionNamespace(); } @EventListener(ApplicationReadyEvent.class) public void initSystemTaskExecutor() { this.asyncSystemTasks.stream() .filter(this::isFromCoordinatorExecutionNameSpace) .forEach(this.systemTaskWorker::startPolling); LOGGER.info( "{} initialized with {} async tasks", SystemTaskWorkerCoordinator.class.getSimpleName(), this.asyncSystemTasks.size()); } @VisibleForTesting boolean isFromCoordinatorExecutionNameSpace(WorkflowSystemTask systemTask) { String queueExecutionNameSpace = QueueUtils.getExecutionNameSpace(systemTask.getTaskType()); return StringUtils.equals(queueExecutionNameSpace, executionNameSpace); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Terminate.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_TERMINATE; import static com.netflix.conductor.common.run.Workflow.WorkflowStatus.*; /** * Task that can terminate a workflow with a given status and modify the workflow's output with a * given parameter, it can act as a "return" statement for conditions where you simply want to * terminate your workflow. For example, if you have a decision where the first condition is met, * you want to execute some tasks, otherwise you want to finish your workflow. * * 

     * ...
 * {
 *  "tasks": [
 *      {
 *          "name": "terminate",
 *          "taskReferenceName": "terminate0",
 *          "inputParameters": {
 *              "terminationStatus": "COMPLETED",
 *              "workflowOutput": "${task0.output}"
 *          },
 *          "type": "TERMINATE",
 *          "startDelay": 0,
 *          "optional": false
 *      }
 *   ]
 * }
 * ...
 *
    * * This task has some validations on creation and execution, they are: - the "terminationStatus" * parameter is mandatory and it can only receive the values "COMPLETED" or "FAILED" - the terminate * task cannot be optional */ @Component(TASK_TYPE_TERMINATE) public class Terminate extends WorkflowSystemTask { private static final String TERMINATION_STATUS_PARAMETER = "terminationStatus"; private static final String TERMINATION_REASON_PARAMETER = "terminationReason"; private static final String TERMINATION_WORKFLOW_OUTPUT = "workflowOutput"; public Terminate() { super(TASK_TYPE_TERMINATE); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { String returnStatus = (String) task.getInputData().get(TERMINATION_STATUS_PARAMETER); if (validateInputStatus(returnStatus)) { task.setOutputData(getInputFromParam(task.getInputData())); task.setStatus(TaskModel.Status.COMPLETED); return true; } task.setReasonForIncompletion("given termination status is not valid"); task.setStatus(TaskModel.Status.FAILED); return false; } public static String getTerminationStatusParameter() { return TERMINATION_STATUS_PARAMETER; } public static String getTerminationReasonParameter() { return TERMINATION_REASON_PARAMETER; } public static String getTerminationWorkflowOutputParameter() { return TERMINATION_WORKFLOW_OUTPUT; } public static Boolean validateInputStatus(String status) { return COMPLETED.name().equals(status) || FAILED.name().equals(status) || TERMINATED.name().equals(status); } @SuppressWarnings("unchecked") private Map getInputFromParam(Map taskInput) { HashMap output = new HashMap<>(); if (taskInput.get(TERMINATION_WORKFLOW_OUTPUT) == null) { return output; } if (taskInput.get(TERMINATION_WORKFLOW_OUTPUT) instanceof HashMap) { output.putAll((HashMap) taskInput.get(TERMINATION_WORKFLOW_OUTPUT)); return output; } output.put("output", taskInput.get(TERMINATION_WORKFLOW_OUTPUT)); return output; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/Wait.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; import static com.netflix.conductor.model.TaskModel.Status.*; @Component(TASK_TYPE_WAIT) public class Wait extends WorkflowSystemTask { public static final String DURATION_INPUT = "duration"; public static final String UNTIL_INPUT = "until"; public Wait() { super(TASK_TYPE_WAIT); } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { task.setStatus(TaskModel.Status.CANCELED); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { long timeOut = task.getWaitTimeout(); if (timeOut == 0) { return false; } if (System.currentTimeMillis() > timeOut) { task.setStatus(COMPLETED); return true; } return false; } public boolean isAsync() { return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/execution/tasks/WorkflowSystemTask.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Optional; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; public abstract class WorkflowSystemTask { private final String taskType; public WorkflowSystemTask(String taskType) { this.taskType = taskType; } /** * Start the task execution. * *

    Called only once, and first, when the task status is SCHEDULED. * * @param workflow Workflow for which the task is being started * @param task Instance of the Task * @param workflowExecutor Workflow Executor */ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { // Do nothing unless overridden by the task implementation } /** * "Execute" the task. * *

    Called after {@link #start(WorkflowModel, TaskModel, WorkflowExecutor)}, if the task * status is not terminal. Can be called more than once. * * @param workflow Workflow for which the task is being started * @param task Instance of the Task * @param workflowExecutor Workflow Executor * @return true, if the execution has changed the task status. return false otherwise. */ public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { return false; } /** * Cancel task execution * * @param workflow Workflow for which the task is being started * @param task Instance of the Task * @param workflowExecutor Workflow Executor */ public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) {} public Optional getEvaluationOffset(TaskModel taskModel, long defaultOffset) { return Optional.empty(); } /** * @return True if the task is supposed to be started asynchronously using internal queues. */ public boolean isAsync() { return false; } /** * @return True to keep task in 'IN_PROGRESS' state, and 'COMPLETE' later by an external * message. */ public boolean isAsyncComplete(TaskModel task) { if (task.getInputData().containsKey("asyncComplete")) { return Optional.ofNullable(task.getInputData().get("asyncComplete")) .map(result -> (Boolean) result) .orElse(false); } else { return Optional.ofNullable(task.getWorkflowTask()) .map(WorkflowTask::isAsyncComplete) .orElse(false); } } /** * @return name of the system task */ public String getTaskType() { return taskType; } /** * Default to true for retrieving tasks when retrieving workflow data. Some cases (e.g. * subworkflows) might not need the tasks at all, and by setting this to false in that case, you * can get a solid performance gain. * * @return true for retrieving tasks when getting workflow */ public boolean isTaskRetrievalRequired() { return true; } @Override public String toString() { return taskType; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/index/NoopIndexDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.index; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.dao.IndexDAO; /** * Dummy implementation of {@link IndexDAO} which does nothing. Nothing is ever indexed, and no * results are ever returned. */ public class NoopIndexDAO implements IndexDAO { @Override public void setup() {} @Override public void indexWorkflow(WorkflowSummary workflowSummary) {} @Override public CompletableFuture asyncIndexWorkflow(WorkflowSummary workflowSummary) { return CompletableFuture.completedFuture(null); } @Override public void indexTask(TaskSummary taskSummary) {} @Override public CompletableFuture asyncIndexTask(TaskSummary taskSummary) { return CompletableFuture.completedFuture(null); } @Override public SearchResult searchWorkflows( String query, String freeText, int start, int count, List sort) { return new SearchResult<>(0, Collections.emptyList()); } @Override public SearchResult searchWorkflowSummary( String query, String freeText, int start, int count, List sort) { return new SearchResult<>(0, Collections.emptyList()); } @Override public SearchResult searchTasks( String query, String freeText, int start, int count, List sort) { return new SearchResult<>(0, Collections.emptyList()); } @Override public SearchResult searchTaskSummary( String query, String freeText, int start, int count, List sort) { return new SearchResult<>(0, Collections.emptyList()); } @Override public void removeWorkflow(String workflowId) {} @Override public CompletableFuture asyncRemoveWorkflow(String workflowId) { return CompletableFuture.completedFuture(null); } @Override public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {} @Override public CompletableFuture asyncUpdateWorkflow( String workflowInstanceId, String[] keys, Object[] values) { return CompletableFuture.completedFuture(null); } @Override public void removeTask(String workflowId, String taskId) {} @Override public CompletableFuture asyncRemoveTask(String workflowId, String taskId) { return CompletableFuture.completedFuture(null); } @Override public void updateTask(String workflowId, String taskId, String[] keys, Object[] values) {} @Override public CompletableFuture asyncUpdateTask( String workflowId, String taskId, String[] keys, Object[] values) { return CompletableFuture.completedFuture(null); } @Override public String get(String workflowInstanceId, String key) { return null; } @Override public void addTaskExecutionLogs(List logs) {} @Override public CompletableFuture asyncAddTaskExecutionLogs(List logs) { return CompletableFuture.completedFuture(null); } @Override public List getTaskExecutionLogs(String taskId) { return Collections.emptyList(); } @Override public void addEventExecution(EventExecution eventExecution) {} @Override public List getEventExecutions(String event) { return Collections.emptyList(); } @Override public CompletableFuture asyncAddEventExecution(EventExecution eventExecution) { return null; } @Override public void addMessage(String queue, Message msg) {} @Override public CompletableFuture asyncAddMessage(String queue, Message message) { return CompletableFuture.completedFuture(null); } @Override public List getMessages(String queue) { return Collections.emptyList(); } @Override public List searchArchivableWorkflows(String indexName, long archiveTtlDays) { return Collections.emptyList(); } @Override public long getWorkflowCount(String query, String freeText) { return 0; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/index/NoopIndexDAOConfiguration.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.index; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.dao.IndexDAO; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.indexing.enabled", havingValue = "false") public class NoopIndexDAOConfiguration { @Bean public IndexDAO noopIndexDAO() { return new NoopIndexDAO(); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/listener/TaskStatusListener.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.listener; import com.netflix.conductor.model.TaskModel; /** * Listener for the Task status change. All methods have default implementation so that * Implementation can choose to override a subset of interested Task statuses. */ public interface TaskStatusListener { default void onTaskScheduled(TaskModel task) {} default void onTaskInProgress(TaskModel task) {} default void onTaskCanceled(TaskModel task) {} default void onTaskFailed(TaskModel task) {} default void onTaskFailedWithTerminalError(TaskModel task) {} default void onTaskCompleted(TaskModel task) {} default void onTaskCompletedWithErrors(TaskModel task) {} default void onTaskTimedOut(TaskModel task) {} default void onTaskSkipped(TaskModel task) {} } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/listener/TaskStatusListenerStub.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.listener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.model.TaskModel; /** Stub listener default implementation */ public class TaskStatusListenerStub implements TaskStatusListener { private static final Logger LOGGER = LoggerFactory.getLogger(TaskStatusListenerStub.class); @Override public void onTaskScheduled(TaskModel task) { LOGGER.debug("Task {} is scheduled", task.getTaskId()); } @Override public void onTaskCanceled(TaskModel task) { LOGGER.debug("Task {} is canceled", task.getTaskId()); } @Override public void onTaskCompleted(TaskModel task) { LOGGER.debug("Task {} is completed", task.getTaskId()); } @Override public void onTaskCompletedWithErrors(TaskModel task) { LOGGER.debug("Task {} is completed with errors", task.getTaskId()); } @Override public void onTaskFailed(TaskModel task) { LOGGER.debug("Task {} is failed", task.getTaskId()); } @Override public void onTaskFailedWithTerminalError(TaskModel task) { LOGGER.debug("Task {} is failed with terminal error", task.getTaskId()); } @Override public void onTaskInProgress(TaskModel task) { LOGGER.debug("Task {} is in-progress", task.getTaskId()); } @Override public void onTaskSkipped(TaskModel task) { LOGGER.debug("Task {} is skipped", task.getTaskId()); } @Override public void onTaskTimedOut(TaskModel task) { LOGGER.debug("Task {} is timed out", task.getTaskId()); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/listener/WorkflowStatusListener.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.listener; import com.netflix.conductor.model.WorkflowModel; /** Listener for the completed and terminated workflows */ public interface WorkflowStatusListener { default void onWorkflowCompletedIfEnabled(WorkflowModel workflow) { if (workflow.getWorkflowDefinition().isWorkflowStatusListenerEnabled()) { onWorkflowCompleted(workflow); } } default void onWorkflowTerminatedIfEnabled(WorkflowModel workflow) { if (workflow.getWorkflowDefinition().isWorkflowStatusListenerEnabled()) { onWorkflowTerminated(workflow); } } default void onWorkflowFinalizedIfEnabled(WorkflowModel workflow) { if (workflow.getWorkflowDefinition().isWorkflowStatusListenerEnabled()) { onWorkflowFinalized(workflow); } } void onWorkflowCompleted(WorkflowModel workflow); void onWorkflowTerminated(WorkflowModel workflow); default void onWorkflowFinalized(WorkflowModel workflow) {} } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/listener/WorkflowStatusListenerStub.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.listener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.model.WorkflowModel; /** Stub listener default implementation */ public class WorkflowStatusListenerStub implements WorkflowStatusListener { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowStatusListenerStub.class); @Override public void onWorkflowCompleted(WorkflowModel workflow) { LOGGER.debug("Workflow {} is completed", workflow.getWorkflowId()); } @Override public void onWorkflowTerminated(WorkflowModel workflow) { LOGGER.debug("Workflow {} is terminated", workflow.getWorkflowId()); } @Override public void onWorkflowFinalized(WorkflowModel workflow) { LOGGER.debug("Workflow {} is finalized", workflow.getWorkflowId()); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/metadata/MetadataMapperService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.metadata; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * Populates metadata definitions within workflow objects. Benefits of loading and populating * metadata definitions upfront could be: * *

      *
    • Immutable definitions within a workflow execution with the added benefit of guaranteeing * consistency at runtime. *
    • Stress is reduced on the storage layer *
    */ @Component public class MetadataMapperService { public static final Logger LOGGER = LoggerFactory.getLogger(MetadataMapperService.class); private final MetadataDAO metadataDAO; public MetadataMapperService(MetadataDAO metadataDAO) { this.metadataDAO = metadataDAO; } public WorkflowDef lookupForWorkflowDefinition(String name, Integer version) { Optional potentialDef = version == null ? lookupLatestWorkflowDefinition(name) : lookupWorkflowDefinition(name, version); // Check if the workflow definition is valid return potentialDef.orElseThrow( () -> { LOGGER.error( "There is no workflow defined with name {} and version {}", name, version); return new NotFoundException( "No such workflow defined. name=%s, version=%s", name, version); }); } @VisibleForTesting Optional lookupWorkflowDefinition(String workflowName, int workflowVersion) { Utils.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name must be specified when searching for a definition"); return metadataDAO.getWorkflowDef(workflowName, workflowVersion); } @VisibleForTesting Optional lookupLatestWorkflowDefinition(String workflowName) { Utils.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name must be specified when searching for a definition"); return metadataDAO.getLatestWorkflowDef(workflowName); } public WorkflowModel populateWorkflowWithDefinitions(WorkflowModel workflow) { Utils.checkNotNull(workflow, "workflow cannot be null"); WorkflowDef workflowDefinition = Optional.ofNullable(workflow.getWorkflowDefinition()) .orElseGet( () -> { WorkflowDef wd = lookupForWorkflowDefinition( workflow.getWorkflowName(), workflow.getWorkflowVersion()); workflow.setWorkflowDefinition(wd); return wd; }); workflowDefinition.collectTasks().forEach(this::populateWorkflowTaskWithDefinition); checkNotEmptyDefinitions(workflowDefinition); return workflow; } public WorkflowDef populateTaskDefinitions(WorkflowDef workflowDefinition) { Utils.checkNotNull(workflowDefinition, "workflowDefinition cannot be null"); workflowDefinition.collectTasks().forEach(this::populateWorkflowTaskWithDefinition); checkNotEmptyDefinitions(workflowDefinition); return workflowDefinition; } private void populateWorkflowTaskWithDefinition(WorkflowTask workflowTask) { Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); if (shouldPopulateTaskDefinition(workflowTask)) { workflowTask.setTaskDefinition(metadataDAO.getTaskDef(workflowTask.getName())); if (workflowTask.getTaskDefinition() == null && workflowTask.getType().equals(TaskType.SIMPLE.name())) { // ad-hoc task def workflowTask.setTaskDefinition(new TaskDef(workflowTask.getName())); } } if (workflowTask.getType().equals(TaskType.SUB_WORKFLOW.name())) { populateVersionForSubWorkflow(workflowTask); } } private void populateVersionForSubWorkflow(WorkflowTask workflowTask) { Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); SubWorkflowParams subworkflowParams = workflowTask.getSubWorkflowParam(); if (subworkflowParams.getVersion() == null) { String subWorkflowName = subworkflowParams.getName(); Integer subWorkflowVersion = metadataDAO .getLatestWorkflowDef(subWorkflowName) .map(WorkflowDef::getVersion) .orElseThrow( () -> { String reason = String.format( "The Task %s defined as a sub-workflow has no workflow definition available ", subWorkflowName); LOGGER.error(reason); return new TerminateWorkflowException(reason); }); subworkflowParams.setVersion(subWorkflowVersion); } } private void checkNotEmptyDefinitions(WorkflowDef workflowDefinition) { Utils.checkNotNull(workflowDefinition, "WorkflowDefinition cannot be null"); // Obtain the names of the tasks with missing definitions Set missingTaskDefinitionNames = workflowDefinition.collectTasks().stream() .filter( workflowTask -> workflowTask.getType().equals(TaskType.SIMPLE.name())) .filter(this::shouldPopulateTaskDefinition) .map(WorkflowTask::getName) .collect(Collectors.toSet()); if (!missingTaskDefinitionNames.isEmpty()) { LOGGER.error( "Cannot find the task definitions for the following tasks used in workflow: {}", missingTaskDefinitionNames); Monitors.recordWorkflowStartError( workflowDefinition.getName(), WorkflowContext.get().getClientApp()); throw new IllegalArgumentException( "Cannot find the task definitions for the following tasks used in workflow: " + missingTaskDefinitionNames); } } public TaskModel populateTaskWithDefinition(TaskModel task) { Utils.checkNotNull(task, "Task cannot be null"); populateWorkflowTaskWithDefinition(task.getWorkflowTask()); return task; } @VisibleForTesting boolean shouldPopulateTaskDefinition(WorkflowTask workflowTask) { Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); Utils.checkNotNull(workflowTask.getType(), "WorkflowTask type cannot be null"); return workflowTask.getTaskDefinition() == null && StringUtils.isNotBlank(workflowTask.getName()); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/operation/StartWorkflowOperation.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.operation; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.event.WorkflowCreationEvent; import com.netflix.conductor.core.event.WorkflowEvaluationEvent; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.metadata.MetadataMapperService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; @Component public class StartWorkflowOperation implements WorkflowOperation { private static final Logger LOGGER = LoggerFactory.getLogger(StartWorkflowOperation.class); private final MetadataMapperService metadataMapperService; private final IDGenerator idGenerator; private final ParametersUtils parametersUtils; private final ExecutionDAOFacade executionDAOFacade; private final ExecutionLockService executionLockService; private final ApplicationEventPublisher eventPublisher; public StartWorkflowOperation( MetadataMapperService metadataMapperService, IDGenerator idGenerator, ParametersUtils parametersUtils, ExecutionDAOFacade executionDAOFacade, ExecutionLockService executionLockService, ApplicationEventPublisher eventPublisher) { this.metadataMapperService = metadataMapperService; this.idGenerator = idGenerator; this.parametersUtils = parametersUtils; this.executionDAOFacade = executionDAOFacade; this.executionLockService = executionLockService; this.eventPublisher = eventPublisher; } @Override public String execute(StartWorkflowInput input) { return startWorkflow(input); } @EventListener(WorkflowCreationEvent.class) public void handleWorkflowCreationEvent(WorkflowCreationEvent workflowCreationEvent) { startWorkflow(workflowCreationEvent.getStartWorkflowInput()); } private String startWorkflow(StartWorkflowInput input) { WorkflowDef workflowDefinition; if (input.getWorkflowDefinition() == null) { workflowDefinition = metadataMapperService.lookupForWorkflowDefinition( input.getName(), input.getVersion()); } else { workflowDefinition = input.getWorkflowDefinition(); } workflowDefinition = metadataMapperService.populateTaskDefinitions(workflowDefinition); // perform validations Map workflowInput = input.getWorkflowInput(); String externalInputPayloadStoragePath = input.getExternalInputPayloadStoragePath(); validateWorkflow(workflowDefinition, workflowInput, externalInputPayloadStoragePath); // Generate ID if it's not present String workflowId = Optional.ofNullable(input.getWorkflowId()).orElseGet(idGenerator::generate); // Persist the Workflow WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); workflow.setCorrelationId(input.getCorrelationId()); workflow.setPriority(input.getPriority() == null ? 0 : input.getPriority()); workflow.setWorkflowDefinition(workflowDefinition); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setParentWorkflowId(input.getParentWorkflowId()); workflow.setParentWorkflowTaskId(input.getParentWorkflowTaskId()); workflow.setOwnerApp(WorkflowContext.get().getClientApp()); workflow.setCreateTime(System.currentTimeMillis()); workflow.setUpdatedBy(null); workflow.setUpdatedTime(null); workflow.setEvent(input.getEvent()); workflow.setTaskToDomain(input.getTaskToDomain()); workflow.setVariables(workflowDefinition.getVariables()); if (workflowInput != null && !workflowInput.isEmpty()) { Map parsedInput = parametersUtils.getWorkflowInput(workflowDefinition, workflowInput); workflow.setInput(parsedInput); } else { workflow.setExternalInputPayloadStoragePath(externalInputPayloadStoragePath); } try { createAndEvaluate(workflow); Monitors.recordWorkflowStartSuccess( workflow.getWorkflowName(), String.valueOf(workflow.getWorkflowVersion()), workflow.getOwnerApp()); return workflowId; } catch (Exception e) { Monitors.recordWorkflowStartError( workflowDefinition.getName(), WorkflowContext.get().getClientApp()); LOGGER.error("Unable to start workflow: {}", workflowDefinition.getName(), e); // It's possible the remove workflow call hits an exception as well, in that case we // want to log both errors to help diagnosis. try { executionDAOFacade.removeWorkflow(workflowId, false); } catch (Exception rwe) { LOGGER.error("Could not remove the workflowId: " + workflowId, rwe); } throw e; } } /* * Acquire and hold the lock till the workflow creation action is completed (in primary and secondary datastores). * This is to ensure that workflow creation action precedes any other action on a given workflow. */ private void createAndEvaluate(WorkflowModel workflow) { if (!executionLockService.acquireLock(workflow.getWorkflowId())) { throw new TransientException("Error acquiring lock when creating workflow: {}"); } try { executionDAOFacade.createWorkflow(workflow); LOGGER.debug( "A new instance of workflow: {} created with id: {}", workflow.getWorkflowName(), workflow.getWorkflowId()); executionDAOFacade.populateWorkflowAndTaskPayloadData(workflow); eventPublisher.publishEvent(new WorkflowEvaluationEvent(workflow)); } finally { executionLockService.releaseLock(workflow.getWorkflowId()); } } /** * Performs validations for starting a workflow * * @throws IllegalArgumentException if the validation fails. */ private void validateWorkflow( WorkflowDef workflowDef, Map workflowInput, String externalStoragePath) { // Check if the input to the workflow is not null if (workflowInput == null && StringUtils.isBlank(externalStoragePath)) { LOGGER.error("The input for the workflow '{}' cannot be NULL", workflowDef.getName()); Monitors.recordWorkflowStartError( workflowDef.getName(), WorkflowContext.get().getClientApp()); throw new IllegalArgumentException("NULL input passed when starting workflow"); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/operation/WorkflowOperation.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.operation; public interface WorkflowOperation { R execute(T input); } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.reconciliation; import java.util.List; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; /** * Periodically polls all running workflows in the system and evaluates them for timeouts and/or * maintain consistency. */ @Component @ConditionalOnProperty( name = "conductor.workflow-reconciler.enabled", havingValue = "true", matchIfMissing = true) public class WorkflowReconciler extends LifecycleAwareComponent { private final WorkflowSweeper workflowSweeper; private final QueueDAO queueDAO; private final int sweeperThreadCount; private final int sweeperWorkflowPollTimeout; private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowReconciler.class); public WorkflowReconciler( WorkflowSweeper workflowSweeper, QueueDAO queueDAO, ConductorProperties properties) { this.workflowSweeper = workflowSweeper; this.queueDAO = queueDAO; this.sweeperThreadCount = properties.getSweeperThreadCount(); this.sweeperWorkflowPollTimeout = (int) properties.getSweeperWorkflowPollTimeout().toMillis(); LOGGER.info( "WorkflowReconciler initialized with {} sweeper threads", properties.getSweeperThreadCount()); } @Scheduled( fixedDelayString = "${conductor.sweep-frequency.millis:500}", initialDelayString = "${conductor.sweep-frequency.millis:500}") public void pollAndSweep() { try { if (!isRunning()) { LOGGER.debug("Component stopped, skip workflow sweep"); } else { List workflowIds = queueDAO.pop(DECIDER_QUEUE, sweeperThreadCount, sweeperWorkflowPollTimeout); if (workflowIds != null) { // wait for all workflow ids to be "swept" CompletableFuture.allOf( workflowIds.stream() .map(workflowSweeper::sweepAsync) .toArray(CompletableFuture[]::new)) .get(); LOGGER.debug( "Sweeper processed {} from the decider queue", String.join(",", workflowIds)); } // NOTE: Disabling the sweeper implicitly disables this metric. recordQueueDepth(); } } catch (Exception e) { Monitors.error(WorkflowReconciler.class.getSimpleName(), "poll"); LOGGER.error("Error when polling for workflows", e); if (e instanceof InterruptedException) { // Restore interrupted state... Thread.currentThread().interrupt(); } } } private void recordQueueDepth() { int currentQueueSize = queueDAO.getSize(DECIDER_QUEUE); Monitors.recordGauge(DECIDER_QUEUE, currentQueueSize); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowRepairService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.reconciliation; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** * A helper service that tries to keep ExecutionDAO and QueueDAO in sync, based on the task or * workflow state. * *

    This service expects that the underlying Queueing layer implements {@link * QueueDAO#containsMessage(String, String)} method. This can be controlled with * conductor.workflow-repair-service.enabled property. */ @Service @ConditionalOnProperty(name = "conductor.workflow-repair-service.enabled", havingValue = "true") public class WorkflowRepairService { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowRepairService.class); private final ExecutionDAO executionDAO; private final QueueDAO queueDAO; private final ConductorProperties properties; private SystemTaskRegistry systemTaskRegistry; /* For system task -> Verify the task isAsync() and not isAsyncComplete() or isAsyncComplete() in SCHEDULED state, and in SCHEDULED or IN_PROGRESS state. (Example: SUB_WORKFLOW tasks in SCHEDULED state) For simple task -> Verify the task is in SCHEDULED state. */ private final Predicate isTaskRepairable = task -> { if (systemTaskRegistry.isSystemTask(task.getTaskType())) { // If system task WorkflowSystemTask workflowSystemTask = systemTaskRegistry.get(task.getTaskType()); return workflowSystemTask.isAsync() && (!workflowSystemTask.isAsyncComplete(task) || (workflowSystemTask.isAsyncComplete(task) && task.getStatus() == TaskModel.Status.SCHEDULED)) && (task.getStatus() == TaskModel.Status.IN_PROGRESS || task.getStatus() == TaskModel.Status.SCHEDULED); } else { // Else if simple task return task.getStatus() == TaskModel.Status.SCHEDULED; } }; public WorkflowRepairService( ExecutionDAO executionDAO, QueueDAO queueDAO, ConductorProperties properties, SystemTaskRegistry systemTaskRegistry) { this.executionDAO = executionDAO; this.queueDAO = queueDAO; this.properties = properties; this.systemTaskRegistry = systemTaskRegistry; LOGGER.info("WorkflowRepairService Initialized"); } /** * Verify and repair if the workflowId exists in deciderQueue, and then if each scheduled task * has relevant message in the queue. */ public boolean verifyAndRepairWorkflow(String workflowId, boolean includeTasks) { WorkflowModel workflow = executionDAO.getWorkflow(workflowId, includeTasks); AtomicBoolean repaired = new AtomicBoolean(false); repaired.set(verifyAndRepairDeciderQueue(workflow)); if (includeTasks) { workflow.getTasks().forEach(task -> repaired.set(verifyAndRepairTask(task))); } return repaired.get(); } /** Verify and repair tasks in a workflow. */ public void verifyAndRepairWorkflowTasks(String workflowId) { WorkflowModel workflow = Optional.ofNullable(executionDAO.getWorkflow(workflowId, true)) .orElseThrow( () -> new NotFoundException( "Could not find workflow: " + workflowId)); verifyAndRepairWorkflowTasks(workflow); } /** Verify and repair tasks in a workflow. */ public void verifyAndRepairWorkflowTasks(WorkflowModel workflow) { workflow.getTasks().forEach(this::verifyAndRepairTask); // repair the parent workflow if needed verifyAndRepairWorkflow(workflow.getParentWorkflowId()); } /** * Verify and fix if Workflow decider queue contains this workflowId. * * @return true - if the workflow was queued for repair */ private boolean verifyAndRepairDeciderQueue(WorkflowModel workflow) { if (!workflow.getStatus().isTerminal()) { return verifyAndRepairWorkflow(workflow.getWorkflowId()); } return false; } /** * Verify if ExecutionDAO and QueueDAO agree for the provided task. * * @param task the task to be repaired * @return true - if the task was queued for repair */ @VisibleForTesting boolean verifyAndRepairTask(TaskModel task) { if (isTaskRepairable.test(task)) { // Ensure QueueDAO contains this taskId String taskQueueName = QueueUtils.getQueueName(task); if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); LOGGER.info( "Task {} in workflow {} re-queued for repairs", task.getTaskId(), task.getWorkflowInstanceId()); Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); return true; } } else if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) && task.getStatus() == TaskModel.Status.IN_PROGRESS) { WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); if (subWorkflow.getStatus().isTerminal()) { LOGGER.info( "Repairing sub workflow task {} for sub workflow {} in workflow {}", task.getTaskId(), task.getSubWorkflowId(), task.getWorkflowInstanceId()); repairSubWorkflowTask(task, subWorkflow); return true; } } return false; } private boolean verifyAndRepairWorkflow(String workflowId) { if (StringUtils.isNotEmpty(workflowId)) { String queueName = Utils.DECIDER_QUEUE; if (!queueDAO.containsMessage(queueName, workflowId)) { queueDAO.push( queueName, workflowId, properties.getWorkflowOffsetTimeout().getSeconds()); LOGGER.info("Workflow {} re-queued for repairs", workflowId); Monitors.recordQueueMessageRepushFromRepairService(queueName); return true; } return false; } return false; } private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { switch (subWorkflow.getStatus()) { case COMPLETED: task.setStatus(TaskModel.Status.COMPLETED); break; case FAILED: task.setStatus(TaskModel.Status.FAILED); break; case TERMINATED: task.setStatus(TaskModel.Status.CANCELED); break; case TIMED_OUT: task.setStatus(TaskModel.Status.TIMED_OUT); break; } task.addOutput(subWorkflow.getOutput()); executionDAO.updateTask(task); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.reconciliation; import java.time.Instant; import java.util.Optional; import java.util.Random; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; @Component public class WorkflowSweeper { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowSweeper.class); private final ConductorProperties properties; private final WorkflowExecutor workflowExecutor; private final WorkflowRepairService workflowRepairService; private final QueueDAO queueDAO; private final ExecutionDAOFacade executionDAOFacade; private static final String CLASS_NAME = WorkflowSweeper.class.getSimpleName(); @Autowired public WorkflowSweeper( WorkflowExecutor workflowExecutor, Optional workflowRepairService, ConductorProperties properties, QueueDAO queueDAO, ExecutionDAOFacade executionDAOFacade) { this.properties = properties; this.queueDAO = queueDAO; this.workflowExecutor = workflowExecutor; this.executionDAOFacade = executionDAOFacade; this.workflowRepairService = workflowRepairService.orElse(null); LOGGER.info("WorkflowSweeper initialized."); } @Async(SWEEPER_EXECUTOR_NAME) public CompletableFuture sweepAsync(String workflowId) { sweep(workflowId); return CompletableFuture.completedFuture(null); } public void sweep(String workflowId) { WorkflowModel workflow = null; try { WorkflowContext workflowContext = new WorkflowContext(properties.getAppId()); WorkflowContext.set(workflowContext); LOGGER.debug("Running sweeper for workflow {}", workflowId); workflow = executionDAOFacade.getWorkflowModel(workflowId, true); if (workflowRepairService != null) { // Verify and repair tasks in the workflow. workflowRepairService.verifyAndRepairWorkflowTasks(workflow); } workflow = workflowExecutor.decideWithLock(workflow); if (workflow != null && workflow.getStatus().isTerminal()) { queueDAO.remove(DECIDER_QUEUE, workflowId); return; } } catch (NotFoundException nfe) { queueDAO.remove(DECIDER_QUEUE, workflowId); LOGGER.info( "Workflow NOT found for id:{}. Removed it from decider queue", workflowId, nfe); return; } catch (Exception e) { Monitors.error(CLASS_NAME, "sweep"); LOGGER.error("Error running sweep for " + workflowId, e); } long workflowOffsetTimeout = workflowOffsetWithJitter(properties.getWorkflowOffsetTimeout().getSeconds()); if (workflow != null) { long startTime = Instant.now().toEpochMilli(); unack(workflow, workflowOffsetTimeout); long endTime = Instant.now().toEpochMilli(); Monitors.recordUnackTime(workflow.getWorkflowName(), endTime - startTime); } else { LOGGER.warn( "Workflow with {} id can not be found. Attempting to unack using the id", workflowId); queueDAO.setUnackTimeout(DECIDER_QUEUE, workflowId, workflowOffsetTimeout * 1000); } } @VisibleForTesting void unack(WorkflowModel workflowModel, long workflowOffsetTimeout) { long postponeDurationSeconds = 0; for (TaskModel taskModel : workflowModel.getTasks()) { if (taskModel.getStatus() == Status.IN_PROGRESS) { if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_WAIT)) { if (taskModel.getWaitTimeout() == 0) { postponeDurationSeconds = workflowOffsetTimeout; } else { long deltaInSeconds = (taskModel.getWaitTimeout() - System.currentTimeMillis()) / 1000; postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; } } else if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_HUMAN)) { postponeDurationSeconds = workflowOffsetTimeout; } else { postponeDurationSeconds = (taskModel.getResponseTimeoutSeconds() != 0) ? taskModel.getResponseTimeoutSeconds() + 1 : workflowOffsetTimeout; } break; } else if (taskModel.getStatus() == Status.SCHEDULED) { Optional taskDefinition = taskModel.getTaskDefinition(); if (taskDefinition.isPresent()) { TaskDef taskDef = taskDefinition.get(); if (taskDef.getPollTimeoutSeconds() != null && taskDef.getPollTimeoutSeconds() != 0) { postponeDurationSeconds = taskDef.getPollTimeoutSeconds() + 1; } else { postponeDurationSeconds = (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1 : workflowOffsetTimeout; } } else { postponeDurationSeconds = (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1 : workflowOffsetTimeout; } break; } } queueDAO.setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), postponeDurationSeconds * 1000); } /** * jitter will be +- (1/3) workflowOffsetTimeout for example, if workflowOffsetTimeout is 45 * seconds, this function returns values between [30-60] seconds * * @param workflowOffsetTimeout * @return */ @VisibleForTesting long workflowOffsetWithJitter(long workflowOffsetTimeout) { long range = workflowOffsetTimeout / 3; long jitter = new Random().nextInt((int) (2 * range + 1)) - range; return workflowOffsetTimeout + jitter; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/storage/DummyPayloadStorage.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.storage; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.util.UUID; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.fasterxml.jackson.databind.ObjectMapper; /** * A dummy implementation of {@link ExternalPayloadStorage} used when no external payload is * configured */ public class DummyPayloadStorage implements ExternalPayloadStorage { private static final Logger LOGGER = LoggerFactory.getLogger(DummyPayloadStorage.class); private ObjectMapper objectMapper; private File payloadDir; public DummyPayloadStorage() { try { this.objectMapper = new ObjectMapper(); this.payloadDir = Files.createTempDirectory("payloads").toFile(); LOGGER.info( "{} initialized in directory: {}", this.getClass().getSimpleName(), payloadDir.getAbsolutePath()); } catch (IOException ioException) { LOGGER.error( "Exception encountered while creating payloads directory : {}", ioException.getMessage()); } } @Override public ExternalStorageLocation getLocation( Operation operation, PayloadType payloadType, String path) { ExternalStorageLocation location = new ExternalStorageLocation(); location.setPath(path + UUID.randomUUID() + ".json"); return location; } @Override public void upload(String path, InputStream payload, long payloadSize) { File file = new File(payloadDir, path); String filePath = file.getAbsolutePath(); try { if (!file.exists() && file.createNewFile()) { LOGGER.debug("Created file: {}", filePath); } IOUtils.copy(payload, new FileOutputStream(file)); LOGGER.debug("Written to {}", filePath); } catch (IOException e) { // just handle this exception here and return empty map so that test will fail in case // this exception is thrown LOGGER.error("Error writing to {}", filePath); } finally { try { if (payload != null) { payload.close(); } } catch (IOException e) { LOGGER.warn("Unable to close input stream when writing to file"); } } } @Override public InputStream download(String path) { try { LOGGER.debug("Reading from {}", path); return new FileInputStream(new File(payloadDir, path)); } catch (IOException e) { LOGGER.error("Error reading {}", path, e); return null; } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/sync/Lock.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.sync; import java.util.concurrent.TimeUnit; /** * Interface implemented by a distributed lock client. * *

    A typical usage: * * 

     *   if (acquireLock(workflowId, 5, TimeUnit.MILLISECONDS)) {
 *      [load and execute workflow....]
 *      ExecutionDAO.updateWorkflow(workflow);  //use optimistic locking
 *   } finally {
 *     releaseLock(workflowId)
 *   }
 *
    */ public interface Lock { /** * Acquires a re-entrant lock on lockId, blocks indefinitely on lockId until it succeeds * * @param lockId resource to lock on */ void acquireLock(String lockId); /** * Acquires a re-entrant lock on lockId, blocks for timeToTry duration before giving up * * @param lockId resource to lock on * @param timeToTry blocks up to timeToTry duration in attempt to acquire the lock * @param unit time unit * @return true, if successfully acquired */ boolean acquireLock(String lockId, long timeToTry, TimeUnit unit); /** * Acquires a re-entrant lock on lockId with provided leaseTime duration. Blocks for timeToTry * duration before giving up * * @param lockId resource to lock on * @param timeToTry blocks up to timeToTry duration in attempt to acquire the lock * @param leaseTime Lock lease expiration duration. * @param unit time unit * @return true, if successfully acquired */ boolean acquireLock(String lockId, long timeToTry, long leaseTime, TimeUnit unit); /** * Release a previously acquired lock * * @param lockId resource to lock on */ void releaseLock(String lockId); /** * Explicitly cleanup lock resources, if releasing it wouldn't do so. * * @param lockId resource to lock on */ void deleteLock(String lockId); } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/sync/local/LocalOnlyLock.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.sync.local; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.sync.Lock; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; public class LocalOnlyLock implements Lock { private static final Logger LOGGER = LoggerFactory.getLogger(LocalOnlyLock.class); private static final CacheLoader LOADER = key -> new ReentrantLock(true); private static final ConcurrentHashMap> SCHEDULEDFUTURES = new ConcurrentHashMap<>(); private static final LoadingCache LOCKIDTOSEMAPHOREMAP = Caffeine.newBuilder().build(LOADER); private static final ThreadGroup THREAD_GROUP = new ThreadGroup("LocalOnlyLock-scheduler"); private static final ThreadFactory THREAD_FACTORY = runnable -> new Thread(THREAD_GROUP, runnable); private static final ScheduledExecutorService SCHEDULER = Executors.newScheduledThreadPool(1, THREAD_FACTORY); @Override public void acquireLock(String lockId) { LOGGER.trace("Locking {}", lockId); LOCKIDTOSEMAPHOREMAP.get(lockId).lock(); } @Override public boolean acquireLock(String lockId, long timeToTry, TimeUnit unit) { try { LOGGER.trace("Locking {} with timeout {} {}", lockId, timeToTry, unit); return LOCKIDTOSEMAPHOREMAP.get(lockId).tryLock(timeToTry, unit); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); } } @Override public boolean acquireLock(String lockId, long timeToTry, long leaseTime, TimeUnit unit) { LOGGER.trace( "Locking {} with timeout {} {} for {} {}", lockId, timeToTry, unit, leaseTime, unit); if (acquireLock(lockId, timeToTry, unit)) { LOGGER.trace("Releasing {} automatically after {} {}", lockId, leaseTime, unit); SCHEDULEDFUTURES.put( lockId, SCHEDULER.schedule(() -> deleteLock(lockId), leaseTime, unit)); return true; } return false; } private void removeLeaseExpirationJob(String lockId) { ScheduledFuture schedFuture = SCHEDULEDFUTURES.get(lockId); if (schedFuture != null && schedFuture.cancel(false)) { SCHEDULEDFUTURES.remove(lockId); LOGGER.trace("lockId {} removed from lease expiration job", lockId); } } @Override public void releaseLock(String lockId) { // Synchronized to prevent race condition between semaphore check and actual release synchronized (LOCKIDTOSEMAPHOREMAP) { if (LOCKIDTOSEMAPHOREMAP.getIfPresent(lockId) == null) { return; } LOGGER.trace("Releasing {}", lockId); LOCKIDTOSEMAPHOREMAP.get(lockId).unlock(); removeLeaseExpirationJob(lockId); } } @Override public void deleteLock(String lockId) { LOGGER.trace("Deleting {}", lockId); LOCKIDTOSEMAPHOREMAP.invalidate(lockId); } @VisibleForTesting LoadingCache cache() { return LOCKIDTOSEMAPHOREMAP; } @VisibleForTesting ConcurrentHashMap> scheduledFutures() { return SCHEDULEDFUTURES; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/sync/local/LocalOnlyLockConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.sync.local; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.sync.Lock; @Configuration @ConditionalOnProperty(name = "conductor.workflow-execution-lock.type", havingValue = "local_only") public class LocalOnlyLockConfiguration { @Bean public Lock provideLock() { return new LocalOnlyLock(); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/sync/noop/NoopLock.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.sync.noop; import java.util.concurrent.TimeUnit; import com.netflix.conductor.core.sync.Lock; public class NoopLock implements Lock { @Override public void acquireLock(String lockId) {} @Override public boolean acquireLock(String lockId, long timeToTry, TimeUnit unit) { return true; } @Override public boolean acquireLock(String lockId, long timeToTry, long leaseTime, TimeUnit unit) { return true; } @Override public void releaseLock(String lockId) {} @Override public void deleteLock(String lockId) {} } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/DateTimeUtils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.text.ParseException; import java.time.Duration; import java.util.Date; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.time.DateUtils; public class DateTimeUtils { private static final String[] patterns = new String[] {"yyyy-MM-dd HH:mm", "yyyy-MM-dd HH:mm z", "yyyy-MM-dd"}; public static Duration parseDuration(String text) { Matcher m = Pattern.compile( "\\s*(?:(\\d+)\\s*(?:days?|d))?" + "\\s*(?:(\\d+)\\s*(?:hours?|hrs?|h))?" + "\\s*(?:(\\d+)\\s*(?:minutes?|mins?|m))?" + "\\s*(?:(\\d+)\\s*(?:seconds?|secs?|s))?" + "\\s*", Pattern.CASE_INSENSITIVE) .matcher(text); if (!m.matches()) throw new IllegalArgumentException("Not valid duration: " + text); int days = (m.start(1) == -1 ? 0 : Integer.parseInt(m.group(1))); int hours = (m.start(2) == -1 ? 0 : Integer.parseInt(m.group(2))); int mins = (m.start(3) == -1 ? 0 : Integer.parseInt(m.group(3))); int secs = (m.start(4) == -1 ? 0 : Integer.parseInt(m.group(4))); return Duration.ofSeconds((days * 86400) + (hours * 60L + mins) * 60L + secs); } public static Date parseDate(String date) throws ParseException { return DateUtils.parseDate(date, patterns); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; /** Provides utility functions to upload and download payloads to {@link ExternalPayloadStorage} */ @Component public class ExternalPayloadStorageUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ExternalPayloadStorageUtils.class); private final ExternalPayloadStorage externalPayloadStorage; private final ConductorProperties properties; private final ObjectMapper objectMapper; public ExternalPayloadStorageUtils( ExternalPayloadStorage externalPayloadStorage, ConductorProperties properties, ObjectMapper objectMapper) { this.externalPayloadStorage = externalPayloadStorage; this.properties = properties; this.objectMapper = objectMapper; } /** * Download the payload from the given path. * * @param path the relative path of the payload in the {@link ExternalPayloadStorage} * @return the payload object * @throws NonTransientException in case of JSON parsing errors or download errors */ @SuppressWarnings("unchecked") public Map downloadPayload(String path) { try (InputStream inputStream = externalPayloadStorage.download(path)) { return objectMapper.readValue( IOUtils.toString(inputStream, StandardCharsets.UTF_8), Map.class); } catch (TransientException te) { throw te; } catch (Exception e) { LOGGER.error("Unable to download payload from external storage path: {}", path, e); throw new NonTransientException( "Unable to download payload from external storage path: " + path, e); } } /** * Verify the payload size and upload to external storage if necessary. * * @param entity the task or workflow for which the payload is to be verified and uploaded * @param payloadType the {@link PayloadType} of the payload * @param {@link TaskModel} or {@link WorkflowModel} * @throws NonTransientException in case of JSON parsing errors or upload errors * @throws TerminateWorkflowException if the payload size is bigger than permissible limit as * per {@link ConductorProperties} */ public void verifyAndUpload(T entity, PayloadType payloadType) { if (!shouldUpload(entity, payloadType)) return; long threshold = 0L; long maxThreshold = 0L; Map payload = new HashMap<>(); String workflowId = ""; switch (payloadType) { case TASK_INPUT: threshold = properties.getTaskInputPayloadSizeThreshold().toKilobytes(); maxThreshold = properties.getMaxTaskInputPayloadSizeThreshold().toKilobytes(); payload = ((TaskModel) entity).getInputData(); workflowId = ((TaskModel) entity).getWorkflowInstanceId(); break; case TASK_OUTPUT: threshold = properties.getTaskOutputPayloadSizeThreshold().toKilobytes(); maxThreshold = properties.getMaxTaskOutputPayloadSizeThreshold().toKilobytes(); payload = ((TaskModel) entity).getOutputData(); workflowId = ((TaskModel) entity).getWorkflowInstanceId(); break; case WORKFLOW_INPUT: threshold = properties.getWorkflowInputPayloadSizeThreshold().toKilobytes(); maxThreshold = properties.getMaxWorkflowInputPayloadSizeThreshold().toKilobytes(); payload = ((WorkflowModel) entity).getInput(); workflowId = ((WorkflowModel) entity).getWorkflowId(); break; case WORKFLOW_OUTPUT: threshold = properties.getWorkflowOutputPayloadSizeThreshold().toKilobytes(); maxThreshold = properties.getMaxWorkflowOutputPayloadSizeThreshold().toKilobytes(); payload = ((WorkflowModel) entity).getOutput(); workflowId = ((WorkflowModel) entity).getWorkflowId(); break; } try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { objectMapper.writeValue(byteArrayOutputStream, payload); byte[] payloadBytes = byteArrayOutputStream.toByteArray(); long payloadSize = payloadBytes.length; final long maxThresholdInBytes = maxThreshold * 1024; if (payloadSize > maxThresholdInBytes) { if (entity instanceof TaskModel) { String errorMsg = String.format( "The payload size: %d of task: %s in workflow: %s is greater than the permissible limit: %d bytes", payloadSize, ((TaskModel) entity).getTaskId(), ((TaskModel) entity).getWorkflowInstanceId(), maxThresholdInBytes); failTask(((TaskModel) entity), payloadType, errorMsg); } else { String errorMsg = String.format( "The payload size: %d of workflow: %s is greater than the permissible limit: %d bytes", payloadSize, ((WorkflowModel) entity).getWorkflowId(), maxThresholdInBytes); failWorkflow(((WorkflowModel) entity), payloadType, errorMsg); } } else if (payloadSize > threshold * 1024) { String externalInputPayloadStoragePath, externalOutputPayloadStoragePath; switch (payloadType) { case TASK_INPUT: externalInputPayloadStoragePath = uploadHelper(payloadBytes, payloadSize, PayloadType.TASK_INPUT); ((TaskModel) entity).externalizeInput(externalInputPayloadStoragePath); Monitors.recordExternalPayloadStorageUsage( ((TaskModel) entity).getTaskDefName(), ExternalPayloadStorage.Operation.WRITE.toString(), PayloadType.TASK_INPUT.toString()); break; case TASK_OUTPUT: externalOutputPayloadStoragePath = uploadHelper(payloadBytes, payloadSize, PayloadType.TASK_OUTPUT); ((TaskModel) entity).externalizeOutput(externalOutputPayloadStoragePath); Monitors.recordExternalPayloadStorageUsage( ((TaskModel) entity).getTaskDefName(), ExternalPayloadStorage.Operation.WRITE.toString(), PayloadType.TASK_OUTPUT.toString()); break; case WORKFLOW_INPUT: externalInputPayloadStoragePath = uploadHelper(payloadBytes, payloadSize, PayloadType.WORKFLOW_INPUT); ((WorkflowModel) entity).externalizeInput(externalInputPayloadStoragePath); Monitors.recordExternalPayloadStorageUsage( ((WorkflowModel) entity).getWorkflowName(), ExternalPayloadStorage.Operation.WRITE.toString(), PayloadType.WORKFLOW_INPUT.toString()); break; case WORKFLOW_OUTPUT: externalOutputPayloadStoragePath = uploadHelper( payloadBytes, payloadSize, PayloadType.WORKFLOW_OUTPUT); ((WorkflowModel) entity) .externalizeOutput(externalOutputPayloadStoragePath); Monitors.recordExternalPayloadStorageUsage( ((WorkflowModel) entity).getWorkflowName(), ExternalPayloadStorage.Operation.WRITE.toString(), PayloadType.WORKFLOW_OUTPUT.toString()); break; } } } catch (TransientException | TerminateWorkflowException te) { throw te; } catch (Exception e) { LOGGER.error( "Unable to upload payload to external storage for workflow: {}", workflowId, e); throw new NonTransientException( "Unable to upload payload to external storage for workflow: " + workflowId, e); } } @VisibleForTesting String uploadHelper( byte[] payloadBytes, long payloadSize, ExternalPayloadStorage.PayloadType payloadType) { ExternalStorageLocation location = externalPayloadStorage.getLocation( ExternalPayloadStorage.Operation.WRITE, payloadType, "", payloadBytes); externalPayloadStorage.upload( location.getPath(), new ByteArrayInputStream(payloadBytes), payloadSize); return location.getPath(); } @VisibleForTesting void failTask(TaskModel task, PayloadType payloadType, String errorMsg) { LOGGER.error(errorMsg); task.setReasonForIncompletion(errorMsg); task.setStatus(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); if (payloadType == PayloadType.TASK_INPUT) { task.setInputData(new HashMap<>()); } else { task.setOutputData(new HashMap<>()); } } @VisibleForTesting void failWorkflow(WorkflowModel workflow, PayloadType payloadType, String errorMsg) { LOGGER.error(errorMsg); if (payloadType == PayloadType.WORKFLOW_INPUT) { workflow.setInput(new HashMap<>()); } else { workflow.setOutput(new HashMap<>()); } throw new TerminateWorkflowException(errorMsg); } @VisibleForTesting boolean shouldUpload(T entity, PayloadType payloadType) { if (entity instanceof TaskModel) { TaskModel taskModel = (TaskModel) entity; if (payloadType == PayloadType.TASK_INPUT) { return !taskModel.getRawInputData().isEmpty(); } else { return !taskModel.getRawOutputData().isEmpty(); } } else { WorkflowModel workflowModel = (WorkflowModel) entity; if (payloadType == PayloadType.WORKFLOW_INPUT) { return !workflowModel.getRawInput().isEmpty(); } else { return !workflowModel.getRawOutput().isEmpty(); } } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/IDGenerator.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.UUID; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @Component @ConditionalOnProperty( name = "conductor.id.generator", havingValue = "default", matchIfMissing = true) /** * ID Generator used by Conductor Note on overriding the ID Generator: The default ID generator uses * UUID v4 as the ID format. By overriding this class it is possible to use different scheme for ID * generation. However, this is not normal and should only be done after very careful consideration. * *

    Please note, if you use Cassandra persistence, the schema uses UUID as the column type and the * IDs have to be valid UUIDs supported by Cassandra. */ public class IDGenerator { public IDGenerator() {} public String generate() { return UUID.randomUUID().toString(); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/JsonUtils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.List; import java.util.Map; import org.springframework.stereotype.Component; import com.fasterxml.jackson.databind.ObjectMapper; /** This class contains utility functions for parsing/expanding JSON. */ @SuppressWarnings("unchecked") @Component public class JsonUtils { private final ObjectMapper objectMapper; public JsonUtils(ObjectMapper objectMapper) { this.objectMapper = objectMapper; } /** * Expands a JSON object into a java object * * @param input the object to be expanded * @return the expanded object containing java types like {@link Map} and {@link List} */ public Object expand(Object input) { if (input instanceof List) { expandList((List) input); return input; } else if (input instanceof Map) { expandMap((Map) input); return input; } else if (input instanceof String) { return getJson((String) input); } else { return input; } } private void expandList(List input) { for (Object value : input) { if (value instanceof String) { if (isJsonString(value.toString())) { value = getJson(value.toString()); } } else if (value instanceof Map) { expandMap((Map) value); } else if (value instanceof List) { expandList((List) value); } } } private void expandMap(Map input) { for (Map.Entry entry : input.entrySet()) { Object value = entry.getValue(); if (value instanceof String) { if (isJsonString(value.toString())) { entry.setValue(getJson(value.toString())); } } else if (value instanceof Map) { expandMap((Map) value); } else if (value instanceof List) { expandList((List) value); } } } /** * Used to obtain a JSONified object from a string * * @param jsonAsString the json object represented in string form * @return the JSONified object representation if the input is a valid json string if the input * is not a valid json string, it will be returned as-is and no exception is thrown */ private Object getJson(String jsonAsString) { try { return objectMapper.readValue(jsonAsString, Object.class); } catch (Exception e) { return jsonAsString; } } private boolean isJsonString(String jsonAsString) { jsonAsString = jsonAsString.trim(); return jsonAsString.startsWith("{") || jsonAsString.startsWith("["); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.utils.EnvUtils; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; /** Used to parse and resolve the JSONPath bindings in the workflow and task definitions. */ @Component public class ParametersUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ParametersUtils.class); private static final Pattern PATTERN = Pattern.compile( "(?=(?> map = new TypeReference<>() {}; public ParametersUtils(ObjectMapper objectMapper) { this.objectMapper = objectMapper; } public Map getTaskInput( Map inputParams, WorkflowModel workflow, TaskDef taskDefinition, String taskId) { if (workflow.getWorkflowDefinition().getSchemaVersion() > 1) { return getTaskInputV2(inputParams, workflow, taskId, taskDefinition); } return getTaskInputV1(workflow, inputParams); } public Map getTaskInputV2( Map input, WorkflowModel workflow, String taskId, TaskDef taskDefinition) { Map inputParams; if (input != null) { inputParams = clone(input); } else { inputParams = new HashMap<>(); } if (taskDefinition != null && taskDefinition.getInputTemplate() != null) { clone(taskDefinition.getInputTemplate()).forEach(inputParams::putIfAbsent); } Map> inputMap = new HashMap<>(); Map workflowParams = new HashMap<>(); workflowParams.put("input", workflow.getInput()); workflowParams.put("output", workflow.getOutput()); workflowParams.put("status", workflow.getStatus()); workflowParams.put("workflowId", workflow.getWorkflowId()); workflowParams.put("parentWorkflowId", workflow.getParentWorkflowId()); workflowParams.put("parentWorkflowTaskId", workflow.getParentWorkflowTaskId()); workflowParams.put("workflowType", workflow.getWorkflowName()); workflowParams.put("version", workflow.getWorkflowVersion()); workflowParams.put("correlationId", workflow.getCorrelationId()); workflowParams.put("reasonForIncompletion", workflow.getReasonForIncompletion()); workflowParams.put("schemaVersion", workflow.getWorkflowDefinition().getSchemaVersion()); workflowParams.put("variables", workflow.getVariables()); inputMap.put("workflow", workflowParams); // For new workflow being started the list of tasks will be empty workflow.getTasks().stream() .map(TaskModel::getReferenceTaskName) .map(workflow::getTaskByRefName) .forEach( task -> { Map taskParams = new HashMap<>(); taskParams.put("input", task.getInputData()); taskParams.put("output", task.getOutputData()); taskParams.put("taskType", task.getTaskType()); if (task.getStatus() != null) { taskParams.put("status", task.getStatus().toString()); } taskParams.put("referenceTaskName", task.getReferenceTaskName()); taskParams.put("retryCount", task.getRetryCount()); taskParams.put("correlationId", task.getCorrelationId()); taskParams.put("pollCount", task.getPollCount()); taskParams.put("taskDefName", task.getTaskDefName()); taskParams.put("scheduledTime", task.getScheduledTime()); taskParams.put("startTime", task.getStartTime()); taskParams.put("endTime", task.getEndTime()); taskParams.put("workflowInstanceId", task.getWorkflowInstanceId()); taskParams.put("taskId", task.getTaskId()); taskParams.put( "reasonForIncompletion", task.getReasonForIncompletion()); taskParams.put("callbackAfterSeconds", task.getCallbackAfterSeconds()); taskParams.put("workerId", task.getWorkerId()); taskParams.put("iteration", task.getIteration()); inputMap.put( task.isLoopOverTask() ? TaskUtils.removeIterationFromTaskRefName( task.getReferenceTaskName()) : task.getReferenceTaskName(), taskParams); }); Configuration option = Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); DocumentContext documentContext = JsonPath.parse(inputMap, option); Map replacedTaskInput = replace(inputParams, documentContext, taskId); if (taskDefinition != null && taskDefinition.getInputTemplate() != null) { // If input for a given key resolves to null, try replacing it with one from // inputTemplate, if it exists. replacedTaskInput.replaceAll( (key, value) -> (value == null) ? taskDefinition.getInputTemplate().get(key) : value); } return replacedTaskInput; } // deep clone using json - POJO private Map clone(Map inputTemplate) { try { byte[] bytes = objectMapper.writeValueAsBytes(inputTemplate); return objectMapper.readValue(bytes, map); } catch (IOException e) { throw new RuntimeException("Unable to clone input params", e); } } public Map replace(Map input, Object json) { Object doc; if (json instanceof String) { doc = JsonPath.parse(json.toString()); } else { doc = json; } Configuration option = Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); DocumentContext documentContext = JsonPath.parse(doc, option); return replace(input, documentContext, null); } public Object replace(String paramString) { Configuration option = Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); DocumentContext documentContext = JsonPath.parse(Collections.emptyMap(), option); return replaceVariables(paramString, documentContext, null); } @SuppressWarnings("unchecked") private Map replace( Map input, DocumentContext documentContext, String taskId) { Map result = new HashMap<>(); for (Entry e : input.entrySet()) { Object newValue; Object value = e.getValue(); if (value instanceof String) { newValue = replaceVariables(value.toString(), documentContext, taskId); } else if (value instanceof Map) { // recursive call newValue = replace((Map) value, documentContext, taskId); } else if (value instanceof List) { newValue = replaceList((List) value, taskId, documentContext); } else { newValue = value; } result.put(e.getKey(), newValue); } return result; } @SuppressWarnings("unchecked") private Object replaceList(List values, String taskId, DocumentContext io) { List replacedList = new LinkedList<>(); for (Object listVal : values) { if (listVal instanceof String) { Object replaced = replaceVariables(listVal.toString(), io, taskId); replacedList.add(replaced); } else if (listVal instanceof Map) { Object replaced = replace((Map) listVal, io, taskId); replacedList.add(replaced); } else if (listVal instanceof List) { Object replaced = replaceList((List) listVal, taskId, io); replacedList.add(replaced); } else { replacedList.add(listVal); } } return replacedList; } private Object replaceVariables( String paramString, DocumentContext documentContext, String taskId) { return replaceVariables(paramString, documentContext, taskId, 0); } private Object replaceVariables( String paramString, DocumentContext documentContext, String taskId, int depth) { var matcher = PATTERN.matcher(paramString); var replacements = new LinkedList(); while (matcher.find()) { var start = matcher.start(); var end = matcher.end(); var match = paramString.substring(start, end); String paramPath = match.substring(2, match.length() - 1); paramPath = replaceVariables(paramPath, documentContext, taskId, depth + 1).toString(); // if the paramPath is blank, meaning no value in between ${ and } // like ${}, ${ } etc, set the value to empty string if (StringUtils.isBlank(paramPath)) { replacements.add(new Replacement("", start, end)); continue; } if (EnvUtils.isEnvironmentVariable(paramPath)) { String sysValue = EnvUtils.getSystemParametersValue(paramPath, taskId); if (sysValue != null) { replacements.add(new Replacement(sysValue, start, end)); } } else { try { replacements.add(new Replacement(documentContext.read(paramPath), start, end)); } catch (Exception e) { LOGGER.warn( "Error reading documentContext for paramPath: {}. Exception: {}", paramPath, e); replacements.add(new Replacement(null, start, end)); } } } if (replacements.size() == 1 && replacements.getFirst().getStartIndex() == 0 && replacements.getFirst().getEndIndex() == paramString.length() && depth == 0) { return replacements.get(0).getReplacement(); } Collections.sort(replacements); var builder = new StringBuilder(paramString); for (int i = replacements.size() - 1; i >= 0; i--) { var replacement = replacements.get(i); builder.replace( replacement.getStartIndex(), replacement.getEndIndex(), Objects.toString(replacement.getReplacement())); } return builder.toString().replaceAll("\\$\\$\\{", "\\${"); } @Deprecated // Workflow schema version 1 is deprecated and new workflows should be using version 2 private Map getTaskInputV1( WorkflowModel workflow, Map inputParams) { Map input = new HashMap<>(); if (inputParams == null) { return input; } Map workflowInput = workflow.getInput(); inputParams.forEach( (paramName, value) -> { String paramPath = "" + value; String[] paramPathComponents = paramPath.split("\\."); Utils.checkArgument( paramPathComponents.length == 3, "Invalid input expression for " + paramName + ", paramPathComponents.size=" + paramPathComponents.length + ", expression=" + paramPath); String source = paramPathComponents[0]; // workflow, or task reference name String type = paramPathComponents[1]; // input/output String name = paramPathComponents[2]; // name of the parameter if ("workflow".equals(source)) { input.put(paramName, workflowInput.get(name)); } else { TaskModel task = workflow.getTaskByRefName(source); if (task != null) { if ("input".equals(type)) { input.put(paramName, task.getInputData().get(name)); } else { input.put(paramName, task.getOutputData().get(name)); } } } }); return input; } public Map getWorkflowInput( WorkflowDef workflowDef, Map inputParams) { if (workflowDef != null && workflowDef.getInputTemplate() != null) { clone(workflowDef.getInputTemplate()).forEach(inputParams::putIfAbsent); } return inputParams; } private static class Replacement implements Comparable { private final int startIndex; private final int endIndex; private final Object replacement; public Replacement(Object replacement, int startIndex, int endIndex) { this.replacement = replacement; this.startIndex = startIndex; this.endIndex = endIndex; } public Object getReplacement() { return replacement; } public int getStartIndex() { return startIndex; } public int getEndIndex() { return endIndex; } @Override public int compareTo(Replacement o) { return Long.compare(startIndex, o.startIndex); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/QueueUtils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.model.TaskModel; public class QueueUtils { public static final String DOMAIN_SEPARATOR = ":"; private static final String ISOLATION_SEPARATOR = "-"; private static final String EXECUTION_NAME_SPACE_SEPARATOR = "@"; public static String getQueueName(TaskModel taskModel) { return getQueueName( taskModel.getTaskType(), taskModel.getDomain(), taskModel.getIsolationGroupId(), taskModel.getExecutionNameSpace()); } public static String getQueueName(Task task) { return getQueueName( task.getTaskType(), task.getDomain(), task.getIsolationGroupId(), task.getExecutionNameSpace()); } /** * Creates a queue name string using taskType, domain, * isolationGroupId and executionNamespace. * * @return domain:taskType@eexecutionNameSpace-isolationGroupId. */ public static String getQueueName( String taskType, String domain, String isolationGroupId, String executionNamespace) { String queueName; if (domain == null) { queueName = taskType; } else { queueName = domain + DOMAIN_SEPARATOR + taskType; } if (executionNamespace != null) { queueName = queueName + EXECUTION_NAME_SPACE_SEPARATOR + executionNamespace; } if (isolationGroupId != null) { queueName = queueName + ISOLATION_SEPARATOR + isolationGroupId; } return queueName; } public static String getQueueNameWithoutDomain(String queueName) { return queueName.substring(queueName.indexOf(DOMAIN_SEPARATOR) + 1); } public static String getExecutionNameSpace(String queueName) { if (StringUtils.contains(queueName, ISOLATION_SEPARATOR) && StringUtils.contains(queueName, EXECUTION_NAME_SPACE_SEPARATOR)) { return StringUtils.substringBetween( queueName, EXECUTION_NAME_SPACE_SEPARATOR, ISOLATION_SEPARATOR); } else if (StringUtils.contains(queueName, EXECUTION_NAME_SPACE_SEPARATOR)) { return StringUtils.substringAfter(queueName, EXECUTION_NAME_SPACE_SEPARATOR); } else { return StringUtils.EMPTY; } } public static boolean isIsolatedQueue(String queue) { return StringUtils.isNotBlank(getIsolationGroup(queue)); } private static String getIsolationGroup(String queue) { return StringUtils.substringAfter(queue, QueueUtils.ISOLATION_SEPARATOR); } public static String getTaskType(String queue) { if (StringUtils.isBlank(queue)) { return StringUtils.EMPTY; } int domainSeperatorIndex = StringUtils.indexOf(queue, DOMAIN_SEPARATOR); int startIndex; if (domainSeperatorIndex == -1) { startIndex = 0; } else { startIndex = domainSeperatorIndex + 1; } int endIndex = StringUtils.indexOf(queue, EXECUTION_NAME_SPACE_SEPARATOR); if (endIndex == -1) { endIndex = StringUtils.lastIndexOf(queue, ISOLATION_SEPARATOR); } if (endIndex == -1) { endIndex = queue.length(); } return StringUtils.substring(queue, startIndex, endIndex); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/SemaphoreUtil.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.concurrent.Semaphore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** A class wrapping a semaphore which holds the number of permits available for processing. */ public class SemaphoreUtil { private static final Logger LOGGER = LoggerFactory.getLogger(SemaphoreUtil.class); private final Semaphore semaphore; public SemaphoreUtil(int numSlots) { LOGGER.debug("Semaphore util initialized with {} permits", numSlots); semaphore = new Semaphore(numSlots); } /** * Signals if processing is allowed based on whether specified number of permits can be * acquired. * * @param numSlots the number of permits to acquire * @return {@code true} - if permit is acquired {@code false} - if permit could not be acquired */ public boolean acquireSlots(int numSlots) { boolean acquired = semaphore.tryAcquire(numSlots); LOGGER.trace("Trying to acquire {} permit: {}", numSlots, acquired); return acquired; } /** Signals that processing is complete and the specified number of permits can be released. */ public void completeProcessing(int numSlots) { LOGGER.trace("Completed execution; releasing permit"); semaphore.release(numSlots); } /** * Gets the number of slots available for processing. * * @return number of available permits */ public int availableSlots() { int available = semaphore.availablePermits(); LOGGER.trace("Number of available permits: {}", available); return available; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/core/utils/Utils.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.core.exception.TransientException; public class Utils { public static final String DECIDER_QUEUE = "_deciderQueue"; /** * ID of the server. Can be host name, IP address or any other meaningful identifier * * @return canonical host name resolved for the instance, "unknown" if resolution fails */ public static String getServerId() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { return "unknown"; } } /** * Split string with "|" as delimiter. * * @param inputStr Input string * @return List of String */ public static List convertStringToList(String inputStr) { List list = new ArrayList<>(); if (StringUtils.isNotBlank(inputStr)) { list = Arrays.asList(inputStr.split("\\|")); } return list; } /** * Ensures the truth of an condition involving one or more parameters to the calling method. * * @param condition a boolean expression * @param errorMessage The exception message use if the input condition is not valid * @throws IllegalArgumentException if input condition is not valid. */ public static void checkArgument(boolean condition, String errorMessage) { if (!condition) { throw new IllegalArgumentException(errorMessage); } } /** * This method checks if the object is null or empty. * * @param object input of type {@link Object}. * @param errorMessage The exception message use if the object is empty or null. * @throws NullPointerException if input object is not valid. */ public static void checkNotNull(Object object, String errorMessage) { if (object == null) { throw new NullPointerException(errorMessage); } } /** * Used to determine if the exception is thrown due to a transient failure and the operation is * expected to succeed upon retrying. * * @param throwable the exception that is thrown * @return true - if the exception is a transient failure *

    false - if the exception is non-transient */ public static boolean isTransientException(Throwable throwable) { if (throwable != null) { return throwable instanceof TransientException; } return true; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/ConcurrentExecutionLimitDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.model.TaskModel; /** * A contract to support concurrency limits of tasks. * * @since v3.3.5. */ public interface ConcurrentExecutionLimitDAO { default void addTaskToLimit(TaskModel task) { throw new UnsupportedOperationException( getClass() + " does not support addTaskToLimit method."); } default void removeTaskFromLimit(TaskModel task) { throw new UnsupportedOperationException( getClass() + " does not support removeTaskFromLimit method."); } /** * Checks if the number of tasks in progress for the given taskDef will exceed the limit if the * task is scheduled to be in progress (given to the worker or for system tasks start() method * called) * * @param task The task to be executed. Limit is set in the Task's definition * @return true if by executing this task, the limit is breached. false otherwise. * @see TaskDef#concurrencyLimit() */ boolean exceedsLimit(TaskModel task); } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/EventHandlerDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import com.netflix.conductor.common.metadata.events.EventHandler; /** An abstraction to enable different Event Handler store implementations */ public interface EventHandlerDAO { /** * @param eventHandler Event handler to be added. *

    NOTE: Will throw an exception if an event handler already exists with the * name */ void addEventHandler(EventHandler eventHandler); /** * @param eventHandler Event handler to be updated. */ void updateEventHandler(EventHandler eventHandler); /** * @param name Removes the event handler from the system */ void removeEventHandler(String name); /** * @return All the event handlers registered in the system */ List getAllEventHandlers(); /** * @param event name of the event * @param activeOnly if true, returns only the active handlers * @return Returns the list of all the event handlers for a given event */ List getEventHandlersForEvent(String event, boolean activeOnly); } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/ExecutionDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; /** Data access layer for storing workflow executions */ public interface ExecutionDAO { /** * @param taskName Name of the task * @param workflowId Workflow instance id * @return List of pending tasks (in_progress) */ List getPendingTasksByWorkflow(String taskName, String workflowId); /** * @param taskType Type of task * @param startKey start * @param count number of tasks to return * @return List of tasks starting from startKey */ List getTasks(String taskType, String startKey, int count); /** * @param tasks tasks to be created * @return List of tasks that were created. *

    Note on the primary key constraint *

    For a given task reference name and retryCount should be considered unique/primary * key. Given two tasks with the same reference name and retryCount only one should be added * to the database. */ List createTasks(List tasks); /** * @param task Task to be updated */ void updateTask(TaskModel task); /** * Checks if the number of tasks in progress for the given taskDef will exceed the limit if the * task is scheduled to be in progress (given to the worker or for system tasks start() method * called) * * @param task The task to be executed. Limit is set in the Task's definition * @return true if by executing this task, the limit is breached. false otherwise. * @see TaskDef#concurrencyLimit() * @deprecated Since v3.3.5. Use {@link ConcurrentExecutionLimitDAO#exceedsLimit(TaskModel)}. */ @Deprecated default boolean exceedsInProgressLimit(TaskModel task) { throw new UnsupportedOperationException( getClass() + "does not support exceedsInProgressLimit"); } /** * @param taskId id of the task to be removed. * @return true if the deletion is successful, false otherwise. */ boolean removeTask(String taskId); /** * @param taskId Task instance id * @return Task */ TaskModel getTask(String taskId); /** * @param taskIds Task instance ids * @return List of tasks */ List getTasks(List taskIds); /** * @param taskType Type of the task for which to retrieve the list of pending tasks * @return List of pending tasks */ List getPendingTasksForTaskType(String taskType); /** * @param workflowId Workflow instance id * @return List of tasks for the given workflow instance id */ List getTasksForWorkflow(String workflowId); /** * @param workflow Workflow to be created * @return Id of the newly created workflow */ String createWorkflow(WorkflowModel workflow); /** * @param workflow Workflow to be updated * @return Id of the updated workflow */ String updateWorkflow(WorkflowModel workflow); /** * @param workflowId workflow instance id * @return true if the deletion is successful, false otherwise */ boolean removeWorkflow(String workflowId); /** * Removes the workflow with ttl seconds * * @param workflowId workflowId workflow instance id * @param ttlSeconds time to live in seconds. * @return */ boolean removeWorkflowWithExpiry(String workflowId, int ttlSeconds); /** * @param workflowType Workflow Type * @param workflowId workflow instance id */ void removeFromPendingWorkflow(String workflowType, String workflowId); /** * @param workflowId workflow instance id * @return Workflow */ WorkflowModel getWorkflow(String workflowId); /** * @param workflowId workflow instance id * @param includeTasks if set, includes the tasks (pending and completed) sorted by Task * Sequence number in Workflow. * @return Workflow instance details */ WorkflowModel getWorkflow(String workflowId, boolean includeTasks); /** * @param workflowName name of the workflow * @param version the workflow version * @return List of workflow ids which are running */ List getRunningWorkflowIds(String workflowName, int version); /** * @param workflowName Name of the workflow * @param version the workflow version * @return List of workflows that are running */ List getPendingWorkflowsByType(String workflowName, int version); /** * @param workflowName Name of the workflow * @return No. of running workflows */ long getPendingWorkflowCount(String workflowName); /** * @param taskDefName Name of the task * @return Number of task currently in IN_PROGRESS status */ long getInProgressTaskCount(String taskDefName); /** * @param workflowName Name of the workflow * @param startTime epoch time * @param endTime epoch time * @return List of workflows between start and end time */ List getWorkflowsByType(String workflowName, Long startTime, Long endTime); /** * @param workflowName workflow name * @param correlationId Correlation Id * @param includeTasks Option to includeTasks in results * @return List of workflows by correlation id */ List getWorkflowsByCorrelationId( String workflowName, String correlationId, boolean includeTasks); /** * @return true, if the DAO implementation is capable of searching across workflows false, if * the DAO implementation cannot perform searches across workflows (and needs to use * indexDAO) */ boolean canSearchAcrossWorkflows(); // Events /** * @param eventExecution Event Execution to be stored * @return true if the event was added. false otherwise when the event by id is already already * stored. */ boolean addEventExecution(EventExecution eventExecution); /** * @param eventExecution Event execution to be updated */ void updateEventExecution(EventExecution eventExecution); /** * @param eventExecution Event execution to be removed */ void removeEventExecution(EventExecution eventExecution); } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/IndexDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import java.util.concurrent.CompletableFuture; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; /** DAO to index the workflow and task details for searching. */ public interface IndexDAO { /** Setup method in charge or initializing/populating the index. */ void setup() throws Exception; /** * This method should return an unique identifier of the indexed doc * * @param workflow Workflow to be indexed */ void indexWorkflow(WorkflowSummary workflow); /** * This method should return an unique identifier of the indexed doc * * @param workflow Workflow to be indexed * @return CompletableFuture of type void */ CompletableFuture asyncIndexWorkflow(WorkflowSummary workflow); /** * @param task Task to be indexed */ void indexTask(TaskSummary task); /** * @param task Task to be indexed asynchronously * @return CompletableFuture of type void */ CompletableFuture asyncIndexTask(TaskSummary task); /** * @param query SQL like query for workflow search parameters. * @param freeText Additional query in free text. Lucene syntax * @param start start start index for pagination * @param count count # of workflow ids to be returned * @param sort sort options * @return List of workflow ids for the matching query */ SearchResult searchWorkflows( String query, String freeText, int start, int count, List sort); /** * @param query SQL like query for workflow search parameters. * @param freeText Additional query in free text. Lucene syntax * @param start start start index for pagination * @param count count # of workflow ids to be returned * @param sort sort options * @return List of workflows for the matching query */ SearchResult searchWorkflowSummary( String query, String freeText, int start, int count, List sort); /** * @param query SQL like query for task search parameters. * @param freeText Additional query in free text. Lucene syntax * @param start start start index for pagination * @param count count # of task ids to be returned * @param sort sort options * @return List of task ids for the matching query */ SearchResult searchTasks( String query, String freeText, int start, int count, List sort); /** * @param query SQL like query for task search parameters. * @param freeText Additional query in free text. Lucene syntax * @param start start start index for pagination * @param count count # of task ids to be returned * @param sort sort options * @return List of tasks for the matching query */ SearchResult searchTaskSummary( String query, String freeText, int start, int count, List sort); /** * Remove the workflow index * * @param workflowId workflow to be removed */ void removeWorkflow(String workflowId); /** * Remove the workflow index * * @param workflowId workflow to be removed * @return CompletableFuture of type void */ CompletableFuture asyncRemoveWorkflow(String workflowId); /** * Updates the index * * @param workflowInstanceId id of the workflow * @param keys keys to be updated * @param values values. Number of keys and values MUST match. */ void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values); /** * Updates the index * * @param workflowInstanceId id of the workflow * @param keys keys to be updated * @param values values. Number of keys and values MUST match. * @return CompletableFuture of type void */ CompletableFuture asyncUpdateWorkflow( String workflowInstanceId, String[] keys, Object[] values); /** * Remove the task index * * @param workflowId workflow containing task * @param taskId task to be removed */ void removeTask(String workflowId, String taskId); /** * Remove the task index asynchronously * * @param workflowId workflow containing task * @param taskId task to be removed * @return CompletableFuture of type void */ CompletableFuture asyncRemoveTask(String workflowId, String taskId); /** * Updates the index * * @param workflowId id of the workflow * @param taskId id of the task * @param keys keys to be updated * @param values values. Number of keys and values MUST match. */ void updateTask(String workflowId, String taskId, String[] keys, Object[] values); /** * Updates the index * * @param workflowId id of the workflow * @param taskId id of the task * @param keys keys to be updated * @param values values. Number of keys and values MUST match. * @return CompletableFuture of type void */ CompletableFuture asyncUpdateTask( String workflowId, String taskId, String[] keys, Object[] values); /** * Retrieves a specific field from the index * * @param workflowInstanceId id of the workflow * @param key field to be retrieved * @return value of the field as string */ String get(String workflowInstanceId, String key); /** * @param logs Task Execution logs to be indexed */ void addTaskExecutionLogs(List logs); /** * @param logs Task Execution logs to be indexed * @return CompletableFuture of type void */ CompletableFuture asyncAddTaskExecutionLogs(List logs); /** * @param taskId Id of the task for which to fetch the execution logs * @return Returns the task execution logs for given task id */ List getTaskExecutionLogs(String taskId); /** * @param eventExecution Event Execution to be indexed */ void addEventExecution(EventExecution eventExecution); List getEventExecutions(String event); /** * @param eventExecution Event Execution to be indexed * @return CompletableFuture of type void */ CompletableFuture asyncAddEventExecution(EventExecution eventExecution); /** * Adds an incoming external message into the index * * @param queue Name of the registered queue * @param msg Message */ void addMessage(String queue, Message msg); /** * Adds an incoming external message into the index * * @param queue Name of the registered queue * @param message {@link Message} * @return CompletableFuture of type Void */ CompletableFuture asyncAddMessage(String queue, Message message); List getMessages(String queue); /** * Search for Workflows completed or failed beyond archiveTtlDays * * @param indexName Name of the index to search * @param archiveTtlDays Archival Time to Live * @return List of worlflow Ids matching the pattern */ List searchArchivableWorkflows(String indexName, long archiveTtlDays); /** * Get total workflow counts that matches the query * * @param query SQL like query for workflow search parameters. * @param freeText Additional query in free text. Lucene syntax * @return Number of matches for the query */ long getWorkflowCount(String query, String freeText); } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/MetadataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import java.util.Optional; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; /** Data access layer for the workflow metadata - task definitions and workflow definitions */ public interface MetadataDAO { /** * @param taskDef task definition to be created */ TaskDef createTaskDef(TaskDef taskDef); /** * @param taskDef task definition to be updated. * @return name of the task definition */ TaskDef updateTaskDef(TaskDef taskDef); /** * @param name Name of the task * @return Task Definition */ TaskDef getTaskDef(String name); /** * @return All the task definitions */ List getAllTaskDefs(); /** * @param name Name of the task */ void removeTaskDef(String name); /** * @param def workflow definition */ void createWorkflowDef(WorkflowDef def); /** * @param def workflow definition */ void updateWorkflowDef(WorkflowDef def); /** * @param name Name of the workflow * @return Workflow Definition */ Optional getLatestWorkflowDef(String name); /** * @param name Name of the workflow * @param version version * @return workflow definition */ Optional getWorkflowDef(String name, int version); /** * @param name Name of the workflow definition to be removed * @param version Version of the workflow definition to be removed */ void removeWorkflowDef(String name, Integer version); /** * @return List of all the workflow definitions */ List getAllWorkflowDefs(); /** * @return List the latest versions of the workflow definitions */ List getAllWorkflowDefsLatestVersions(); } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/PollDataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import com.netflix.conductor.common.metadata.tasks.PollData; /** An abstraction to enable different PollData store implementations */ public interface PollDataDAO { /** * Updates the {@link PollData} information with the most recently polled data for a task queue. * * @param taskDefName name of the task as specified in the task definition * @param domain domain in which this task is being polled from * @param workerId the identifier of the worker polling for this task */ void updateLastPollData(String taskDefName, String domain, String workerId); /** * Retrieve the {@link PollData} for the given task in the given domain. * * @param taskDefName name of the task as specified in the task definition * @param domain domain for which {@link PollData} is being requested * @return the {@link PollData} for the given task queue in the specified domain */ PollData getPollData(String taskDefName, String domain); /** * Retrieve the {@link PollData} for the given task across all domains. * * @param taskDefName name of the task as specified in the task definition * @return the {@link PollData} for the given task queue in all domains */ List getPollData(String taskDefName); /** * Retrieve the {@link PollData} for all task types * * @return the {@link PollData} for all task types */ default List getAllPollData() { throw new UnsupportedOperationException( "The selected PollDataDAO (" + this.getClass().getSimpleName() + ") does not implement the getAllPollData() method"); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/QueueDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import java.util.Map; import com.netflix.conductor.core.events.queue.Message; /** DAO responsible for managing queuing for the tasks. */ public interface QueueDAO { /** * @param queueName name of the queue * @param id message id * @param offsetTimeInSecond time in seconds, after which the message should be marked visible. * (for timed queues) */ void push(String queueName, String id, long offsetTimeInSecond); /** * @param queueName name of the queue * @param id message id * @param priority message priority (between 0 and 99) * @param offsetTimeInSecond time in seconds, after which the message should be marked visible. * (for timed queues) */ void push(String queueName, String id, int priority, long offsetTimeInSecond); /** * @param queueName Name of the queue * @param messages messages to be pushed. */ void push(String queueName, List messages); /** * @param queueName Name of the queue * @param id message id * @param offsetTimeInSecond time in seconds, after which the message should be marked visible. * (for timed queues) * @return true if the element was added to the queue. false otherwise indicating the element * already exists in the queue. */ boolean pushIfNotExists(String queueName, String id, long offsetTimeInSecond); /** * @param queueName Name of the queue * @param id message id * @param priority message priority (between 0 and 99) * @param offsetTimeInSecond time in seconds, after which the message should be marked visible. * (for timed queues) * @return true if the element was added to the queue. false otherwise indicating the element * already exists in the queue. */ boolean pushIfNotExists(String queueName, String id, int priority, long offsetTimeInSecond); /** * @param queueName Name of the queue * @param count number of messages to be read from the queue * @param timeout timeout in milliseconds * @return list of elements from the named queue */ List pop(String queueName, int count, int timeout); /** * @param queueName Name of the queue * @param count number of messages to be read from the queue * @param timeout timeout in milliseconds * @return list of elements from the named queue */ List pollMessages(String queueName, int count, int timeout); /** * @param queueName Name of the queue * @param messageId Message id */ void remove(String queueName, String messageId); /** * @param queueName Name of the queue * @return size of the queue */ int getSize(String queueName); /** * @param queueName Name of the queue * @param messageId Message Id * @return true if the message was found and ack'ed */ boolean ack(String queueName, String messageId); /** * Extend the lease of the unacknowledged message for longer period. * * @param queueName Name of the queue * @param messageId Message Id * @param unackTimeout timeout in milliseconds for which the unack lease should be extended. * (replaces the current value with this value) * @return true if the message was updated with extended lease. false otherwise. */ boolean setUnackTimeout(String queueName, String messageId, long unackTimeout); /** * @param queueName Name of the queue */ void flush(String queueName); /** * @return key : queue name, value: size of the queue */ Map queuesDetail(); /** * @return key : queue name, value: map of shard name to size and unack queue size */ Map>> queuesDetailVerbose(); default void processUnacks(String queueName) {} /** * Resets the offsetTime on a message to 0, without pulling out the message from the queue * * @param queueName name of the queue * @param id message id * @return true if the message is in queue and the change was successful else returns false */ boolean resetOffsetTime(String queueName, String id); /** * Postpone a given message with postponeDurationInSeconds, so that the message won't be * available for further polls until specified duration. By default, the message is removed and * pushed backed with postponeDurationInSeconds to be backwards compatible. * * @param queueName name of the queue * @param messageId message id * @param priority message priority (between 0 and 99) * @param postponeDurationInSeconds duration in seconds by which the message is to be postponed */ default boolean postpone( String queueName, String messageId, int priority, long postponeDurationInSeconds) { remove(queueName, messageId); push(queueName, messageId, priority, postponeDurationInSeconds); return true; } /** * Check if the message with given messageId exists in the Queue. * * @param queueName * @param messageId * @return */ default boolean containsMessage(String queueName, String messageId) { throw new UnsupportedOperationException( "Please ensure your provided Queue implementation overrides and implements this method."); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/dao/RateLimitingDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.model.TaskModel; /** An abstraction to enable different Rate Limiting implementations */ public interface RateLimitingDAO { /** * Checks if the Task is rate limited or not based on the {@link * TaskModel#getRateLimitPerFrequency()} and {@link TaskModel#getRateLimitFrequencyInSeconds()} * * @param task: which needs to be evaluated whether it is rateLimited or not * @return true: If the {@link TaskModel} is rateLimited false: If the {@link TaskModel} is not * rateLimited */ boolean exceedsRateLimitPerFrequency(TaskModel task, TaskDef taskDef); } ================================================ FILE: core/src/main/java/com/netflix/conductor/metrics/Monitors.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.metrics; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.DistributionSummary; import com.netflix.spectator.api.Gauge; import com.netflix.spectator.api.Id; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Spectator; import com.netflix.spectator.api.Timer; import com.netflix.spectator.api.histogram.PercentileTimer; public class Monitors { private static final Registry registry = Spectator.globalRegistry(); public static final String NO_DOMAIN = "NO_DOMAIN"; private static final Map, Counter>> counters = new ConcurrentHashMap<>(); private static final Map, PercentileTimer>> timers = new ConcurrentHashMap<>(); private static final Map, Gauge>> gauges = new ConcurrentHashMap<>(); private static final Map, DistributionSummary>> distributionSummaries = new ConcurrentHashMap<>(); public static final String classQualifier = "WorkflowMonitor"; private Monitors() {} /** * Increment a counter that is used to measure the rate at which some event is occurring. * Consider a simple queue, counters would be used to measure things like the rate at which * items are being inserted and removed. * * @param className * @param name * @param additionalTags */ private static void counter(String className, String name, String... additionalTags) { getCounter(className, name, additionalTags).increment(); } /** * Set a gauge is a handle to get the current value. Typical examples for gauges would be the * size of a queue or number of threads in the running state. Since gauges are sampled, there is * no information about what might have occurred between samples. * * @param className * @param name * @param measurement * @param additionalTags */ private static void gauge( String className, String name, long measurement, String... additionalTags) { getGauge(className, name, additionalTags).set(measurement); } /** * Records a value for an event as a distribution summary. Unlike a gauge, this is sampled * multiple times during a minute or everytime a new value is recorded. * * @param className * @param name * @param additionalTags */ private static void distributionSummary( String className, String name, long value, String... additionalTags) { getDistributionSummary(className, name, additionalTags).record(value); } private static Timer getTimer(String className, String name, String... additionalTags) { Map tags = toMap(className, additionalTags); return timers.computeIfAbsent(name, s -> new ConcurrentHashMap<>()) .computeIfAbsent( tags, t -> { Id id = registry.createId(name, tags); return PercentileTimer.get(registry, id); }); } private static Counter getCounter(String className, String name, String... additionalTags) { Map tags = toMap(className, additionalTags); return counters.computeIfAbsent(name, s -> new ConcurrentHashMap<>()) .computeIfAbsent( tags, t -> { Id id = registry.createId(name, tags); return registry.counter(id); }); } private static Gauge getGauge(String className, String name, String... additionalTags) { Map tags = toMap(className, additionalTags); return gauges.computeIfAbsent(name, s -> new ConcurrentHashMap<>()) .computeIfAbsent( tags, t -> { Id id = registry.createId(name, tags); return registry.gauge(id); }); } private static DistributionSummary getDistributionSummary( String className, String name, String... additionalTags) { Map tags = toMap(className, additionalTags); return distributionSummaries .computeIfAbsent(name, s -> new ConcurrentHashMap<>()) .computeIfAbsent( tags, t -> { Id id = registry.createId(name, tags); return registry.distributionSummary(id); }); } private static Map toMap(String className, String... additionalTags) { Map tags = new HashMap<>(); tags.put("class", className); for (int j = 0; j < additionalTags.length - 1; j++) { String tk = additionalTags[j]; String tv = "" + additionalTags[j + 1]; if (!tv.isEmpty()) { tags.put(tk, tv); } j++; } return tags; } /** * @param className Name of the class * @param methodName Method name */ public static void error(String className, String methodName) { getCounter(className, "workflow_server_error", "methodName", methodName).increment(); } public static void recordGauge(String name, long count) { gauge(classQualifier, name, count); } public static void recordCounter(String name, long count, String... additionalTags) { getCounter(classQualifier, name, additionalTags).increment(count); } public static void recordQueueWaitTime(String taskType, long queueWaitTime) { getTimer(classQualifier, "task_queue_wait", "taskType", taskType) .record(queueWaitTime, TimeUnit.MILLISECONDS); } public static void recordTaskExecutionTime( String taskType, long duration, boolean includesRetries, TaskModel.Status status) { getTimer( classQualifier, "task_execution", "taskType", taskType, "includeRetries", "" + includesRetries, "status", status.name()) .record(duration, TimeUnit.MILLISECONDS); } public static void recordWorkflowDecisionTime(long duration) { getTimer(classQualifier, "workflow_decision").record(duration, TimeUnit.MILLISECONDS); } public static void recordTaskPollError(String taskType, String exception) { recordTaskPollError(taskType, NO_DOMAIN, exception); } public static void recordTaskPollError(String taskType, String domain, String exception) { counter( classQualifier, "task_poll_error", "taskType", taskType, "domain", domain, "exception", exception); } public static void recordTaskPoll(String taskType) { counter(classQualifier, "task_poll", "taskType", taskType); } public static void recordTaskPollCount(String taskType, int count) { recordTaskPollCount(taskType, NO_DOMAIN, count); } public static void recordTaskPollCount(String taskType, String domain, int count) { getCounter(classQualifier, "task_poll_count", "taskType", taskType, "domain", domain) .increment(count); } public static void recordQueueDepth(String taskType, long size, String ownerApp) { gauge( classQualifier, "task_queue_depth", size, "taskType", taskType, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordTaskInProgress(String taskType, long size, String ownerApp) { gauge( classQualifier, "task_in_progress", size, "taskType", taskType, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordRunningWorkflows(long count, String name, String ownerApp) { gauge( classQualifier, "workflow_running", count, "workflowName", name, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordNumTasksInWorkflow(long count, String name, String version) { distributionSummary( classQualifier, "tasks_in_workflow", count, "workflowName", name, "version", version); } public static void recordTaskTimeout(String taskType) { counter(classQualifier, "task_timeout", "taskType", taskType); } public static void recordTaskResponseTimeout(String taskType) { counter(classQualifier, "task_response_timeout", "taskType", taskType); } public static void recordTaskPendingTime(String taskType, String workflowType, long duration) { gauge( classQualifier, "task_pending_time", duration, "workflowName", workflowType, "taskType", taskType); } public static void recordWorkflowTermination( String workflowType, WorkflowModel.Status status, String ownerApp) { counter( classQualifier, "workflow_failure", "workflowName", workflowType, "status", status.name(), "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordWorkflowStartSuccess( String workflowType, String version, String ownerApp) { counter( classQualifier, "workflow_start_success", "workflowName", workflowType, "version", version, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordWorkflowStartError(String workflowType, String ownerApp) { counter( classQualifier, "workflow_start_error", "workflowName", workflowType, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")); } public static void recordUpdateConflict( String taskType, String workflowType, WorkflowModel.Status status) { counter( classQualifier, "task_update_conflict", "workflowName", workflowType, "taskType", taskType, "workflowStatus", status.name()); } public static void recordUpdateConflict( String taskType, String workflowType, TaskModel.Status status) { counter( classQualifier, "task_update_conflict", "workflowName", workflowType, "taskType", taskType, "taskStatus", status.name()); } public static void recordTaskUpdateError(String taskType, String workflowType) { counter( classQualifier, "task_update_error", "workflowName", workflowType, "taskType", taskType); } public static void recordTaskExtendLeaseError(String taskType, String workflowType) { counter( classQualifier, "task_extendLease_error", "workflowName", workflowType, "taskType", taskType); } public static void recordTaskQueueOpError(String taskType, String workflowType) { counter( classQualifier, "task_queue_op_error", "workflowName", workflowType, "taskType", taskType); } public static void recordWorkflowCompletion( String workflowType, long duration, String ownerApp) { getTimer( classQualifier, "workflow_execution", "workflowName", workflowType, "ownerApp", StringUtils.defaultIfBlank(ownerApp, "unknown")) .record(duration, TimeUnit.MILLISECONDS); } public static void recordUnackTime(String workflowType, long duration) { getTimer(classQualifier, "workflow_unack", "workflowName", workflowType) .record(duration, TimeUnit.MILLISECONDS); } public static void recordTaskRateLimited(String taskDefName, int limit) { gauge(classQualifier, "task_rate_limited", limit, "taskType", taskDefName); } public static void recordTaskConcurrentExecutionLimited(String taskDefName, int limit) { gauge(classQualifier, "task_concurrent_execution_limited", limit, "taskType", taskDefName); } public static void recordEventQueueMessagesProcessed( String queueType, String queueName, int count) { getCounter( classQualifier, "event_queue_messages_processed", "queueType", queueType, "queueName", queueName) .increment(count); } public static void recordObservableQMessageReceivedErrors(String queueType) { counter(classQualifier, "observable_queue_error", "queueType", queueType); } public static void recordEventQueueMessagesHandled(String queueType, String queueName) { counter( classQualifier, "event_queue_messages_handled", "queueType", queueType, "queueName", queueName); } public static void recordEventQueueMessagesError(String queueType, String queueName) { counter( classQualifier, "event_queue_messages_error", "queueType", queueType, "queueName", queueName); } public static void recordEventExecutionSuccess(String event, String handler, String action) { counter( classQualifier, "event_execution_success", "event", event, "handler", handler, "action", action); } public static void recordEventExecutionError( String event, String handler, String action, String exceptionClazz) { counter( classQualifier, "event_execution_error", "event", event, "handler", handler, "action", action, "exception", exceptionClazz); } public static void recordEventActionError(String action, String entityName, String event) { counter( classQualifier, "event_action_error", "action", action, "entityName", entityName, "event", event); } public static void recordDaoRequests( String dao, String action, String taskType, String workflowType) { counter( classQualifier, "dao_requests", "dao", dao, "action", action, "taskType", StringUtils.defaultIfBlank(taskType, "unknown"), "workflowType", StringUtils.defaultIfBlank(workflowType, "unknown")); } public static void recordDaoEventRequests(String dao, String action, String event) { counter(classQualifier, "dao_event_requests", "dao", dao, "action", action, "event", event); } public static void recordDaoPayloadSize( String dao, String action, String taskType, String workflowType, int size) { gauge( classQualifier, "dao_payload_size", size, "dao", dao, "action", action, "taskType", StringUtils.defaultIfBlank(taskType, "unknown"), "workflowType", StringUtils.defaultIfBlank(workflowType, "unknown")); } public static void recordExternalPayloadStorageUsage( String name, String operation, String payloadType) { counter( classQualifier, "external_payload_storage_usage", "name", name, "operation", operation, "payloadType", payloadType); } public static void recordDaoError(String dao, String action) { counter(classQualifier, "dao_errors", "dao", dao, "action", action); } public static void recordAckTaskError(String taskType) { counter(classQualifier, "task_ack_error", "taskType", taskType); } public static void recordESIndexTime(String action, String docType, long val) { getTimer(Monitors.classQualifier, action, "docType", docType) .record(val, TimeUnit.MILLISECONDS); } public static void recordWorkerQueueSize(String queueType, int val) { gauge(Monitors.classQualifier, "indexing_worker_queue", val, "queueType", queueType); } public static void recordDiscardedIndexingCount(String queueType) { counter(Monitors.classQualifier, "discarded_index_count", "queueType", queueType); } public static void recordAcquireLockUnsuccessful() { counter(classQualifier, "acquire_lock_unsuccessful"); } public static void recordAcquireLockFailure(String exceptionClassName) { counter(classQualifier, "acquire_lock_failure", "exceptionType", exceptionClassName); } public static void recordWorkflowArchived(String workflowType, WorkflowModel.Status status) { counter( classQualifier, "workflow_archived", "workflowName", workflowType, "workflowStatus", status.name()); } public static void recordArchivalDelayQueueSize(int val) { gauge(classQualifier, "workflow_archival_delay_queue_size", val); } public static void recordDiscardedArchivalCount() { counter(classQualifier, "discarded_archival_count"); } public static void recordSystemTaskWorkerPollingLimited(String queueName) { counter(classQualifier, "system_task_worker_polling_limited", "queueName", queueName); } public static void recordEventQueuePollSize(String queueType, int val) { gauge(Monitors.classQualifier, "event_queue_poll", val, "queueType", queueType); } public static void recordQueueMessageRepushFromRepairService(String queueName) { counter(classQualifier, "queue_message_repushed", "queueName", queueName); } public static void recordTaskExecLogSize(int val) { gauge(classQualifier, "task_exec_log_size", val); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/metrics/WorkflowMonitor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.metrics; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.service.MetadataService; import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; @Component @ConditionalOnProperty( name = "conductor.workflow-monitor.enabled", havingValue = "true", matchIfMissing = true) public class WorkflowMonitor { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowMonitor.class); private final MetadataService metadataService; private final QueueDAO queueDAO; private final ExecutionDAOFacade executionDAOFacade; private final int metadataRefreshInterval; private final Set asyncSystemTasks; private List taskDefs; private List workflowDefs; private int refreshCounter = 0; public WorkflowMonitor( MetadataService metadataService, QueueDAO queueDAO, ExecutionDAOFacade executionDAOFacade, @Value("${conductor.workflow-monitor.metadata-refresh-interval:10}") int metadataRefreshInterval, @Qualifier(ASYNC_SYSTEM_TASKS_QUALIFIER) Set asyncSystemTasks) { this.metadataService = metadataService; this.queueDAO = queueDAO; this.executionDAOFacade = executionDAOFacade; this.metadataRefreshInterval = metadataRefreshInterval; this.asyncSystemTasks = asyncSystemTasks; LOGGER.info("{} initialized.", WorkflowMonitor.class.getSimpleName()); } @Scheduled( initialDelayString = "${conductor.workflow-monitor.stats.initial-delay:120000}", fixedDelayString = "${conductor.workflow-monitor.stats.delay:60000}") public void reportMetrics() { try { if (refreshCounter <= 0) { workflowDefs = metadataService.getWorkflowDefs(); taskDefs = new ArrayList<>(metadataService.getTaskDefs()); refreshCounter = metadataRefreshInterval; } getPendingWorkflowToOwnerAppMap(workflowDefs) .forEach( (workflowName, ownerApp) -> { long count = executionDAOFacade.getPendingWorkflowCount(workflowName); Monitors.recordRunningWorkflows(count, workflowName, ownerApp); }); taskDefs.forEach( taskDef -> { long size = queueDAO.getSize(taskDef.getName()); long inProgressCount = executionDAOFacade.getInProgressTaskCount(taskDef.getName()); Monitors.recordQueueDepth(taskDef.getName(), size, taskDef.getOwnerApp()); if (taskDef.concurrencyLimit() > 0) { Monitors.recordTaskInProgress( taskDef.getName(), inProgressCount, taskDef.getOwnerApp()); } }); asyncSystemTasks.forEach( workflowSystemTask -> { long size = queueDAO.getSize(workflowSystemTask.getTaskType()); long inProgressCount = executionDAOFacade.getInProgressTaskCount( workflowSystemTask.getTaskType()); Monitors.recordQueueDepth(workflowSystemTask.getTaskType(), size, "system"); Monitors.recordTaskInProgress( workflowSystemTask.getTaskType(), inProgressCount, "system"); }); refreshCounter--; } catch (Exception e) { LOGGER.error("Error while publishing scheduled metrics", e); } } /** * Pending workflow data does not contain information about version. We only need the owner app * and workflow name, and we only need to query for the workflow once. */ @VisibleForTesting Map getPendingWorkflowToOwnerAppMap(List workflowDefs) { final Map> groupedWorkflowDefs = workflowDefs.stream().collect(Collectors.groupingBy(WorkflowDef::getName)); Map workflowNameToOwnerMap = new HashMap<>(); groupedWorkflowDefs.forEach( (key, value) -> { final WorkflowDef workflowDef = value.stream() .max(Comparator.comparing(WorkflowDef::getVersion)) .orElseThrow(NoSuchElementException::new); workflowNameToOwnerMap.put(key, workflowDef.getOwnerApp()); }); return workflowNameToOwnerMap; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/model/TaskModel.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.model; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.protobuf.Any; public class TaskModel { public enum Status { IN_PROGRESS(false, true, true), CANCELED(true, false, false), FAILED(true, false, true), FAILED_WITH_TERMINAL_ERROR(true, false, false), COMPLETED(true, true, true), COMPLETED_WITH_ERRORS(true, true, true), SCHEDULED(false, true, true), TIMED_OUT(true, false, true), SKIPPED(true, true, false); private final boolean terminal; private final boolean successful; private final boolean retriable; Status(boolean terminal, boolean successful, boolean retriable) { this.terminal = terminal; this.successful = successful; this.retriable = retriable; } public boolean isTerminal() { return terminal; } public boolean isSuccessful() { return successful; } public boolean isRetriable() { return retriable; } } private String taskType; private Status status; private String referenceTaskName; private int retryCount; private int seq; private String correlationId; private int pollCount; private String taskDefName; /** Time when the task was scheduled */ private long scheduledTime; /** Time when the task was first polled */ private long startTime; /** Time when the task completed executing */ private long endTime; /** Time when the task was last updated */ private long updateTime; private int startDelayInSeconds; private String retriedTaskId; private boolean retried; private boolean executed; private boolean callbackFromWorker = true; private long responseTimeoutSeconds; private String workflowInstanceId; private String workflowType; private String taskId; private String reasonForIncompletion; private long callbackAfterSeconds; private String workerId; private WorkflowTask workflowTask; private String domain; private Any inputMessage; private Any outputMessage; private int rateLimitPerFrequency; private int rateLimitFrequencyInSeconds; private String externalInputPayloadStoragePath; private String externalOutputPayloadStoragePath; private int workflowPriority; private String executionNameSpace; private String isolationGroupId; private int iteration; private String subWorkflowId; // Timeout after which the wait task should be marked as completed private long waitTimeout; /** * Used to note that a sub workflow associated with SUB_WORKFLOW task has an action performed on * it directly. */ private boolean subworkflowChanged; @JsonIgnore private Map inputPayload = new HashMap<>(); @JsonIgnore private Map outputPayload = new HashMap<>(); @JsonIgnore private Map inputData = new HashMap<>(); @JsonIgnore private Map outputData = new HashMap<>(); public String getTaskType() { return taskType; } public void setTaskType(String taskType) { this.taskType = taskType; } public Status getStatus() { return status; } public void setStatus(Status status) { this.status = status; } @JsonIgnore public Map getInputData() { if (!inputPayload.isEmpty() && !inputData.isEmpty()) { inputData.putAll(inputPayload); inputPayload = new HashMap<>(); return inputData; } else if (inputPayload.isEmpty()) { return inputData; } else { return inputPayload; } } @JsonIgnore public void setInputData(Map inputData) { if (inputData == null) { inputData = new HashMap<>(); } this.inputData = inputData; } /** * @deprecated Used only for JSON serialization and deserialization. */ @JsonProperty("inputData") @Deprecated public void setRawInputData(Map inputData) { setInputData(inputData); } /** * @deprecated Used only for JSON serialization and deserialization. */ @JsonProperty("inputData") @Deprecated public Map getRawInputData() { return inputData; } public String getReferenceTaskName() { return referenceTaskName; } public void setReferenceTaskName(String referenceTaskName) { this.referenceTaskName = referenceTaskName; } public int getRetryCount() { return retryCount; } public void setRetryCount(int retryCount) { this.retryCount = retryCount; } public int getSeq() { return seq; } public void setSeq(int seq) { this.seq = seq; } public String getCorrelationId() { return correlationId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public int getPollCount() { return pollCount; } public void setPollCount(int pollCount) { this.pollCount = pollCount; } public String getTaskDefName() { if (taskDefName == null || "".equals(taskDefName)) { taskDefName = taskType; } return taskDefName; } public void setTaskDefName(String taskDefName) { this.taskDefName = taskDefName; } public long getScheduledTime() { return scheduledTime; } public void setScheduledTime(long scheduledTime) { this.scheduledTime = scheduledTime; } public long getStartTime() { return startTime; } public void setStartTime(long startTime) { this.startTime = startTime; } public long getEndTime() { return endTime; } public void setEndTime(long endTime) { this.endTime = endTime; } public long getUpdateTime() { return updateTime; } public void setUpdateTime(long updateTime) { this.updateTime = updateTime; } public int getStartDelayInSeconds() { return startDelayInSeconds; } public void setStartDelayInSeconds(int startDelayInSeconds) { this.startDelayInSeconds = startDelayInSeconds; } public String getRetriedTaskId() { return retriedTaskId; } public void setRetriedTaskId(String retriedTaskId) { this.retriedTaskId = retriedTaskId; } public boolean isRetried() { return retried; } public void setRetried(boolean retried) { this.retried = retried; } public boolean isExecuted() { return executed; } public void setExecuted(boolean executed) { this.executed = executed; } public boolean isCallbackFromWorker() { return callbackFromWorker; } public void setCallbackFromWorker(boolean callbackFromWorker) { this.callbackFromWorker = callbackFromWorker; } public long getResponseTimeoutSeconds() { return responseTimeoutSeconds; } public void setResponseTimeoutSeconds(long responseTimeoutSeconds) { this.responseTimeoutSeconds = responseTimeoutSeconds; } public String getWorkflowInstanceId() { return workflowInstanceId; } public void setWorkflowInstanceId(String workflowInstanceId) { this.workflowInstanceId = workflowInstanceId; } public String getWorkflowType() { return workflowType; } public void setWorkflowType(String workflowType) { this.workflowType = workflowType; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getReasonForIncompletion() { return reasonForIncompletion; } public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = reasonForIncompletion; } public long getCallbackAfterSeconds() { return callbackAfterSeconds; } public void setCallbackAfterSeconds(long callbackAfterSeconds) { this.callbackAfterSeconds = callbackAfterSeconds; } public String getWorkerId() { return workerId; } public void setWorkerId(String workerId) { this.workerId = workerId; } @JsonIgnore public Map getOutputData() { if (!outputPayload.isEmpty() && !outputData.isEmpty()) { // Combine payload + data // data has precedence over payload because: // with external storage enabled, payload contains the old values // while data contains the latest and if payload took precedence, it // would remove latest outputs outputPayload.forEach(outputData::putIfAbsent); outputPayload = new HashMap<>(); return outputData; } else if (outputPayload.isEmpty()) { return outputData; } else { return outputPayload; } } @JsonIgnore public void setOutputData(Map outputData) { if (outputData == null) { outputData = new HashMap<>(); } this.outputData = outputData; } /** * @deprecated Used only for JSON serialization and deserialization. */ @JsonProperty("outputData") @Deprecated public void setRawOutputData(Map inputData) { setOutputData(inputData); } /** * @deprecated Used only for JSON serialization and deserialization. */ @JsonProperty("outputData") @Deprecated public Map getRawOutputData() { return outputData; } public WorkflowTask getWorkflowTask() { return workflowTask; } public void setWorkflowTask(WorkflowTask workflowTask) { this.workflowTask = workflowTask; } public String getDomain() { return domain; } public void setDomain(String domain) { this.domain = domain; } public Any getInputMessage() { return inputMessage; } public void setInputMessage(Any inputMessage) { this.inputMessage = inputMessage; } public Any getOutputMessage() { return outputMessage; } public void setOutputMessage(Any outputMessage) { this.outputMessage = outputMessage; } public int getRateLimitPerFrequency() { return rateLimitPerFrequency; } public void setRateLimitPerFrequency(int rateLimitPerFrequency) { this.rateLimitPerFrequency = rateLimitPerFrequency; } public int getRateLimitFrequencyInSeconds() { return rateLimitFrequencyInSeconds; } public void setRateLimitFrequencyInSeconds(int rateLimitFrequencyInSeconds) { this.rateLimitFrequencyInSeconds = rateLimitFrequencyInSeconds; } public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } public int getWorkflowPriority() { return workflowPriority; } public void setWorkflowPriority(int workflowPriority) { this.workflowPriority = workflowPriority; } public String getExecutionNameSpace() { return executionNameSpace; } public void setExecutionNameSpace(String executionNameSpace) { this.executionNameSpace = executionNameSpace; } public String getIsolationGroupId() { return isolationGroupId; } public void setIsolationGroupId(String isolationGroupId) { this.isolationGroupId = isolationGroupId; } public int getIteration() { return iteration; } public void setIteration(int iteration) { this.iteration = iteration; } public String getSubWorkflowId() { // For backwards compatibility if (StringUtils.isNotBlank(subWorkflowId)) { return subWorkflowId; } else { return this.getOutputData() != null && this.getOutputData().get("subWorkflowId") != null ? (String) this.getOutputData().get("subWorkflowId") : this.getInputData() != null ? (String) this.getInputData().get("subWorkflowId") : null; } } public void setSubWorkflowId(String subWorkflowId) { this.subWorkflowId = subWorkflowId; // For backwards compatibility if (this.outputData != null && this.outputData.containsKey("subWorkflowId")) { this.outputData.put("subWorkflowId", subWorkflowId); } } public boolean isSubworkflowChanged() { return subworkflowChanged; } public void setSubworkflowChanged(boolean subworkflowChanged) { this.subworkflowChanged = subworkflowChanged; } public void incrementPollCount() { ++this.pollCount; } /** * @return {@link Optional} containing the task definition if available */ public Optional getTaskDefinition() { return Optional.ofNullable(this.getWorkflowTask()).map(WorkflowTask::getTaskDefinition); } public boolean isLoopOverTask() { return iteration > 0; } public long getWaitTimeout() { return waitTimeout; } public void setWaitTimeout(long waitTimeout) { this.waitTimeout = waitTimeout; } /** * @return the queueWaitTime */ public long getQueueWaitTime() { if (this.startTime > 0 && this.scheduledTime > 0) { if (this.updateTime > 0 && getCallbackAfterSeconds() > 0) { long waitTime = System.currentTimeMillis() - (this.updateTime + (getCallbackAfterSeconds() * 1000)); return waitTime > 0 ? waitTime : 0; } else { return this.startTime - this.scheduledTime; } } return 0L; } /** * @return a copy of the task instance */ public TaskModel copy() { TaskModel copy = new TaskModel(); BeanUtils.copyProperties(this, copy); return copy; } public void externalizeInput(String path) { this.inputPayload = this.inputData; this.inputData = new HashMap<>(); this.externalInputPayloadStoragePath = path; } public void externalizeOutput(String path) { this.outputPayload = this.outputData; this.outputData = new HashMap<>(); this.externalOutputPayloadStoragePath = path; } public void internalizeInput(Map data) { this.inputData = new HashMap<>(); this.inputPayload = data; } public void internalizeOutput(Map data) { this.outputData = new HashMap<>(); this.outputPayload = data; } @Override public String toString() { return "TaskModel{" + "taskType='" + taskType + '\'' + ", status=" + status + ", inputData=" + inputData + ", referenceTaskName='" + referenceTaskName + '\'' + ", retryCount=" + retryCount + ", seq=" + seq + ", correlationId='" + correlationId + '\'' + ", pollCount=" + pollCount + ", taskDefName='" + taskDefName + '\'' + ", scheduledTime=" + scheduledTime + ", startTime=" + startTime + ", endTime=" + endTime + ", updateTime=" + updateTime + ", startDelayInSeconds=" + startDelayInSeconds + ", retriedTaskId='" + retriedTaskId + '\'' + ", retried=" + retried + ", executed=" + executed + ", callbackFromWorker=" + callbackFromWorker + ", responseTimeoutSeconds=" + responseTimeoutSeconds + ", workflowInstanceId='" + workflowInstanceId + '\'' + ", workflowType='" + workflowType + '\'' + ", taskId='" + taskId + '\'' + ", reasonForIncompletion='" + reasonForIncompletion + '\'' + ", callbackAfterSeconds=" + callbackAfterSeconds + ", workerId='" + workerId + '\'' + ", outputData=" + outputData + ", workflowTask=" + workflowTask + ", domain='" + domain + '\'' + ", waitTimeout='" + waitTimeout + '\'' + ", inputMessage=" + inputMessage + ", outputMessage=" + outputMessage + ", rateLimitPerFrequency=" + rateLimitPerFrequency + ", rateLimitFrequencyInSeconds=" + rateLimitFrequencyInSeconds + ", externalInputPayloadStoragePath='" + externalInputPayloadStoragePath + '\'' + ", externalOutputPayloadStoragePath='" + externalOutputPayloadStoragePath + '\'' + ", workflowPriority=" + workflowPriority + ", executionNameSpace='" + executionNameSpace + '\'' + ", isolationGroupId='" + isolationGroupId + '\'' + ", iteration=" + iteration + ", subWorkflowId='" + subWorkflowId + '\'' + ", subworkflowChanged=" + subworkflowChanged + '}'; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TaskModel taskModel = (TaskModel) o; return getRetryCount() == taskModel.getRetryCount() && getSeq() == taskModel.getSeq() && getPollCount() == taskModel.getPollCount() && getScheduledTime() == taskModel.getScheduledTime() && getStartTime() == taskModel.getStartTime() && getEndTime() == taskModel.getEndTime() && getUpdateTime() == taskModel.getUpdateTime() && getStartDelayInSeconds() == taskModel.getStartDelayInSeconds() && isRetried() == taskModel.isRetried() && isExecuted() == taskModel.isExecuted() && isCallbackFromWorker() == taskModel.isCallbackFromWorker() && getResponseTimeoutSeconds() == taskModel.getResponseTimeoutSeconds() && getCallbackAfterSeconds() == taskModel.getCallbackAfterSeconds() && getRateLimitPerFrequency() == taskModel.getRateLimitPerFrequency() && getRateLimitFrequencyInSeconds() == taskModel.getRateLimitFrequencyInSeconds() && getWorkflowPriority() == taskModel.getWorkflowPriority() && getIteration() == taskModel.getIteration() && isSubworkflowChanged() == taskModel.isSubworkflowChanged() && Objects.equals(getTaskType(), taskModel.getTaskType()) && getStatus() == taskModel.getStatus() && Objects.equals(getInputData(), taskModel.getInputData()) && Objects.equals(getReferenceTaskName(), taskModel.getReferenceTaskName()) && Objects.equals(getCorrelationId(), taskModel.getCorrelationId()) && Objects.equals(getTaskDefName(), taskModel.getTaskDefName()) && Objects.equals(getRetriedTaskId(), taskModel.getRetriedTaskId()) && Objects.equals(getWorkflowInstanceId(), taskModel.getWorkflowInstanceId()) && Objects.equals(getWorkflowType(), taskModel.getWorkflowType()) && Objects.equals(getTaskId(), taskModel.getTaskId()) && Objects.equals(getReasonForIncompletion(), taskModel.getReasonForIncompletion()) && Objects.equals(getWorkerId(), taskModel.getWorkerId()) && Objects.equals(getWaitTimeout(), taskModel.getWaitTimeout()) && Objects.equals(outputData, taskModel.outputData) && Objects.equals(outputPayload, taskModel.outputPayload) && Objects.equals(getWorkflowTask(), taskModel.getWorkflowTask()) && Objects.equals(getDomain(), taskModel.getDomain()) && Objects.equals(getInputMessage(), taskModel.getInputMessage()) && Objects.equals(getOutputMessage(), taskModel.getOutputMessage()) && Objects.equals( getExternalInputPayloadStoragePath(), taskModel.getExternalInputPayloadStoragePath()) && Objects.equals( getExternalOutputPayloadStoragePath(), taskModel.getExternalOutputPayloadStoragePath()) && Objects.equals(getExecutionNameSpace(), taskModel.getExecutionNameSpace()) && Objects.equals(getIsolationGroupId(), taskModel.getIsolationGroupId()) && Objects.equals(getSubWorkflowId(), taskModel.getSubWorkflowId()); } @Override public int hashCode() { return Objects.hash( getTaskType(), getStatus(), getInputData(), getReferenceTaskName(), getRetryCount(), getSeq(), getCorrelationId(), getPollCount(), getTaskDefName(), getScheduledTime(), getStartTime(), getEndTime(), getUpdateTime(), getStartDelayInSeconds(), getRetriedTaskId(), isRetried(), isExecuted(), isCallbackFromWorker(), getResponseTimeoutSeconds(), getWorkflowInstanceId(), getWorkflowType(), getTaskId(), getReasonForIncompletion(), getCallbackAfterSeconds(), getWorkerId(), getWaitTimeout(), outputData, outputPayload, getWorkflowTask(), getDomain(), getInputMessage(), getOutputMessage(), getRateLimitPerFrequency(), getRateLimitFrequencyInSeconds(), getExternalInputPayloadStoragePath(), getExternalOutputPayloadStoragePath(), getWorkflowPriority(), getExecutionNameSpace(), getIsolationGroupId(), getIteration(), getSubWorkflowId(), isSubworkflowChanged()); } public Task toTask() { Task task = new Task(); BeanUtils.copyProperties(this, task); task.setStatus(Task.Status.valueOf(status.name())); // ensure that input/output is properly represented if (externalInputPayloadStoragePath != null) { task.setInputData(new HashMap<>()); } if (externalOutputPayloadStoragePath != null) { task.setOutputData(new HashMap<>()); } return task; } public static Task.Status mapToTaskStatus(TaskModel.Status status) { return Task.Status.valueOf(status.name()); } public void addInput(String key, Object value) { this.inputData.put(key, value); } public void addInput(Map inputData) { if (inputData != null) { this.inputData.putAll(inputData); } } public void addOutput(String key, Object value) { this.outputData.put(key, value); } public void addOutput(Map outputData) { if (outputData != null) { this.outputData.putAll(outputData); } } public void clearOutput() { this.outputData.clear(); this.outputPayload.clear(); this.externalOutputPayloadStoragePath = null; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/model/WorkflowModel.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.model; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.core.utils.Utils; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; public class WorkflowModel { public enum Status { RUNNING(false, false), COMPLETED(true, true), FAILED(true, false), TIMED_OUT(true, false), TERMINATED(true, false), PAUSED(false, true); private final boolean terminal; private final boolean successful; Status(boolean terminal, boolean successful) { this.terminal = terminal; this.successful = successful; } public boolean isTerminal() { return terminal; } public boolean isSuccessful() { return successful; } } private Status status = Status.RUNNING; private long endTime; private String workflowId; private String parentWorkflowId; private String parentWorkflowTaskId; private List tasks = new LinkedList<>(); private String correlationId; private String reRunFromWorkflowId; private String reasonForIncompletion; private String event; private Map taskToDomain = new HashMap<>(); @JsonInclude(JsonInclude.Include.NON_EMPTY) private Set failedReferenceTaskNames = new HashSet<>(); @JsonInclude(JsonInclude.Include.NON_EMPTY) private Set failedTaskNames = new HashSet<>(); private WorkflowDef workflowDefinition; private String externalInputPayloadStoragePath; private String externalOutputPayloadStoragePath; private int priority; private Map variables = new HashMap<>(); private long lastRetriedTime; private String ownerApp; private Long createTime; private Long updatedTime; private String createdBy; private String updatedBy; // Capture the failed taskId if the workflow execution failed because of task failure private String failedTaskId; private Status previousStatus; @JsonIgnore private Map input = new HashMap<>(); @JsonIgnore private Map output = new HashMap<>(); @JsonIgnore private Map inputPayload = new HashMap<>(); @JsonIgnore private Map outputPayload = new HashMap<>(); public Status getPreviousStatus() { return previousStatus; } public void setPreviousStatus(Status status) { this.previousStatus = status; } public Status getStatus() { return status; } public void setStatus(Status status) { // update previous status if current status changed if (this.status != status) { setPreviousStatus(this.status); } this.status = status; } public long getEndTime() { return endTime; } public void setEndTime(long endTime) { this.endTime = endTime; } public String getWorkflowId() { return workflowId; } public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } public String getParentWorkflowId() { return parentWorkflowId; } public void setParentWorkflowId(String parentWorkflowId) { this.parentWorkflowId = parentWorkflowId; } public String getParentWorkflowTaskId() { return parentWorkflowTaskId; } public void setParentWorkflowTaskId(String parentWorkflowTaskId) { this.parentWorkflowTaskId = parentWorkflowTaskId; } public List getTasks() { return tasks; } public void setTasks(List tasks) { this.tasks = tasks; } @JsonIgnore public Map getInput() { if (!inputPayload.isEmpty() && !input.isEmpty()) { input.putAll(inputPayload); inputPayload = new HashMap<>(); return input; } else if (inputPayload.isEmpty()) { return input; } else { return inputPayload; } } @JsonIgnore public void setInput(Map input) { if (input == null) { input = new HashMap<>(); } this.input = input; } @JsonIgnore public Map getOutput() { if (!outputPayload.isEmpty() && !output.isEmpty()) { output.putAll(outputPayload); outputPayload = new HashMap<>(); return output; } else if (outputPayload.isEmpty()) { return output; } else { return outputPayload; } } @JsonIgnore public void setOutput(Map output) { if (output == null) { output = new HashMap<>(); } this.output = output; } /** * @deprecated Used only for JSON serialization and deserialization. */ @Deprecated @JsonProperty("input") public Map getRawInput() { return input; } /** * @deprecated Used only for JSON serialization and deserialization. */ @Deprecated @JsonProperty("input") public void setRawInput(Map input) { setInput(input); } /** * @deprecated Used only for JSON serialization and deserialization. */ @Deprecated @JsonProperty("output") public Map getRawOutput() { return output; } /** * @deprecated Used only for JSON serialization and deserialization. */ @Deprecated @JsonProperty("output") public void setRawOutput(Map output) { setOutput(output); } public String getCorrelationId() { return correlationId; } public void setCorrelationId(String correlationId) { this.correlationId = correlationId; } public String getReRunFromWorkflowId() { return reRunFromWorkflowId; } public void setReRunFromWorkflowId(String reRunFromWorkflowId) { this.reRunFromWorkflowId = reRunFromWorkflowId; } public String getReasonForIncompletion() { return reasonForIncompletion; } public void setReasonForIncompletion(String reasonForIncompletion) { this.reasonForIncompletion = reasonForIncompletion; } public String getEvent() { return event; } public void setEvent(String event) { this.event = event; } public Map getTaskToDomain() { return taskToDomain; } public void setTaskToDomain(Map taskToDomain) { this.taskToDomain = taskToDomain; } public Set getFailedReferenceTaskNames() { return failedReferenceTaskNames; } public void setFailedReferenceTaskNames(Set failedReferenceTaskNames) { this.failedReferenceTaskNames = failedReferenceTaskNames; } public Set getFailedTaskNames() { return failedTaskNames; } public void setFailedTaskNames(Set failedTaskNames) { this.failedTaskNames = failedTaskNames; } public WorkflowDef getWorkflowDefinition() { return workflowDefinition; } public void setWorkflowDefinition(WorkflowDef workflowDefinition) { this.workflowDefinition = workflowDefinition; } public String getExternalInputPayloadStoragePath() { return externalInputPayloadStoragePath; } public void setExternalInputPayloadStoragePath(String externalInputPayloadStoragePath) { this.externalInputPayloadStoragePath = externalInputPayloadStoragePath; } public String getExternalOutputPayloadStoragePath() { return externalOutputPayloadStoragePath; } public void setExternalOutputPayloadStoragePath(String externalOutputPayloadStoragePath) { this.externalOutputPayloadStoragePath = externalOutputPayloadStoragePath; } public int getPriority() { return priority; } public void setPriority(int priority) { if (priority < 0 || priority > 99) { throw new IllegalArgumentException("priority MUST be between 0 and 99 (inclusive)"); } this.priority = priority; } public Map getVariables() { return variables; } public void setVariables(Map variables) { this.variables = variables; } public long getLastRetriedTime() { return lastRetriedTime; } public void setLastRetriedTime(long lastRetriedTime) { this.lastRetriedTime = lastRetriedTime; } public String getOwnerApp() { return ownerApp; } public void setOwnerApp(String ownerApp) { this.ownerApp = ownerApp; } public Long getCreateTime() { return createTime; } public void setCreateTime(Long createTime) { this.createTime = createTime; } public Long getUpdatedTime() { return updatedTime; } public void setUpdatedTime(Long updatedTime) { this.updatedTime = updatedTime; } public String getCreatedBy() { return createdBy; } public void setCreatedBy(String createdBy) { this.createdBy = createdBy; } public String getUpdatedBy() { return updatedBy; } public void setUpdatedBy(String updatedBy) { this.updatedBy = updatedBy; } public String getFailedTaskId() { return failedTaskId; } public void setFailedTaskId(String failedTaskId) { this.failedTaskId = failedTaskId; } /** * Convenience method for accessing the workflow definition name. * * @return the workflow definition name. */ public String getWorkflowName() { Utils.checkNotNull(workflowDefinition, "Workflow definition is null"); return workflowDefinition.getName(); } /** * Convenience method for accessing the workflow definition version. * * @return the workflow definition version. */ public int getWorkflowVersion() { Utils.checkNotNull(workflowDefinition, "Workflow definition is null"); return workflowDefinition.getVersion(); } public boolean hasParent() { return StringUtils.isNotEmpty(parentWorkflowId); } /** * A string representation of all relevant fields that identify this workflow. Intended for use * in log and other system generated messages. */ public String toShortString() { String name = workflowDefinition != null ? workflowDefinition.getName() : null; Integer version = workflowDefinition != null ? workflowDefinition.getVersion() : null; return String.format("%s.%s/%s", name, version, workflowId); } public TaskModel getTaskByRefName(String refName) { if (refName == null) { throw new RuntimeException( "refName passed is null. Check the workflow execution. For dynamic tasks, make sure referenceTaskName is set to a not null value"); } LinkedList found = new LinkedList<>(); for (TaskModel task : tasks) { if (task.getReferenceTaskName() == null) { throw new RuntimeException( "Task " + task.getTaskDefName() + ", seq=" + task.getSeq() + " does not have reference name specified."); } if (task.getReferenceTaskName().equals(refName)) { found.add(task); } } if (found.isEmpty()) { return null; } return found.getLast(); } public void externalizeInput(String path) { this.inputPayload = this.input; this.input = new HashMap<>(); this.externalInputPayloadStoragePath = path; } public void externalizeOutput(String path) { this.outputPayload = this.output; this.output = new HashMap<>(); this.externalOutputPayloadStoragePath = path; } public void internalizeInput(Map data) { this.input = new HashMap<>(); this.inputPayload = data; } public void internalizeOutput(Map data) { this.output = new HashMap<>(); this.outputPayload = data; } @Override public String toString() { String name = workflowDefinition != null ? workflowDefinition.getName() : null; Integer version = workflowDefinition != null ? workflowDefinition.getVersion() : null; return String.format("%s.%s/%s.%s", name, version, workflowId, status); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; WorkflowModel that = (WorkflowModel) o; return getEndTime() == that.getEndTime() && getPriority() == that.getPriority() && getLastRetriedTime() == that.getLastRetriedTime() && getStatus() == that.getStatus() && Objects.equals(getWorkflowId(), that.getWorkflowId()) && Objects.equals(getParentWorkflowId(), that.getParentWorkflowId()) && Objects.equals(getParentWorkflowTaskId(), that.getParentWorkflowTaskId()) && Objects.equals(getTasks(), that.getTasks()) && Objects.equals(getInput(), that.getInput()) && Objects.equals(output, that.output) && Objects.equals(outputPayload, that.outputPayload) && Objects.equals(getCorrelationId(), that.getCorrelationId()) && Objects.equals(getReRunFromWorkflowId(), that.getReRunFromWorkflowId()) && Objects.equals(getReasonForIncompletion(), that.getReasonForIncompletion()) && Objects.equals(getEvent(), that.getEvent()) && Objects.equals(getTaskToDomain(), that.getTaskToDomain()) && Objects.equals(getFailedReferenceTaskNames(), that.getFailedReferenceTaskNames()) && Objects.equals(getFailedTaskNames(), that.getFailedTaskNames()) && Objects.equals(getWorkflowDefinition(), that.getWorkflowDefinition()) && Objects.equals( getExternalInputPayloadStoragePath(), that.getExternalInputPayloadStoragePath()) && Objects.equals( getExternalOutputPayloadStoragePath(), that.getExternalOutputPayloadStoragePath()) && Objects.equals(getVariables(), that.getVariables()) && Objects.equals(getOwnerApp(), that.getOwnerApp()) && Objects.equals(getCreateTime(), that.getCreateTime()) && Objects.equals(getUpdatedTime(), that.getUpdatedTime()) && Objects.equals(getCreatedBy(), that.getCreatedBy()) && Objects.equals(getUpdatedBy(), that.getUpdatedBy()); } @Override public int hashCode() { return Objects.hash( getStatus(), getEndTime(), getWorkflowId(), getParentWorkflowId(), getParentWorkflowTaskId(), getTasks(), getInput(), output, outputPayload, getCorrelationId(), getReRunFromWorkflowId(), getReasonForIncompletion(), getEvent(), getTaskToDomain(), getFailedReferenceTaskNames(), getFailedTaskNames(), getWorkflowDefinition(), getExternalInputPayloadStoragePath(), getExternalOutputPayloadStoragePath(), getPriority(), getVariables(), getLastRetriedTime(), getOwnerApp(), getCreateTime(), getUpdatedTime(), getCreatedBy(), getUpdatedBy()); } public Workflow toWorkflow() { Workflow workflow = new Workflow(); BeanUtils.copyProperties(this, workflow); workflow.setStatus(Workflow.WorkflowStatus.valueOf(this.status.name())); workflow.setTasks(tasks.stream().map(TaskModel::toTask).collect(Collectors.toList())); workflow.setUpdateTime(this.updatedTime); // ensure that input/output is properly represented if (externalInputPayloadStoragePath != null) { workflow.setInput(new HashMap<>()); } if (externalOutputPayloadStoragePath != null) { workflow.setOutput(new HashMap<>()); } return workflow; } public void addInput(String key, Object value) { this.input.put(key, value); } public void addInput(Map inputData) { if (inputData != null) { this.input.putAll(inputData); } } public void addOutput(String key, Object value) { this.output.put(key, value); } public void addOutput(Map outputData) { if (outputData != null) { this.output.putAll(outputData); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/AdminService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import java.util.Map; import javax.validation.constraints.NotEmpty; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.metadata.tasks.Task; @Validated public interface AdminService { /** * Queue up all the running workflows for sweep. * * @param workflowId Id of the workflow * @return the id of the workflow instance that can be use for tracking. */ String requeueSweep( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Get all the configuration parameters. * * @return all the configuration parameters. */ Map getAllConfig(); /** * Get the list of pending tasks for a given task type. * * @param taskType Name of the task * @param start Start index of pagination * @param count Number of entries * @return list of pending {@link Task} */ List getListOfPendingTask( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType, Integer start, Integer count); /** * Verify that the Workflow is consistent, and run repairs as needed. * * @param workflowId id of the workflow to be returned * @return true, if repair was successful */ boolean verifyAndRepairWorkflowConsistency( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Get registered queues. * * @param verbose `true|false` for verbose logs * @return map of event queues */ Map getEventQueues(boolean verbose); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/AdminServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.springframework.boot.info.BuildProperties; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Audit; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.EventQueueManager; import com.netflix.conductor.core.reconciliation.WorkflowRepairService; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.QueueDAO; @Audit @Trace @Service public class AdminServiceImpl implements AdminService { private final ConductorProperties properties; private final ExecutionService executionService; private final QueueDAO queueDAO; private final WorkflowRepairService workflowRepairService; private final EventQueueManager eventQueueManager; private final BuildProperties buildProperties; public AdminServiceImpl( ConductorProperties properties, ExecutionService executionService, QueueDAO queueDAO, Optional workflowRepairService, Optional eventQueueManager, Optional buildProperties) { this.properties = properties; this.executionService = executionService; this.queueDAO = queueDAO; this.workflowRepairService = workflowRepairService.orElse(null); this.eventQueueManager = eventQueueManager.orElse(null); this.buildProperties = buildProperties.orElse(null); } /** * Get all the configuration parameters. * * @return all the configuration parameters. */ public Map getAllConfig() { Map configs = properties.getAll(); configs.putAll(getBuildProperties()); return configs; } /** * Get all build properties * * @return all the build properties. */ private Map getBuildProperties() { if (buildProperties == null) return Collections.emptyMap(); Map buildProps = new HashMap<>(); buildProps.put("version", buildProperties.getVersion()); buildProps.put("buildDate", buildProperties.getTime()); return buildProps; } /** * Get the list of pending tasks for a given task type. * * @param taskType Name of the task * @param start Start index of pagination * @param count Number of entries * @return list of pending {@link Task} */ public List getListOfPendingTask(String taskType, Integer start, Integer count) { List tasks = executionService.getPendingTasksForTaskType(taskType); int total = start + count; total = Math.min(tasks.size(), total); if (start > tasks.size()) { start = tasks.size(); } return tasks.subList(start, total); } @Override public boolean verifyAndRepairWorkflowConsistency(String workflowId) { if (workflowRepairService == null) { throw new IllegalStateException( WorkflowRepairService.class.getSimpleName() + " is disabled."); } return workflowRepairService.verifyAndRepairWorkflow(workflowId, true); } /** * Queue up the workflow for sweep. * * @param workflowId Id of the workflow * @return the id of the workflow instance that can be use for tracking. */ public String requeueSweep(String workflowId) { boolean pushed = queueDAO.pushIfNotExists( Utils.DECIDER_QUEUE, workflowId, properties.getWorkflowOffsetTimeout().getSeconds()); return pushed + "." + workflowId; } /** * Get registered queues. * * @param verbose `true|false` for verbose logs * @return map of event queues */ public Map getEventQueues(boolean verbose) { if (eventQueueManager == null) { throw new IllegalStateException("Event processing is DISABLED"); } return (verbose ? eventQueueManager.getQueueSizes() : eventQueueManager.getQueues()); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/EventService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.metadata.events.EventHandler; @Validated public interface EventService { /** * Add a new event handler. * * @param eventHandler Instance of {@link EventHandler} */ void addEventHandler( @NotNull(message = "EventHandler cannot be null.") @Valid EventHandler eventHandler); /** * Update an existing event handler. * * @param eventHandler Instance of {@link EventHandler} */ void updateEventHandler( @NotNull(message = "EventHandler cannot be null.") @Valid EventHandler eventHandler); /** * Remove an event handler. * * @param name Event name */ void removeEventHandlerStatus( @NotEmpty(message = "EventHandler name cannot be null or empty.") String name); /** * Get all the event handlers. * * @return list of {@link EventHandler} */ List getEventHandlers(); /** * Get event handlers for a given event. * * @param event Event Name * @param activeOnly `true|false` for active only events * @return list of {@link EventHandler} */ List getEventHandlersForEvent( @NotEmpty(message = "Event cannot be null or empty.") String event, boolean activeOnly); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/EventServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import org.springframework.stereotype.Service; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.core.events.EventQueues; @Service public class EventServiceImpl implements EventService { private final MetadataService metadataService; public EventServiceImpl(MetadataService metadataService, EventQueues eventQueues) { this.metadataService = metadataService; } /** * Add a new event handler. * * @param eventHandler Instance of {@link EventHandler} */ public void addEventHandler(EventHandler eventHandler) { metadataService.addEventHandler(eventHandler); } /** * Update an existing event handler. * * @param eventHandler Instance of {@link EventHandler} */ public void updateEventHandler(EventHandler eventHandler) { metadataService.updateEventHandler(eventHandler); } /** * Remove an event handler. * * @param name Event name */ public void removeEventHandlerStatus(String name) { metadataService.removeEventHandlerStatus(name); } /** * Get all the event handlers. * * @return list of {@link EventHandler} */ public List getEventHandlers() { return metadataService.getAllEventHandlers(); } /** * Get event handlers for a given event. * * @param event Event Name * @param activeOnly `true|false` for active only events * @return list of {@link EventHandler} */ public List getEventHandlersForEvent(String event, boolean activeOnly) { return metadataService.getEventHandlersForEvent(event, activeOnly); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/ExecutionLockService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.sync.Lock; import com.netflix.conductor.metrics.Monitors; @Service @Trace public class ExecutionLockService { private static final Logger LOGGER = LoggerFactory.getLogger(ExecutionLockService.class); private final ConductorProperties properties; private final Lock lock; private final long lockLeaseTime; private final long lockTimeToTry; @Autowired public ExecutionLockService(ConductorProperties properties, Lock lock) { this.properties = properties; this.lock = lock; this.lockLeaseTime = properties.getLockLeaseTime().toMillis(); this.lockTimeToTry = properties.getLockTimeToTry().toMillis(); } /** * Tries to acquire lock with reasonable timeToTry duration and lease time. Exits if a lock * cannot be acquired. Considering that the workflow decide can be triggered through multiple * entry points, and periodically through the sweeper service, do not block on acquiring the * lock, as the order of execution of decides on a workflow doesn't matter. * * @param lockId * @return */ public boolean acquireLock(String lockId) { return acquireLock(lockId, lockTimeToTry, lockLeaseTime); } public boolean acquireLock(String lockId, long timeToTryMs) { return acquireLock(lockId, timeToTryMs, lockLeaseTime); } public boolean acquireLock(String lockId, long timeToTryMs, long leaseTimeMs) { if (properties.isWorkflowExecutionLockEnabled()) { if (!lock.acquireLock(lockId, timeToTryMs, leaseTimeMs, TimeUnit.MILLISECONDS)) { LOGGER.debug( "Thread {} failed to acquire lock to lockId {}.", Thread.currentThread().getId(), lockId); Monitors.recordAcquireLockUnsuccessful(); return false; } LOGGER.debug( "Thread {} acquired lock to lockId {}.", Thread.currentThread().getId(), lockId); } return true; } /** * Blocks until it gets the lock for workflowId * * @param lockId */ public void waitForLock(String lockId) { if (properties.isWorkflowExecutionLockEnabled()) { lock.acquireLock(lockId); LOGGER.debug( "Thread {} acquired lock to lockId {}.", Thread.currentThread().getId(), lockId); } } public void releaseLock(String lockId) { if (properties.isWorkflowExecutionLockEnabled()) { lock.releaseLock(lockId); LOGGER.debug( "Thread {} released lock to lockId {}.", Thread.currentThread().getId(), lockId); } } public void deleteLock(String lockId) { if (properties.isWorkflowExecutionLockEnabled()) { lock.deleteLock(lockId); LOGGER.debug("Thread {} deleted lockId {}.", Thread.currentThread().getId(), lockId); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/ExecutionService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.*; import com.netflix.conductor.common.run.*; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.common.utils.ExternalPayloadStorage.Operation; import com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; @Trace @Service public class ExecutionService { private static final Logger LOGGER = LoggerFactory.getLogger(ExecutionService.class); private final WorkflowExecutor workflowExecutor; private final ExecutionDAOFacade executionDAOFacade; private final QueueDAO queueDAO; private final ExternalPayloadStorage externalPayloadStorage; private final SystemTaskRegistry systemTaskRegistry; private final long queueTaskMessagePostponeSecs; private static final int MAX_POLL_TIMEOUT_MS = 5000; private static final int POLL_COUNT_ONE = 1; private static final int POLLING_TIMEOUT_IN_MS = 100; public ExecutionService( WorkflowExecutor workflowExecutor, ExecutionDAOFacade executionDAOFacade, QueueDAO queueDAO, ConductorProperties properties, ExternalPayloadStorage externalPayloadStorage, SystemTaskRegistry systemTaskRegistry) { this.workflowExecutor = workflowExecutor; this.executionDAOFacade = executionDAOFacade; this.queueDAO = queueDAO; this.externalPayloadStorage = externalPayloadStorage; this.queueTaskMessagePostponeSecs = properties.getTaskExecutionPostponeDuration().getSeconds(); this.systemTaskRegistry = systemTaskRegistry; } public Task poll(String taskType, String workerId) { return poll(taskType, workerId, null); } public Task poll(String taskType, String workerId, String domain) { List tasks = poll(taskType, workerId, domain, 1, 100); if (tasks.isEmpty()) { return null; } return tasks.get(0); } public List poll(String taskType, String workerId, int count, int timeoutInMilliSecond) { return poll(taskType, workerId, null, count, timeoutInMilliSecond); } public List poll( String taskType, String workerId, String domain, int count, int timeoutInMilliSecond) { if (timeoutInMilliSecond > MAX_POLL_TIMEOUT_MS) { throw new IllegalArgumentException( "Long Poll Timeout value cannot be more than 5 seconds"); } String queueName = QueueUtils.getQueueName(taskType, domain, null, null); List taskIds = new LinkedList<>(); List tasks = new LinkedList<>(); try { taskIds = queueDAO.pop(queueName, count, timeoutInMilliSecond); } catch (Exception e) { LOGGER.error( "Error polling for task: {} from worker: {} in domain: {}, count: {}", taskType, workerId, domain, count, e); Monitors.error(this.getClass().getCanonicalName(), "taskPoll"); Monitors.recordTaskPollError(taskType, domain, e.getClass().getSimpleName()); } for (String taskId : taskIds) { try { TaskModel taskModel = executionDAOFacade.getTaskModel(taskId); if (taskModel == null || taskModel.getStatus().isTerminal()) { // Remove taskId(s) without a valid Task/terminal state task from the queue queueDAO.remove(queueName, taskId); LOGGER.debug("Removed task: {} from the queue: {}", taskId, queueName); continue; } if (executionDAOFacade.exceedsInProgressLimit(taskModel)) { // Postpone this message, so that it would be available for poll again. queueDAO.postpone( queueName, taskId, taskModel.getWorkflowPriority(), queueTaskMessagePostponeSecs); LOGGER.debug( "Postponed task: {} in queue: {} by {} seconds", taskId, queueName, queueTaskMessagePostponeSecs); continue; } TaskDef taskDef = taskModel.getTaskDefinition().isPresent() ? taskModel.getTaskDefinition().get() : null; if (taskModel.getRateLimitPerFrequency() > 0 && executionDAOFacade.exceedsRateLimitPerFrequency(taskModel, taskDef)) { // Postpone this message, so that it would be available for poll again. queueDAO.postpone( queueName, taskId, taskModel.getWorkflowPriority(), queueTaskMessagePostponeSecs); LOGGER.debug( "RateLimit Execution limited for {}:{}, limit:{}", taskId, taskModel.getTaskDefName(), taskModel.getRateLimitPerFrequency()); continue; } taskModel.setStatus(TaskModel.Status.IN_PROGRESS); if (taskModel.getStartTime() == 0) { taskModel.setStartTime(System.currentTimeMillis()); Monitors.recordQueueWaitTime( taskModel.getTaskDefName(), taskModel.getQueueWaitTime()); } taskModel.setCallbackAfterSeconds( 0); // reset callbackAfterSeconds when giving the task to the worker taskModel.setWorkerId(workerId); taskModel.incrementPollCount(); executionDAOFacade.updateTask(taskModel); tasks.add(taskModel.toTask()); } catch (Exception e) { // db operation failed for dequeued message, re-enqueue with a delay LOGGER.warn( "DB operation failed for task: {}, postponing task in queue", taskId, e); Monitors.recordTaskPollError(taskType, domain, e.getClass().getSimpleName()); queueDAO.postpone(queueName, taskId, 0, queueTaskMessagePostponeSecs); } } executionDAOFacade.updateTaskLastPoll(taskType, domain, workerId); Monitors.recordTaskPoll(queueName); tasks.forEach(this::ackTaskReceived); return tasks; } public Task getLastPollTask(String taskType, String workerId, String domain) { List tasks = poll(taskType, workerId, domain, POLL_COUNT_ONE, POLLING_TIMEOUT_IN_MS); if (tasks.isEmpty()) { LOGGER.debug( "No Task available for the poll: /tasks/poll/{}?{}&{}", taskType, workerId, domain); return null; } Task task = tasks.get(0); ackTaskReceived(task); LOGGER.debug( "The Task {} being returned for /tasks/poll/{}?{}&{}", task, taskType, workerId, domain); return task; } public List getPollData(String taskType) { return executionDAOFacade.getTaskPollData(taskType); } public List getAllPollData() { try { return executionDAOFacade.getAllPollData(); } catch (UnsupportedOperationException uoe) { List allPollData = new ArrayList<>(); Map queueSizes = queueDAO.queuesDetail(); queueSizes .keySet() .forEach( queueName -> { try { if (!queueName.contains(QueueUtils.DOMAIN_SEPARATOR)) { allPollData.addAll( getPollData( QueueUtils.getQueueNameWithoutDomain( queueName))); } } catch (Exception e) { LOGGER.error("Unable to fetch all poll data!", e); } }); return allPollData; } } public void terminateWorkflow(String workflowId, String reason) { workflowExecutor.terminateWorkflow(workflowId, reason); } public void updateTask(TaskResult taskResult) { workflowExecutor.updateTask(taskResult); } public List getTasks(String taskType, String startKey, int count) { return executionDAOFacade.getTasksByName(taskType, startKey, count); } public Task getTask(String taskId) { return executionDAOFacade.getTask(taskId); } public Task getPendingTaskForWorkflow(String taskReferenceName, String workflowId) { return executionDAOFacade.getTasksForWorkflow(workflowId).stream() .filter(task -> !task.getStatus().isTerminal()) .filter(task -> task.getReferenceTaskName().equals(taskReferenceName)) .findFirst() // There can only be one task by a given reference name running at a // time. .orElse(null); } /** * This method removes the task from the un-acked Queue * * @param taskId: the taskId that needs to be updated and removed from the unacked queue * @return True in case of successful removal of the taskId from the un-acked queue */ public boolean ackTaskReceived(String taskId) { return Optional.ofNullable(getTask(taskId)).map(this::ackTaskReceived).orElse(false); } public boolean ackTaskReceived(Task task) { return queueDAO.ack(QueueUtils.getQueueName(task), task.getTaskId()); } public Map getTaskQueueSizes(List taskDefNames) { Map sizes = new HashMap<>(); for (String taskDefName : taskDefNames) { sizes.put(taskDefName, getTaskQueueSize(taskDefName)); } return sizes; } public Integer getTaskQueueSize(String queueName) { return queueDAO.getSize(queueName); } public void removeTaskFromQueue(String taskId) { Task task = getTask(taskId); if (task == null) { throw new NotFoundException("No such task found by taskId: %s", taskId); } queueDAO.remove(QueueUtils.getQueueName(task), taskId); } public int requeuePendingTasks(String taskType) { int count = 0; List tasks = getPendingTasksForTaskType(taskType); for (Task pending : tasks) { if (systemTaskRegistry.isSystemTask(pending.getTaskType())) { continue; } if (pending.getStatus().isTerminal()) { continue; } LOGGER.debug( "Requeuing Task: {} of taskType: {} in Workflow: {}", pending.getTaskId(), pending.getTaskType(), pending.getWorkflowInstanceId()); boolean pushed = requeue(pending); if (pushed) { count++; } } return count; } private boolean requeue(Task pending) { long callback = pending.getCallbackAfterSeconds(); if (callback < 0) { callback = 0; } queueDAO.remove(QueueUtils.getQueueName(pending), pending.getTaskId()); long now = System.currentTimeMillis(); callback = callback - ((now - pending.getUpdateTime()) / 1000); if (callback < 0) { callback = 0; } return queueDAO.pushIfNotExists( QueueUtils.getQueueName(pending), pending.getTaskId(), pending.getWorkflowPriority(), callback); } public List getWorkflowInstances( String workflowName, String correlationId, boolean includeClosed, boolean includeTasks) { List workflows = executionDAOFacade.getWorkflowsByCorrelationId(workflowName, correlationId, false); return workflows.stream() .parallel() .filter( workflow -> { if (includeClosed || workflow.getStatus() .equals(Workflow.WorkflowStatus.RUNNING)) { // including tasks for subset of workflows to increase performance if (includeTasks) { List tasks = executionDAOFacade.getTasksForWorkflow( workflow.getWorkflowId()); tasks.sort(Comparator.comparingInt(Task::getSeq)); workflow.setTasks(tasks); } return true; } else { return false; } }) .collect(Collectors.toList()); } public Workflow getExecutionStatus(String workflowId, boolean includeTasks) { return executionDAOFacade.getWorkflow(workflowId, includeTasks); } public List getRunningWorkflows(String workflowName, int version) { return executionDAOFacade.getRunningWorkflowIds(workflowName, version); } public void removeWorkflow(String workflowId, boolean archiveWorkflow) { executionDAOFacade.removeWorkflow(workflowId, archiveWorkflow); } public SearchResult search( String query, String freeText, int start, int size, List sortOptions) { return executionDAOFacade.searchWorkflowSummary(query, freeText, start, size, sortOptions); } public SearchResult searchV2( String query, String freeText, int start, int size, List sortOptions) { SearchResult result = executionDAOFacade.searchWorkflows(query, freeText, start, size, sortOptions); List workflows = result.getResults().stream() .parallel() .map( workflowId -> { try { return executionDAOFacade.getWorkflow(workflowId, false); } catch (Exception e) { LOGGER.error( "Error fetching workflow by id: {}", workflowId, e); return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); int missing = result.getResults().size() - workflows.size(); long totalHits = result.getTotalHits() - missing; return new SearchResult<>(totalHits, workflows); } public SearchResult searchWorkflowByTasks( String query, String freeText, int start, int size, List sortOptions) { SearchResult taskSummarySearchResult = searchTaskSummary(query, freeText, start, size, sortOptions); List workflowSummaries = taskSummarySearchResult.getResults().stream() .parallel() .map( taskSummary -> { try { String workflowId = taskSummary.getWorkflowId(); return new WorkflowSummary( executionDAOFacade.getWorkflow(workflowId, false)); } catch (Exception e) { LOGGER.error( "Error fetching workflow by id: {}", taskSummary.getWorkflowId(), e); return null; } }) .filter(Objects::nonNull) .distinct() .collect(Collectors.toList()); int missing = taskSummarySearchResult.getResults().size() - workflowSummaries.size(); long totalHits = taskSummarySearchResult.getTotalHits() - missing; return new SearchResult<>(totalHits, workflowSummaries); } public SearchResult searchWorkflowByTasksV2( String query, String freeText, int start, int size, List sortOptions) { SearchResult taskSummarySearchResult = searchTasks(query, freeText, start, size, sortOptions); List workflows = taskSummarySearchResult.getResults().stream() .parallel() .map( taskSummary -> { try { String workflowId = taskSummary.getWorkflowId(); return executionDAOFacade.getWorkflow(workflowId, false); } catch (Exception e) { LOGGER.error( "Error fetching workflow by id: {}", taskSummary.getWorkflowId(), e); return null; } }) .filter(Objects::nonNull) .distinct() .collect(Collectors.toList()); int missing = taskSummarySearchResult.getResults().size() - workflows.size(); long totalHits = taskSummarySearchResult.getTotalHits() - missing; return new SearchResult<>(totalHits, workflows); } public SearchResult searchTasks( String query, String freeText, int start, int size, List sortOptions) { SearchResult result = executionDAOFacade.searchTasks(query, freeText, start, size, sortOptions); List workflows = result.getResults().stream() .parallel() .map( task -> { try { return new TaskSummary(executionDAOFacade.getTask(task)); } catch (Exception e) { LOGGER.error("Error fetching task by id: {}", task, e); return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); int missing = result.getResults().size() - workflows.size(); long totalHits = result.getTotalHits() - missing; return new SearchResult<>(totalHits, workflows); } public SearchResult searchTaskSummary( String query, String freeText, int start, int size, List sortOptions) { return executionDAOFacade.searchTaskSummary(query, freeText, start, size, sortOptions); } public SearchResult getSearchTasks( String query, String freeText, int start, /*@Max(value = MAX_SEARCH_SIZE, message = "Cannot return more than {value} workflows." + " Please use pagination.")*/ int size, String sortString) { return searchTaskSummary( query, freeText, start, size, Utils.convertStringToList(sortString)); } public SearchResult getSearchTasksV2( String query, String freeText, int start, int size, String sortString) { SearchResult result = executionDAOFacade.searchTasks( query, freeText, start, size, Utils.convertStringToList(sortString)); List tasks = result.getResults().stream() .parallel() .map( task -> { try { return executionDAOFacade.getTask(task); } catch (Exception e) { LOGGER.error("Error fetching task by id: {}", task, e); return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); int missing = result.getResults().size() - tasks.size(); long totalHits = result.getTotalHits() - missing; return new SearchResult<>(totalHits, tasks); } public List getPendingTasksForTaskType(String taskType) { return executionDAOFacade.getPendingTasksForTaskType(taskType); } public boolean addEventExecution(EventExecution eventExecution) { return executionDAOFacade.addEventExecution(eventExecution); } public void removeEventExecution(EventExecution eventExecution) { executionDAOFacade.removeEventExecution(eventExecution); } public void updateEventExecution(EventExecution eventExecution) { executionDAOFacade.updateEventExecution(eventExecution); } /** * @param queue Name of the registered queueDAO * @param msg Message */ public void addMessage(String queue, Message msg) { executionDAOFacade.addMessage(queue, msg); } /** * Adds task logs * * @param taskId Id of the task * @param log logs */ public void log(String taskId, String log) { TaskExecLog executionLog = new TaskExecLog(); executionLog.setTaskId(taskId); executionLog.setLog(log); executionLog.setCreatedTime(System.currentTimeMillis()); executionDAOFacade.addTaskExecLog(Collections.singletonList(executionLog)); } /** * @param taskId Id of the task for which to retrieve logs * @return Execution Logs (logged by the worker) */ public List getTaskLogs(String taskId) { return executionDAOFacade.getTaskExecutionLogs(taskId); } /** * Get external uri for the payload * * @param path the path for which the external storage location is to be populated * @param operation the type of {@link Operation} to be performed * @param type the {@link PayloadType} at the external uri * @return the external uri at which the payload is stored/to be stored */ public ExternalStorageLocation getExternalStorageLocation( String path, String operation, String type) { try { ExternalPayloadStorage.Operation payloadOperation = ExternalPayloadStorage.Operation.valueOf(StringUtils.upperCase(operation)); ExternalPayloadStorage.PayloadType payloadType = ExternalPayloadStorage.PayloadType.valueOf(StringUtils.upperCase(type)); return externalPayloadStorage.getLocation(payloadOperation, payloadType, path); } catch (Exception e) { String errorMsg = String.format( "Invalid input - Operation: %s, PayloadType: %s", operation, type); LOGGER.error(errorMsg); throw new IllegalArgumentException(errorMsg); } } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/MetadataService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import java.util.Map; import java.util.Optional; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDefSummary; import com.netflix.conductor.common.model.BulkResponse; @Validated public interface MetadataService { /** * @param taskDefinitions Task Definitions to register */ void registerTaskDef( @NotNull(message = "TaskDefList cannot be empty or null") @Size(min = 1, message = "TaskDefList is empty") List<@Valid TaskDef> taskDefinitions); /** * @param taskDefinition Task Definition to be updated */ void updateTaskDef(@NotNull(message = "TaskDef cannot be null") @Valid TaskDef taskDefinition); /** * @param taskType Remove task definition */ void unregisterTaskDef(@NotEmpty(message = "TaskName cannot be null or empty") String taskType); /** * @return List of all the registered tasks */ List getTaskDefs(); /** * @param taskType Task to retrieve * @return Task Definition */ TaskDef getTaskDef(@NotEmpty(message = "TaskType cannot be null or empty") String taskType); /** * @param def Workflow definition to be updated */ void updateWorkflowDef(@NotNull(message = "WorkflowDef cannot be null") @Valid WorkflowDef def); /** * @param workflowDefList Workflow definitions to be updated. */ BulkResponse updateWorkflowDef( @NotNull(message = "WorkflowDef list name cannot be null or empty") @Size(min = 1, message = "WorkflowDefList is empty") List<@NotNull(message = "WorkflowDef cannot be null") @Valid WorkflowDef> workflowDefList); /** * @param name Name of the workflow to retrieve * @param version Optional. Version. If null, then retrieves the latest * @return Workflow definition */ WorkflowDef getWorkflowDef( @NotEmpty(message = "Workflow name cannot be null or empty") String name, Integer version); /** * @param name Name of the workflow to retrieve * @return Latest version of the workflow definition */ Optional getLatestWorkflow( @NotEmpty(message = "Workflow name cannot be null or empty") String name); /** * @return Returns all workflow defs (all versions) */ List getWorkflowDefs(); /** * @return Returns workflow names and versions only (no definition bodies) */ Map> getWorkflowNamesAndVersions(); void registerWorkflowDef( @NotNull(message = "WorkflowDef cannot be null") @Valid WorkflowDef workflowDef); /** * Validates a {@link WorkflowDef}. * * @param workflowDef The {@link WorkflowDef} object. */ default void validateWorkflowDef( @NotNull(message = "WorkflowDef cannot be null") @Valid WorkflowDef workflowDef) { // do nothing, WorkflowDef is annotated with @Valid and calling this method will validate it } /** * @param name Name of the workflow definition to be removed * @param version Version of the workflow definition to be removed */ void unregisterWorkflowDef( @NotEmpty(message = "Workflow name cannot be null or empty") String name, @NotNull(message = "Version cannot be null") Integer version); /** * @param eventHandler Event handler to be added. Will throw an exception if an event handler * already exists with the name */ void addEventHandler( @NotNull(message = "EventHandler cannot be null") @Valid EventHandler eventHandler); /** * @param eventHandler Event handler to be updated. */ void updateEventHandler( @NotNull(message = "EventHandler cannot be null") @Valid EventHandler eventHandler); /** * @param name Removes the event handler from the system */ void removeEventHandlerStatus( @NotEmpty(message = "EventName cannot be null or empty") String name); /** * @return All the event handlers registered in the system */ List getAllEventHandlers(); /** * @param event name of the event * @param activeOnly if true, returns only the active handlers * @return Returns the list of all the event handlers for a given event */ List getEventHandlersForEvent( @NotEmpty(message = "EventName cannot be null or empty") String event, boolean activeOnly); List getWorkflowDefsLatestVersions(); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/MetadataServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.common.constraints.OwnerEmailMandatoryConstraint; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDefSummary; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.validations.ValidationContext; @Service public class MetadataServiceImpl implements MetadataService { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataServiceImpl.class); private final MetadataDAO metadataDAO; private final EventHandlerDAO eventHandlerDAO; public MetadataServiceImpl( MetadataDAO metadataDAO, EventHandlerDAO eventHandlerDAO, ConductorProperties properties) { this.metadataDAO = metadataDAO; this.eventHandlerDAO = eventHandlerDAO; ValidationContext.initialize(metadataDAO); OwnerEmailMandatoryConstraint.WorkflowTaskValidValidator.setOwnerEmailMandatory( properties.isOwnerEmailMandatory()); } /** * @param taskDefinitions Task Definitions to register */ public void registerTaskDef(List taskDefinitions) { for (TaskDef taskDefinition : taskDefinitions) { taskDefinition.setCreatedBy(WorkflowContext.get().getClientApp()); taskDefinition.setCreateTime(System.currentTimeMillis()); taskDefinition.setUpdatedBy(null); taskDefinition.setUpdateTime(null); metadataDAO.createTaskDef(taskDefinition); } } @Override public void validateWorkflowDef(WorkflowDef workflowDef) { // do nothing, WorkflowDef is annotated with @Valid and calling this method will validate it } /** * @param taskDefinition Task Definition to be updated */ public void updateTaskDef(TaskDef taskDefinition) { TaskDef existing = metadataDAO.getTaskDef(taskDefinition.getName()); if (existing == null) { throw new NotFoundException("No such task by name %s", taskDefinition.getName()); } taskDefinition.setUpdatedBy(WorkflowContext.get().getClientApp()); taskDefinition.setUpdateTime(System.currentTimeMillis()); metadataDAO.updateTaskDef(taskDefinition); } /** * @param taskType Remove task definition */ public void unregisterTaskDef(String taskType) { metadataDAO.removeTaskDef(taskType); } /** * @return List of all the registered tasks */ public List getTaskDefs() { return metadataDAO.getAllTaskDefs(); } /** * @param taskType Task to retrieve * @return Task Definition */ public TaskDef getTaskDef(String taskType) { TaskDef taskDef = metadataDAO.getTaskDef(taskType); if (taskDef == null) { throw new NotFoundException("No such taskType found by name: %s", taskType); } return taskDef; } /** * @param workflowDef Workflow definition to be updated */ public void updateWorkflowDef(WorkflowDef workflowDef) { workflowDef.setUpdateTime(System.currentTimeMillis()); metadataDAO.updateWorkflowDef(workflowDef); } /** * @param workflowDefList Workflow definitions to be updated. */ public BulkResponse updateWorkflowDef(List workflowDefList) { BulkResponse bulkResponse = new BulkResponse(); for (WorkflowDef workflowDef : workflowDefList) { try { updateWorkflowDef(workflowDef); bulkResponse.appendSuccessResponse(workflowDef.getName()); } catch (Exception e) { LOGGER.error("bulk update workflow def failed, name {} ", workflowDef.getName(), e); bulkResponse.appendFailedResponse(workflowDef.getName(), e.getMessage()); } } return bulkResponse; } /** * @param name Name of the workflow to retrieve * @param version Optional. Version. If null, then retrieves the latest * @return Workflow definition */ public WorkflowDef getWorkflowDef(String name, Integer version) { Optional workflowDef; if (version == null) { workflowDef = metadataDAO.getLatestWorkflowDef(name); } else { workflowDef = metadataDAO.getWorkflowDef(name, version); } return workflowDef.orElseThrow( () -> new NotFoundException( "No such workflow found by name: %s, version: %d", name, version)); } /** * @param name Name of the workflow to retrieve * @return Latest version of the workflow definition */ public Optional getLatestWorkflow(String name) { return metadataDAO.getLatestWorkflowDef(name); } public List getWorkflowDefs() { return metadataDAO.getAllWorkflowDefs(); } public void registerWorkflowDef(WorkflowDef workflowDef) { workflowDef.setCreateTime(System.currentTimeMillis()); metadataDAO.createWorkflowDef(workflowDef); } /** * @param name Name of the workflow definition to be removed * @param version Version of the workflow definition to be removed */ public void unregisterWorkflowDef(String name, Integer version) { metadataDAO.removeWorkflowDef(name, version); } /** * @param eventHandler Event handler to be added. Will throw an exception if an event handler * already exists with the name */ public void addEventHandler(EventHandler eventHandler) { eventHandlerDAO.addEventHandler(eventHandler); } /** * @param eventHandler Event handler to be updated. */ public void updateEventHandler(EventHandler eventHandler) { eventHandlerDAO.updateEventHandler(eventHandler); } /** * @param name Removes the event handler from the system */ public void removeEventHandlerStatus(String name) { eventHandlerDAO.removeEventHandler(name); } /** * @return All the event handlers registered in the system */ public List getAllEventHandlers() { return eventHandlerDAO.getAllEventHandlers(); } /** * @param event name of the event * @param activeOnly if true, returns only the active handlers * @return Returns the list of all the event handlers for a given event */ public List getEventHandlersForEvent(String event, boolean activeOnly) { return eventHandlerDAO.getEventHandlersForEvent(event, activeOnly); } @Override public List getWorkflowDefsLatestVersions() { return metadataDAO.getAllWorkflowDefsLatestVersions(); } public Map> getWorkflowNamesAndVersions() { List workflowDefs = metadataDAO.getAllWorkflowDefs(); Map> retval = new HashMap<>(); for (WorkflowDef def : workflowDefs) { String workflowName = def.getName(); WorkflowDefSummary summary = fromWorkflowDef(def); retval.putIfAbsent(workflowName, new TreeSet()); TreeSet versions = retval.get(workflowName); versions.add(summary); } return retval; } private WorkflowDefSummary fromWorkflowDef(WorkflowDef def) { WorkflowDefSummary summary = new WorkflowDefSummary(); summary.setName(def.getName()); summary.setVersion(def.getVersion()); summary.setCreateTime(def.getCreateTime()); return summary; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/TaskService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import java.util.Map; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; @Validated public interface TaskService { /** * Poll for a task of a certain type. * * @param taskType Task name * @param workerId Id of the workflow * @param domain Domain of the workflow * @return polled {@link Task} */ Task poll( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType, String workerId, String domain); /** * Batch Poll for a task of a certain type. * * @param taskType Task Name * @param workerId Id of the workflow * @param domain Domain of the workflow * @param count Number of tasks * @param timeout Timeout for polling in milliseconds * @return list of {@link Task} */ List batchPoll( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType, String workerId, String domain, Integer count, Integer timeout); /** * Get in progress tasks. The results are paginated. * * @param taskType Task Name * @param startKey Start index of pagination * @param count Number of entries * @return list of {@link Task} */ List getTasks( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType, String startKey, Integer count); /** * Get in progress task for a given workflow id. * * @param workflowId Id of the workflow * @param taskReferenceName Task reference name. * @return instance of {@link Task} */ Task getPendingTaskForWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, @NotEmpty(message = "TaskReferenceName cannot be null or empty.") String taskReferenceName); /** * Updates a task. * * @param taskResult Instance of {@link TaskResult} * @return task Id of the updated task. */ String updateTask( @NotNull(message = "TaskResult cannot be null or empty.") @Valid TaskResult taskResult); /** * Ack Task is received. * * @param taskId Id of the task * @param workerId Id of the worker * @return `true|false` if task if received or not */ String ackTaskReceived( @NotEmpty(message = "TaskId cannot be null or empty.") String taskId, String workerId); /** * Ack Task is received. * * @param taskId Id of the task * @return `true|false` if task if received or not */ boolean ackTaskReceived(@NotEmpty(message = "TaskId cannot be null or empty.") String taskId); /** * Log Task Execution Details. * * @param taskId Id of the task * @param log Details you want to log */ void log(@NotEmpty(message = "TaskId cannot be null or empty.") String taskId, String log); /** * Get Task Execution Logs. * * @param taskId Id of the task. * @return list of {@link TaskExecLog} */ List getTaskLogs( @NotEmpty(message = "TaskId cannot be null or empty.") String taskId); /** * Get task by Id. * * @param taskId Id of the task. * @return instance of {@link Task} */ Task getTask(@NotEmpty(message = "TaskId cannot be null or empty.") String taskId); /** * Remove Task from a Task type queue. * * @param taskType Task Name * @param taskId ID of the task */ void removeTaskFromQueue( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType, @NotEmpty(message = "TaskId cannot be null or empty.") String taskId); /** * Remove Task from a Task type queue. * * @param taskId ID of the task */ void removeTaskFromQueue(@NotEmpty(message = "TaskId cannot be null or empty.") String taskId); /** * Get Task type queue sizes. * * @param taskTypes List of task types. * @return map of task type as Key and queue size as value. */ Map getTaskQueueSizes(List taskTypes); /** * Get the queue size for a Task Type. The input can optionally include domain, * isolationGroupId and executionNamespace. * * @return */ Integer getTaskQueueSize( String taskType, String domain, String isolationGroupId, String executionNamespace); /** * Get the details about each queue. * * @return map of queue details. */ Map>> allVerbose(); /** * Get the details about each queue. * * @return map of details about each queue. */ Map getAllQueueDetails(); /** * Get the last poll data for a given task type. * * @param taskType Task Name * @return list of {@link PollData} */ List getPollData( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType); /** * Get the last poll data for all task types. * * @return list of {@link PollData} */ List getAllPollData(); /** * Requeue pending tasks. * * @param taskType Task name. * @return number of tasks requeued. */ String requeuePendingTask( @NotEmpty(message = "TaskType cannot be null or empty.") String taskType); /** * Search for tasks based in payload and other parameters. Use sort options as ASC or DESC e.g. * sort=name or sort=workflowId. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult search( int start, int size, String sort, String freeText, String query); /** * Search for tasks based in payload and other parameters. Use sort options as ASC or DESC e.g. * sort=name or sort=workflowId. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchV2(int start, int size, String sort, String freeText, String query); /** * Get the external storage location where the task output payload is stored/to be stored * * @param path the path for which the external storage location is to be populated * @param operation the operation to be performed (read or write) * @param payloadType the type of payload (input or output) * @return {@link ExternalStorageLocation} containing the uri and the path to the payload is * stored in external storage */ ExternalStorageLocation getExternalStorageLocation( String path, String operation, String payloadType); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/TaskServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Audit; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; @Audit @Trace @Service public class TaskServiceImpl implements TaskService { private static final Logger LOGGER = LoggerFactory.getLogger(TaskServiceImpl.class); private final ExecutionService executionService; private final QueueDAO queueDAO; public TaskServiceImpl(ExecutionService executionService, QueueDAO queueDAO) { this.executionService = executionService; this.queueDAO = queueDAO; } /** * Poll for a task of a certain type. * * @param taskType Task name * @param workerId id of the workflow * @param domain Domain of the workflow * @return polled {@link Task} */ public Task poll(String taskType, String workerId, String domain) { LOGGER.debug("Task being polled: /tasks/poll/{}?{}&{}", taskType, workerId, domain); Task task = executionService.getLastPollTask(taskType, workerId, domain); if (task != null) { LOGGER.debug( "The Task {} being returned for /tasks/poll/{}?{}&{}", task, taskType, workerId, domain); } Monitors.recordTaskPollCount(taskType, domain, 1); return task; } /** * Batch Poll for a task of a certain type. * * @param taskType Task Name * @param workerId id of the workflow * @param domain Domain of the workflow * @param count Number of tasks * @param timeout Timeout for polling in milliseconds * @return list of {@link Task} */ public List batchPoll( String taskType, String workerId, String domain, Integer count, Integer timeout) { List polledTasks = executionService.poll(taskType, workerId, domain, count, timeout); LOGGER.debug( "The Tasks {} being returned for /tasks/poll/{}?{}&{}", polledTasks.stream().map(Task::getTaskId).collect(Collectors.toList()), taskType, workerId, domain); Monitors.recordTaskPollCount(taskType, domain, polledTasks.size()); return polledTasks; } /** * Get in progress tasks. The results are paginated. * * @param taskType Task Name * @param startKey Start index of pagination * @param count Number of entries * @return list of {@link Task} */ public List getTasks(String taskType, String startKey, Integer count) { return executionService.getTasks(taskType, startKey, count); } /** * Get in progress task for a given workflow id. * * @param workflowId id of the workflow * @param taskReferenceName Task reference name. * @return instance of {@link Task} */ public Task getPendingTaskForWorkflow(String workflowId, String taskReferenceName) { return executionService.getPendingTaskForWorkflow(taskReferenceName, workflowId); } /** * Updates a task. * * @param taskResult Instance of {@link TaskResult} * @return task Id of the updated task. */ public String updateTask(TaskResult taskResult) { LOGGER.debug( "Update Task: {} with callback time: {}", taskResult, taskResult.getCallbackAfterSeconds()); executionService.updateTask(taskResult); LOGGER.debug( "Task: {} updated successfully with callback time: {}", taskResult, taskResult.getCallbackAfterSeconds()); return taskResult.getTaskId(); } /** * Ack Task is received. * * @param taskId id of the task * @param workerId id of the worker * @return `true|false` if task is received or not */ public String ackTaskReceived(String taskId, String workerId) { LOGGER.debug("Ack received for task: {} from worker: {}", taskId, workerId); return String.valueOf(ackTaskReceived(taskId)); } /** * Ack Task is received. * * @param taskId id of the task * @return `true|false` if task is received or not */ public boolean ackTaskReceived(String taskId) { LOGGER.debug("Ack received for task: {}", taskId); AtomicBoolean ackResult = new AtomicBoolean(false); try { ackResult.set(executionService.ackTaskReceived(taskId)); } catch (Exception e) { // Fail the task and let decide reevaluate the workflow, thereby preventing workflow // being stuck from transient ack errors. String errorMsg = String.format("Error when trying to ack task %s", taskId); LOGGER.error(errorMsg, e); Task task = executionService.getTask(taskId); Monitors.recordAckTaskError(task.getTaskType()); failTask(task, errorMsg); ackResult.set(false); } return ackResult.get(); } /** Updates the task with FAILED status; On exception, fails the workflow. */ private void failTask(Task task, String errorMsg) { try { TaskResult taskResult = new TaskResult(); taskResult.setStatus(TaskResult.Status.FAILED); taskResult.setTaskId(task.getTaskId()); taskResult.setWorkflowInstanceId(task.getWorkflowInstanceId()); taskResult.setReasonForIncompletion(errorMsg); executionService.updateTask(taskResult); } catch (Exception e) { LOGGER.error( "Unable to fail task: {} in workflow: {}", task.getTaskId(), task.getWorkflowInstanceId(), e); executionService.terminateWorkflow( task.getWorkflowInstanceId(), "Failed to ack task: " + task.getTaskId()); } } /** * Log Task Execution Details. * * @param taskId id of the task * @param log Details you want to log */ public void log(String taskId, String log) { executionService.log(taskId, log); } /** * Get Task Execution Logs. * * @param taskId id of the task. * @return list of {@link TaskExecLog} */ public List getTaskLogs(String taskId) { return executionService.getTaskLogs(taskId); } /** * Get task by Id. * * @param taskId id of the task. * @return instance of {@link Task} */ public Task getTask(String taskId) { return executionService.getTask(taskId); } /** * Remove Task from a Task type queue. * * @param taskType Task Name * @param taskId ID of the task */ public void removeTaskFromQueue(String taskType, String taskId) { executionService.removeTaskFromQueue(taskId); } /** * Remove Task from a Task type queue. * * @param taskId ID of the task */ public void removeTaskFromQueue(String taskId) { executionService.removeTaskFromQueue(taskId); } /** * Get Task type queue sizes. * * @param taskTypes List of task types. * @return map of task type as Key and queue size as value. */ public Map getTaskQueueSizes(List taskTypes) { return executionService.getTaskQueueSizes(taskTypes); } @Override public Integer getTaskQueueSize( String taskType, String domain, String isolationGroupId, String executionNamespace) { String queueName = QueueUtils.getQueueName( taskType, StringUtils.trimToNull(domain), StringUtils.trimToNull(isolationGroupId), StringUtils.trimToNull(executionNamespace)); return executionService.getTaskQueueSize(queueName); } /** * Get the details about each queue. * * @return map of queue details. */ public Map>> allVerbose() { return queueDAO.queuesDetailVerbose(); } /** * Get the details about each queue. * * @return map of details about each queue. */ public Map getAllQueueDetails() { return queueDAO.queuesDetail().entrySet().stream() .sorted(Entry.comparingByKey()) .collect( Collectors.toMap( Entry::getKey, Entry::getValue, (v1, v2) -> v1, LinkedHashMap::new)); } /** * Get the last poll data for a given task type. * * @param taskType Task Name * @return list of {@link PollData} */ public List getPollData(String taskType) { return executionService.getPollData(taskType); } /** * Get the last poll data for all task types. * * @return list of {@link PollData} */ public List getAllPollData() { return executionService.getAllPollData(); } /** * Requeue pending tasks. * * @param taskType Task name. * @return number of tasks requeued. */ public String requeuePendingTask(String taskType) { return String.valueOf(executionService.requeuePendingTasks(taskType)); } /** * Search for tasks based in payload and other parameters. Use sort options as ASC or DESC e.g. * sort=name or sort=workflowId. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult search( int start, int size, String sort, String freeText, String query) { return executionService.getSearchTasks(query, freeText, start, size, sort); } /** * Search for tasks based in payload and other parameters. Use sort options as ASC or DESC e.g. * sort=name or sort=workflowId. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchV2( int start, int size, String sort, String freeText, String query) { return executionService.getSearchTasksV2(query, freeText, start, size, sort); } /** * Get the external storage location where the task output payload is stored/to be stored * * @param path the path for which the external storage location is to be populated * @param operation the operation to be performed (read or write) * @param type the type of payload (input or output) * @return {@link ExternalStorageLocation} containing the uri and the path to the payload is * stored in external storage */ public ExternalStorageLocation getExternalStorageLocation( String path, String operation, String type) { return executionService.getExternalStorageLocation(path, operation, type); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/WorkflowBulkService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.Size; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.model.BulkResponse; @Validated public interface WorkflowBulkService { int MAX_REQUEST_ITEMS = 1000; BulkResponse pauseWorkflow( @NotEmpty(message = "WorkflowIds list cannot be null.") @Size( max = MAX_REQUEST_ITEMS, message = "Cannot process more than {max} workflows. Please use multiple requests.") List workflowIds); BulkResponse resumeWorkflow( @NotEmpty(message = "WorkflowIds list cannot be null.") @Size( max = MAX_REQUEST_ITEMS, message = "Cannot process more than {max} workflows. Please use multiple requests.") List workflowIds); BulkResponse restart( @NotEmpty(message = "WorkflowIds list cannot be null.") @Size( max = MAX_REQUEST_ITEMS, message = "Cannot process more than {max} workflows. Please use multiple requests.") List workflowIds, boolean useLatestDefinitions); BulkResponse retry( @NotEmpty(message = "WorkflowIds list cannot be null.") @Size( max = MAX_REQUEST_ITEMS, message = "Cannot process more than {max} workflows. Please use multiple requests.") List workflowIds); BulkResponse terminate( @NotEmpty(message = "WorkflowIds list cannot be null.") @Size( max = MAX_REQUEST_ITEMS, message = "Cannot process more than {max} workflows. Please use multiple requests.") List workflowIds, String reason); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/WorkflowBulkServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Audit; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.core.execution.WorkflowExecutor; @Audit @Trace @Service public class WorkflowBulkServiceImpl implements WorkflowBulkService { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowBulkService.class); private final WorkflowExecutor workflowExecutor; public WorkflowBulkServiceImpl(WorkflowExecutor workflowExecutor) { this.workflowExecutor = workflowExecutor; } /** * Pause the list of workflows. * * @param workflowIds - list of workflow Ids to perform pause operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ public BulkResponse pauseWorkflow(List workflowIds) { BulkResponse bulkResponse = new BulkResponse(); for (String workflowId : workflowIds) { try { workflowExecutor.pauseWorkflow(workflowId); bulkResponse.appendSuccessResponse(workflowId); } catch (Exception e) { LOGGER.error( "bulk pauseWorkflow exception, workflowId {}, message: {} ", workflowId, e.getMessage(), e); bulkResponse.appendFailedResponse(workflowId, e.getMessage()); } } return bulkResponse; } /** * Resume the list of workflows. * * @param workflowIds - list of workflow Ids to perform resume operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ public BulkResponse resumeWorkflow(List workflowIds) { BulkResponse bulkResponse = new BulkResponse(); for (String workflowId : workflowIds) { try { workflowExecutor.resumeWorkflow(workflowId); bulkResponse.appendSuccessResponse(workflowId); } catch (Exception e) { LOGGER.error( "bulk resumeWorkflow exception, workflowId {}, message: {} ", workflowId, e.getMessage(), e); bulkResponse.appendFailedResponse(workflowId, e.getMessage()); } } return bulkResponse; } /** * Restart the list of workflows. * * @param workflowIds - list of workflow Ids to perform restart operation on * @param useLatestDefinitions if true, use latest workflow and task definitions upon restart * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ public BulkResponse restart(List workflowIds, boolean useLatestDefinitions) { BulkResponse bulkResponse = new BulkResponse(); for (String workflowId : workflowIds) { try { workflowExecutor.restart(workflowId, useLatestDefinitions); bulkResponse.appendSuccessResponse(workflowId); } catch (Exception e) { LOGGER.error( "bulk restart exception, workflowId {}, message: {} ", workflowId, e.getMessage(), e); bulkResponse.appendFailedResponse(workflowId, e.getMessage()); } } return bulkResponse; } /** * Retry the last failed task for each workflow from the list. * * @param workflowIds - list of workflow Ids to perform retry operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ public BulkResponse retry(List workflowIds) { BulkResponse bulkResponse = new BulkResponse(); for (String workflowId : workflowIds) { try { workflowExecutor.retry(workflowId, false); bulkResponse.appendSuccessResponse(workflowId); } catch (Exception e) { LOGGER.error( "bulk retry exception, workflowId {}, message: {} ", workflowId, e.getMessage(), e); bulkResponse.appendFailedResponse(workflowId, e.getMessage()); } } return bulkResponse; } /** * Terminate workflows execution. * * @param workflowIds - list of workflow Ids to perform terminate operation on * @param reason - description to be specified for the terminated workflow for future * references. * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ public BulkResponse terminate(List workflowIds, String reason) { BulkResponse bulkResponse = new BulkResponse(); for (String workflowId : workflowIds) { try { workflowExecutor.terminateWorkflow(workflowId, reason); bulkResponse.appendSuccessResponse(workflowId); } catch (Exception e) { LOGGER.error( "bulk terminate exception, workflowId {}, message: {} ", workflowId, e.getMessage(), e); bulkResponse.appendFailedResponse(workflowId, e.getMessage()); } } return bulkResponse; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/WorkflowService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import java.util.Map; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.springframework.validation.annotation.Validated; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; @Validated public interface WorkflowService { /** * Start a new workflow with StartWorkflowRequest, which allows task to be executed in a domain. * * @param startWorkflowRequest StartWorkflow request for the workflow you want to start. * @return the id of the workflow instance that can be use for tracking. */ String startWorkflow( @NotNull(message = "StartWorkflowRequest cannot be null") @Valid StartWorkflowRequest startWorkflowRequest); /** * Start a new workflow. Returns the ID of the workflow instance that can be later used for * tracking. * * @param name Name of the workflow you want to start. * @param version Version of the workflow you want to start. * @param correlationId CorrelationID of the workflow you want to start. * @param priority Priority of the workflow you want to start. * @param input Input to the workflow you want to start. * @return the id of the workflow instance that can be use for tracking. */ String startWorkflow( @NotEmpty(message = "Workflow name cannot be null or empty") String name, Integer version, String correlationId, @Min(value = 0, message = "0 is the minimum priority value") @Max(value = 99, message = "99 is the maximum priority value") Integer priority, Map input); /** * Start a new workflow. Returns the ID of the workflow instance that can be later used for * tracking. * * @param name Name of the workflow you want to start. * @param version Version of the workflow you want to start. * @param correlationId CorrelationID of the workflow you want to start. * @param priority Priority of the workflow you want to start. * @param input Input to the workflow you want to start. * @param externalInputPayloadStoragePath * @param taskToDomain * @param workflowDef - workflow definition * @return the id of the workflow instance that can be use for tracking. */ String startWorkflow( String name, Integer version, String correlationId, Integer priority, Map input, String externalInputPayloadStoragePath, Map taskToDomain, WorkflowDef workflowDef); /** * Lists workflows for the given correlation id. * * @param name Name of the workflow. * @param correlationId CorrelationID of the workflow you want to list. * @param includeClosed IncludeClosed workflow which are not running. * @param includeTasks Includes tasks associated with workflows. * @return a list of {@link Workflow} */ List getWorkflows( @NotEmpty(message = "Workflow name cannot be null or empty") String name, String correlationId, boolean includeClosed, boolean includeTasks); /** * Lists workflows for the given correlation id. * * @param name Name of the workflow. * @param includeClosed CorrelationID of the workflow you want to start. * @param includeTasks IncludeClosed workflow which are not running. * @param correlationIds Includes tasks associated with workflows. * @return a {@link Map} of {@link String} as key and a list of {@link Workflow} as value */ Map> getWorkflows( @NotEmpty(message = "Workflow name cannot be null or empty") String name, boolean includeClosed, boolean includeTasks, List correlationIds); /** * Gets the workflow by workflow Id. * * @param workflowId Id of the workflow. * @param includeTasks Includes tasks associated with workflow. * @return an instance of {@link Workflow} */ Workflow getExecutionStatus( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, boolean includeTasks); /** * Removes the workflow from the system. * * @param workflowId WorkflowID of the workflow you want to remove from system. * @param archiveWorkflow Archives the workflow and associated tasks instead of removing them. */ void deleteWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, boolean archiveWorkflow); /** * Retrieves all the running workflows. * * @param workflowName Name of the workflow. * @param version Version of the workflow. * @param startTime Starttime of the workflow. * @param endTime EndTime of the workflow * @return a list of workflow Ids. */ List getRunningWorkflows( @NotEmpty(message = "Workflow name cannot be null or empty.") String workflowName, Integer version, Long startTime, Long endTime); /** * Starts the decision task for a workflow. * * @param workflowId WorkflowId of the workflow. */ void decideWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Pauses the workflow given a worklfowId. * * @param workflowId WorkflowId of the workflow. */ void pauseWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Resumes the workflow. * * @param workflowId WorkflowId of the workflow. */ void resumeWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Skips a given task from a current running workflow. * * @param workflowId WorkflowId of the workflow. * @param taskReferenceName The task reference name. * @param skipTaskRequest {@link SkipTaskRequest} for task you want to skip. */ void skipTaskFromWorkflow( @NotEmpty(message = "WorkflowId name cannot be null or empty.") String workflowId, @NotEmpty(message = "TaskReferenceName cannot be null or empty.") String taskReferenceName, SkipTaskRequest skipTaskRequest); /** * Reruns the workflow from a specific task. * * @param workflowId WorkflowId of the workflow you want to rerun. * @param request (@link RerunWorkflowRequest) for the workflow. * @return WorkflowId of the rerun workflow. */ String rerunWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, @NotNull(message = "RerunWorkflowRequest cannot be null.") RerunWorkflowRequest request); /** * Restarts a completed workflow. * * @param workflowId WorkflowId of the workflow. * @param useLatestDefinitions if true, use the latest workflow and task definitions upon * restart */ void restartWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, boolean useLatestDefinitions); /** * Retries the last failed task. * * @param workflowId WorkflowId of the workflow. */ void retryWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, boolean resumeSubworkflowTasks); /** * Resets callback times of all non-terminal SIMPLE tasks to 0. * * @param workflowId WorkflowId of the workflow. */ void resetWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId); /** * Terminate workflow execution. * * @param workflowId WorkflowId of the workflow. * @param reason Reason for terminating the workflow. */ void terminateWorkflow( @NotEmpty(message = "WorkflowId cannot be null or empty.") String workflowId, String reason); /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflows( int start, @Max( value = 5_000, message = "Cannot return more than {value} workflows. Please use pagination.") int size, String sort, String freeText, String query); /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsV2( int start, @Max( value = 5_000, message = "Cannot return more than {value} workflows. Please use pagination.") int size, String sort, String freeText, String query); /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflows( int start, @Max( value = 5_000, message = "Cannot return more than {value} workflows. Please use pagination.") int size, List sort, String freeText, String query); /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsV2( int start, @Max( value = 5_000, message = "Cannot return more than {value} workflows. Please use pagination.") int size, List sort, String freeText, String query); /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsByTasks( int start, int size, String sort, String freeText, String query); /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsByTasksV2( int start, int size, String sort, String freeText, String query); /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsByTasks( int start, int size, List sort, String freeText, String query); /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ SearchResult searchWorkflowsByTasksV2( int start, int size, List sort, String freeText, String query); /** * Get the external storage location where the workflow input payload is stored/to be stored * * @param path the path for which the external storage location is to be populated * @param operation the operation to be performed (read or write) * @param payloadType the type of payload (input or output) * @return {@link ExternalStorageLocation} containing the uri and the path to the payload is * stored in external storage */ ExternalStorageLocation getExternalStorageLocation( String path, String operation, String payloadType); } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/WorkflowServiceImpl.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.springframework.stereotype.Service; import com.netflix.conductor.annotations.Audit; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.Utils; @Audit @Trace @Service public class WorkflowServiceImpl implements WorkflowService { private final WorkflowExecutor workflowExecutor; private final ExecutionService executionService; private final MetadataService metadataService; private final StartWorkflowOperation startWorkflowOperation; public WorkflowServiceImpl( WorkflowExecutor workflowExecutor, ExecutionService executionService, MetadataService metadataService, StartWorkflowOperation startWorkflowOperation) { this.workflowExecutor = workflowExecutor; this.executionService = executionService; this.metadataService = metadataService; this.startWorkflowOperation = startWorkflowOperation; } /** * Start a new workflow with StartWorkflowRequest, which allows task to be executed in a domain. * * @param startWorkflowRequest StartWorkflow request for the workflow you want to start. * @return the id of the workflow instance that can be use for tracking. */ public String startWorkflow(StartWorkflowRequest startWorkflowRequest) { return startWorkflowOperation.execute(new StartWorkflowInput(startWorkflowRequest)); } /** * Start a new workflow with StartWorkflowRequest, which allows task to be executed in a domain. * * @param name Name of the workflow you want to start. * @param version Version of the workflow you want to start. * @param correlationId CorrelationID of the workflow you want to start. * @param priority Priority of the workflow you want to start. * @param input Input to the workflow you want to start. * @param externalInputPayloadStoragePath the relative path in external storage where input * * payload is located * @param taskToDomain the task to domain mapping * @param workflowDef - workflow definition * @return the id of the workflow instance that can be use for tracking. */ public String startWorkflow( String name, Integer version, String correlationId, Integer priority, Map input, String externalInputPayloadStoragePath, Map taskToDomain, WorkflowDef workflowDef) { StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName(name); startWorkflowInput.setVersion(version); startWorkflowInput.setCorrelationId(correlationId); startWorkflowInput.setPriority(priority); startWorkflowInput.setWorkflowInput(input); startWorkflowInput.setExternalInputPayloadStoragePath(externalInputPayloadStoragePath); startWorkflowInput.setTaskToDomain(taskToDomain); startWorkflowInput.setWorkflowDefinition(workflowDef); return startWorkflowOperation.execute(startWorkflowInput); } /** * Start a new workflow. Returns the ID of the workflow instance that can be later used for * tracking. * * @param name Name of the workflow you want to start. * @param version Version of the workflow you want to start. * @param correlationId CorrelationID of the workflow you want to start. * @param priority Priority of the workflow you want to start. * @param input Input to the workflow you want to start. * @return the id of the workflow instance that can be use for tracking. */ public String startWorkflow( String name, Integer version, String correlationId, Integer priority, Map input) { WorkflowDef workflowDef = metadataService.getWorkflowDef(name, version); if (workflowDef == null) { throw new NotFoundException( "No such workflow found by name: %s, version: %d", name, version); } StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName(workflowDef.getName()); startWorkflowInput.setVersion(workflowDef.getVersion()); startWorkflowInput.setCorrelationId(correlationId); startWorkflowInput.setPriority(priority); startWorkflowInput.setWorkflowInput(input); return startWorkflowOperation.execute(startWorkflowInput); } /** * Lists workflows for the given correlation id. * * @param name Name of the workflow. * @param correlationId CorrelationID of the workflow you want to start. * @param includeClosed IncludeClosed workflow which are not running. * @param includeTasks Includes tasks associated with workflows. * @return a list of {@link Workflow} */ public List getWorkflows( String name, String correlationId, boolean includeClosed, boolean includeTasks) { return executionService.getWorkflowInstances( name, correlationId, includeClosed, includeTasks); } /** * Lists workflows for the given correlation id. * * @param name Name of the workflow. * @param includeClosed CorrelationID of the workflow you want to start. * @param includeTasks IncludeClosed workflow which are not running. * @param correlationIds Includes tasks associated with workflows. * @return a {@link Map} of {@link String} as key and a list of {@link Workflow} as value */ public Map> getWorkflows( String name, boolean includeClosed, boolean includeTasks, List correlationIds) { Map> workflowMap = new HashMap<>(); for (String correlationId : correlationIds) { List workflows = executionService.getWorkflowInstances( name, correlationId, includeClosed, includeTasks); workflowMap.put(correlationId, workflows); } return workflowMap; } /** * Gets the workflow by workflow id. * * @param workflowId id of the workflow. * @param includeTasks Includes tasks associated with workflow. * @return an instance of {@link Workflow} */ public Workflow getExecutionStatus(String workflowId, boolean includeTasks) { Workflow workflow = executionService.getExecutionStatus(workflowId, includeTasks); if (workflow == null) { throw new NotFoundException("Workflow with id: %s not found.", workflowId); } return workflow; } /** * Removes the workflow from the system. * * @param workflowId WorkflowID of the workflow you want to remove from system. * @param archiveWorkflow Archives the workflow and associated tasks instead of removing them. */ public void deleteWorkflow(String workflowId, boolean archiveWorkflow) { executionService.removeWorkflow(workflowId, archiveWorkflow); } /** * Retrieves all the running workflows. * * @param workflowName Name of the workflow. * @param version Version of the workflow. * @param startTime start time of the workflow. * @param endTime EndTime of the workflow * @return a list of workflow Ids. */ public List getRunningWorkflows( String workflowName, Integer version, Long startTime, Long endTime) { if (Optional.ofNullable(startTime).orElse(0L) != 0 && Optional.ofNullable(endTime).orElse(0L) != 0) { return workflowExecutor.getWorkflows(workflowName, version, startTime, endTime); } else { version = Optional.ofNullable(version) .orElseGet( () -> { WorkflowDef workflowDef = metadataService.getWorkflowDef(workflowName, null); return workflowDef.getVersion(); }); return workflowExecutor.getRunningWorkflowIds(workflowName, version); } } /** * Starts the decision task for a workflow. * * @param workflowId WorkflowId of the workflow. */ public void decideWorkflow(String workflowId) { workflowExecutor.decide(workflowId); } /** * Pauses the workflow given a workflowId. * * @param workflowId WorkflowId of the workflow. */ public void pauseWorkflow(String workflowId) { workflowExecutor.pauseWorkflow(workflowId); } /** * Resumes the workflow. * * @param workflowId WorkflowId of the workflow. */ public void resumeWorkflow(String workflowId) { workflowExecutor.resumeWorkflow(workflowId); } /** * Skips a given task from a current running workflow. * * @param workflowId WorkflowId of the workflow. * @param taskReferenceName The task reference name. * @param skipTaskRequest {@link SkipTaskRequest} for task you want to skip. */ public void skipTaskFromWorkflow( String workflowId, String taskReferenceName, SkipTaskRequest skipTaskRequest) { workflowExecutor.skipTaskFromWorkflow(workflowId, taskReferenceName, skipTaskRequest); } /** * Reruns the workflow from a specific task. * * @param workflowId WorkflowId of the workflow you want to rerun. * @param request (@link RerunWorkflowRequest) for the workflow. * @return WorkflowId of the rerun workflow. */ public String rerunWorkflow(String workflowId, RerunWorkflowRequest request) { request.setReRunFromWorkflowId(workflowId); return workflowExecutor.rerun(request); } /** * Restarts a completed workflow. * * @param workflowId WorkflowId of the workflow. * @param useLatestDefinitions if true, use the latest workflow and task definitions upon * restart */ public void restartWorkflow(String workflowId, boolean useLatestDefinitions) { workflowExecutor.restart(workflowId, useLatestDefinitions); } /** * Retries the last failed task. * * @param workflowId WorkflowId of the workflow. */ public void retryWorkflow(String workflowId, boolean resumeSubworkflowTasks) { workflowExecutor.retry(workflowId, resumeSubworkflowTasks); } /** * Resets callback times of all non-terminal SIMPLE tasks to 0. * * @param workflowId WorkflowId of the workflow. */ public void resetWorkflow(String workflowId) { workflowExecutor.resetCallbacksForWorkflow(workflowId); } /** * Terminate workflow execution. * * @param workflowId WorkflowId of the workflow. * @param reason Reason for terminating the workflow. */ public void terminateWorkflow(String workflowId, String reason) { workflowExecutor.terminateWorkflow(workflowId, reason); } /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflows( int start, int size, String sort, String freeText, String query) { return executionService.search( query, freeText, start, size, Utils.convertStringToList(sort)); } /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsV2( int start, int size, String sort, String freeText, String query) { return executionService.searchV2( query, freeText, start, size, Utils.convertStringToList(sort)); } /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflows( int start, int size, List sort, String freeText, String query) { return executionService.search(query, freeText, start, size, sort); } /** * Search for workflows based on payload and given parameters. Use sort options as sort ASCor * DESC e.g. sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsV2( int start, int size, List sort, String freeText, String query) { return executionService.searchV2(query, freeText, start, size, sort); } /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsByTasks( int start, int size, String sort, String freeText, String query) { return executionService.searchWorkflowByTasks( query, freeText, start, size, Utils.convertStringToList(sort)); } /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort Sorting type ASC|DESC * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsByTasksV2( int start, int size, String sort, String freeText, String query) { return executionService.searchWorkflowByTasksV2( query, freeText, start, size, Utils.convertStringToList(sort)); } /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsByTasks( int start, int size, List sort, String freeText, String query) { return executionService.searchWorkflowByTasks(query, freeText, start, size, sort); } /** * Search for workflows based on task parameters. Use sort options as sort ASC or DESC e.g. * sort=name or sort=workflowId:DESC. If order is not specified, defaults to ASC. * * @param start Start index of pagination * @param size Number of entries * @param sort list of sorting options, separated by "|" delimiter * @param freeText Text you want to search * @param query Query you want to search * @return instance of {@link SearchResult} */ public SearchResult searchWorkflowsByTasksV2( int start, int size, List sort, String freeText, String query) { return executionService.searchWorkflowByTasksV2(query, freeText, start, size, sort); } /** * Get the external storage location where the workflow input payload is stored/to be stored * * @param path the path for which the external storage location is to be populated * @param operation the operation to be performed (read or write) * @param type the type of payload (input or output) * @return {@link ExternalStorageLocation} containing the uri and the path to the payload is * stored in external storage */ public ExternalStorageLocation getExternalStorageLocation( String path, String operation, String type) { return executionService.getExternalStorageLocation(path, operation, type); } } ================================================ FILE: core/src/main/java/com/netflix/conductor/service/WorkflowTestService.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowTestRequest; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.model.TaskModel; @Component public class WorkflowTestService { private static final int MAX_LOOPS = 20_000; private static final Set operators = new HashSet<>(); static { operators.add(TaskType.TASK_TYPE_JOIN); operators.add(TaskType.TASK_TYPE_DO_WHILE); operators.add(TaskType.TASK_TYPE_SET_VARIABLE); operators.add(TaskType.TASK_TYPE_FORK); operators.add(TaskType.TASK_TYPE_INLINE); operators.add(TaskType.TASK_TYPE_TERMINATE); operators.add(TaskType.TASK_TYPE_DECISION); operators.add(TaskType.TASK_TYPE_DYNAMIC); operators.add(TaskType.TASK_TYPE_FORK_JOIN); operators.add(TaskType.TASK_TYPE_FORK_JOIN_DYNAMIC); operators.add(TaskType.TASK_TYPE_SWITCH); operators.add(TaskType.TASK_TYPE_SUB_WORKFLOW); } private final WorkflowService workflowService; private final ExecutionDAO executionDAO; private final ExecutionService workflowExecutionService; public WorkflowTestService( WorkflowService workflowService, ExecutionDAO executionDAO, ExecutionService workflowExecutionService) { this.workflowService = workflowService; this.executionDAO = executionDAO; this.workflowExecutionService = workflowExecutionService; } public Workflow testWorkflow(WorkflowTestRequest request) { request.setName(request.getName()); request.setVersion(request.getVersion()); String domain = UUID.randomUUID().toString(); // Ensure the workflows started for the testing are not picked by any workers request.getTaskToDomain().put("*", domain); String workflowId = workflowService.startWorkflow(request); return testWorkflow(request, workflowId); } private Workflow testWorkflow(WorkflowTestRequest request, String workflowId) { Map> mockData = request.getTaskRefToMockOutput(); Workflow workflow; int loopCount = 0; do { loopCount++; workflow = workflowService.getExecutionStatus(workflowId, true); if (loopCount > MAX_LOOPS) { // Short circuit to avoid large loops return workflow; } List runningTasksMissingInput = workflow.getTasks().stream() .filter(task -> !operators.contains(task.getTaskType())) .filter(t -> !t.getStatus().isTerminal()) .filter(t2 -> !mockData.containsKey(t2.getReferenceTaskName())) .map(task -> task.getReferenceTaskName()) .collect(Collectors.toList()); if (!runningTasksMissingInput.isEmpty()) { break; } Stream runningTasks = workflow.getTasks().stream().filter(t -> !t.getStatus().isTerminal()); runningTasks.forEach( running -> { if (running.getTaskType().equals(TaskType.SUB_WORKFLOW.name())) { String subWorkflowId = running.getSubWorkflowId(); WorkflowTestRequest subWorkflowTestRequest = request.getSubWorkflowTestRequest() .get(running.getReferenceTaskName()); if (subWorkflowId != null && subWorkflowTestRequest != null) { testWorkflow(subWorkflowTestRequest, subWorkflowId); } } String refName = running.getReferenceTaskName(); List taskMock = mockData.get(refName); if (taskMock == null || taskMock.isEmpty() || operators.contains(running.getTaskType())) { mockData.remove(refName); workflowService.decideWorkflow(workflowId); } else { WorkflowTestRequest.TaskMock task = taskMock.remove(0); if (task.getExecutionTime() > 0 || task.getQueueWaitTime() > 0) { TaskModel existing = executionDAO.getTask(running.getTaskId()); existing.setScheduledTime( System.currentTimeMillis() - (task.getExecutionTime() + task.getQueueWaitTime())); existing.setStartTime( System.currentTimeMillis() - task.getExecutionTime()); existing.setStatus( TaskModel.Status.valueOf(task.getStatus().name())); existing.getOutputData().putAll(task.getOutput()); executionDAO.updateTask(existing); workflowService.decideWorkflow(workflowId); } else { TaskResult taskResult = new TaskResult(running); taskResult.setStatus(task.getStatus()); taskResult.getOutputData().putAll(task.getOutput()); workflowExecutionService.updateTask(taskResult); } } }); } while (!workflow.getStatus().isTerminal() && !mockData.isEmpty()); return workflow; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/validations/ValidationContext.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.validations; import com.netflix.conductor.dao.MetadataDAO; /** * This context is defined to get access to {@link MetadataDAO} inside {@link * WorkflowTaskTypeConstraint} constraint validator to validate {@link * com.netflix.conductor.common.metadata.workflow.WorkflowTask}. */ public class ValidationContext { private static MetadataDAO metadataDAO; public static void initialize(MetadataDAO metadataDAO) { ValidationContext.metadataDAO = metadataDAO; } public static MetadataDAO getMetadataDAO() { return metadataDAO; } } ================================================ FILE: core/src/main/java/com/netflix/conductor/validations/WorkflowTaskTypeConstraint.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.validations; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.text.ParseException; import java.time.format.DateTimeParseException; import java.util.Map; import java.util.Optional; import javax.script.ScriptException; import javax.validation.Constraint; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.events.ScriptEvaluator; import com.netflix.conductor.core.utils.DateTimeUtils; import static com.netflix.conductor.core.execution.tasks.Terminate.getTerminationStatusParameter; import static com.netflix.conductor.core.execution.tasks.Terminate.validateInputStatus; import static com.netflix.conductor.core.execution.tasks.Wait.DURATION_INPUT; import static com.netflix.conductor.core.execution.tasks.Wait.UNTIL_INPUT; import static java.lang.annotation.ElementType.ANNOTATION_TYPE; import static java.lang.annotation.ElementType.TYPE; /** * This constraint class validates following things. 1. Correct parameters are set depending on task * type. */ @Documented @Constraint(validatedBy = WorkflowTaskTypeConstraint.WorkflowTaskValidator.class) @Target({TYPE, ANNOTATION_TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface WorkflowTaskTypeConstraint { String message() default ""; Class[] groups() default {}; Class[] payload() default {}; class WorkflowTaskValidator implements ConstraintValidator { final String PARAM_REQUIRED_STRING_FORMAT = "%s field is required for taskType: %s taskName: %s"; @Override public void initialize(WorkflowTaskTypeConstraint constraintAnnotation) {} @Override public boolean isValid(WorkflowTask workflowTask, ConstraintValidatorContext context) { context.disableDefaultConstraintViolation(); boolean valid = true; // depending on task type check if required parameters are set or not switch (workflowTask.getType()) { case TaskType.TASK_TYPE_EVENT: valid = isEventTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_DECISION: valid = isDecisionTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_SWITCH: valid = isSwitchTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_DYNAMIC: valid = isDynamicTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_FORK_JOIN_DYNAMIC: valid = isDynamicForkJoinValid(workflowTask, context); break; case TaskType.TASK_TYPE_HTTP: valid = isHttpTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_FORK_JOIN: valid = isForkJoinTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_TERMINATE: valid = isTerminateTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_KAFKA_PUBLISH: valid = isKafkaPublishTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_DO_WHILE: valid = isDoWhileTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_SUB_WORKFLOW: valid = isSubWorkflowTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_JSON_JQ_TRANSFORM: valid = isJSONJQTransformTaskValid(workflowTask, context); break; case TaskType.TASK_TYPE_WAIT: valid = isWaitTaskValid(workflowTask, context); break; } return valid; } private boolean isEventTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getSink() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "sink", TaskType.TASK_TYPE_EVENT, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isDecisionTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getCaseValueParam() == null && workflowTask.getCaseExpression() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "caseValueParam or caseExpression", TaskType.DECISION, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (workflowTask.getDecisionCases() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "decisionCases", TaskType.DECISION, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } else if ((workflowTask.getDecisionCases() != null || workflowTask.getCaseExpression() != null) && (workflowTask.getDecisionCases().size() == 0)) { String message = String.format( "decisionCases should have atleast one task for taskType: %s taskName: %s", TaskType.DECISION, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (workflowTask.getCaseExpression() != null) { try { validateScriptExpression( workflowTask.getCaseExpression(), workflowTask.getInputParameters()); } catch (Exception ee) { String message = String.format( ee.getMessage() + ", taskType: DECISION taskName %s", workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } } return valid; } private void validateScriptExpression( String expression, Map inputParameters) { try { Object returnValue = ScriptEvaluator.eval(expression, inputParameters); } catch (ScriptException e) { throw new IllegalArgumentException( String.format("Expression is not well formatted: %s", e.getMessage())); } } private boolean isSwitchTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getEvaluatorType() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "evaluatorType", TaskType.SWITCH, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } else if (workflowTask.getExpression() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "expression", TaskType.SWITCH, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (workflowTask.getDecisionCases() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "decisionCases", TaskType.SWITCH, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } else if (workflowTask.getDecisionCases() != null && workflowTask.getDecisionCases().size() == 0) { String message = String.format( "decisionCases should have atleast one task for taskType: %s taskName: %s", TaskType.SWITCH, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if ("javascript".equals(workflowTask.getEvaluatorType()) && workflowTask.getExpression() != null) { try { validateScriptExpression( workflowTask.getExpression(), workflowTask.getInputParameters()); } catch (Exception ee) { String message = String.format( ee.getMessage() + ", taskType: SWITCH taskName %s", workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } } return valid; } private boolean isDoWhileTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getLoopCondition() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "loopExpression", TaskType.DO_WHILE, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (workflowTask.getLoopOver() == null || workflowTask.getLoopOver().size() == 0) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "loopover", TaskType.DO_WHILE, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isDynamicTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getDynamicTaskNameParam() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "dynamicTaskNameParam", TaskType.DYNAMIC, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isWaitTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; String duration = Optional.ofNullable(workflowTask.getInputParameters().get(DURATION_INPUT)) .orElse("") .toString(); String until = Optional.ofNullable(workflowTask.getInputParameters().get(UNTIL_INPUT)) .orElse("") .toString(); if (StringUtils.isNotBlank(duration) && StringUtils.isNotBlank(until)) { String message = "Both 'duration' and 'until' specified. Please provide only one input"; context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } try { if (StringUtils.isNotBlank(duration) && !(duration.startsWith("${"))) { DateTimeUtils.parseDuration(duration); } else if (StringUtils.isNotBlank(until) && !(until.startsWith("${"))) { DateTimeUtils.parseDate(until); } } catch (DateTimeParseException e) { String message = "Unable to parse date "; context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } catch (IllegalArgumentException e) { String message = "Either date or duration is passed as null "; context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } catch (ParseException e) { String message = "Unable to parse date "; context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } catch (Exception e) { String message = "Wait time specified is invalid. The duration must be in "; context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isDynamicForkJoinValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; // For DYNAMIC_FORK_JOIN_TASK support dynamicForkJoinTasksParam or combination of // dynamicForkTasksParam and dynamicForkTasksInputParamName. // Both are not allowed. if (workflowTask.getDynamicForkJoinTasksParam() != null && (workflowTask.getDynamicForkTasksParam() != null || workflowTask.getDynamicForkTasksInputParamName() != null)) { String message = String.format( "dynamicForkJoinTasksParam or combination of dynamicForkTasksInputParamName and dynamicForkTasksParam cam be used for taskType: %s taskName: %s", TaskType.FORK_JOIN_DYNAMIC, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); return false; } if (workflowTask.getDynamicForkJoinTasksParam() != null) { return valid; } else { if (workflowTask.getDynamicForkTasksParam() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "dynamicForkTasksParam", TaskType.FORK_JOIN_DYNAMIC, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (workflowTask.getDynamicForkTasksInputParamName() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "dynamicForkTasksInputParamName", TaskType.FORK_JOIN_DYNAMIC, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } } return valid; } private boolean isHttpTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; boolean isInputParameterSet = false; boolean isInputTemplateSet = false; // Either http_request in WorkflowTask inputParam should be set or in inputTemplate // Taskdef should be set if (workflowTask.getInputParameters() != null && workflowTask.getInputParameters().containsKey("http_request")) { isInputParameterSet = true; } TaskDef taskDef = Optional.ofNullable(workflowTask.getTaskDefinition()) .orElse( ValidationContext.getMetadataDAO() .getTaskDef(workflowTask.getName())); if (taskDef != null && taskDef.getInputTemplate() != null && taskDef.getInputTemplate().containsKey("http_request")) { isInputTemplateSet = true; } if (!(isInputParameterSet || isInputTemplateSet)) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "inputParameters.http_request", TaskType.HTTP, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isForkJoinTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getForkTasks() != null && (workflowTask.getForkTasks().size() == 0)) { String message = String.format( "forkTasks should have atleast one task for taskType: %s taskName: %s", TaskType.FORK_JOIN, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isTerminateTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; Object inputStatusParam = workflowTask.getInputParameters().get(getTerminationStatusParameter()); if (workflowTask.isOptional()) { String message = String.format( "terminate task cannot be optional, taskName: %s", workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } if (inputStatusParam == null || !validateInputStatus(inputStatusParam.toString())) { String message = String.format( "terminate task must have an %s parameter and must be set to COMPLETED or FAILED, taskName: %s", getTerminationStatusParameter(), workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isKafkaPublishTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; boolean isInputParameterSet = false; boolean isInputTemplateSet = false; // Either kafka_request in WorkflowTask inputParam should be set or in inputTemplate // Taskdef should be set if (workflowTask.getInputParameters() != null && workflowTask.getInputParameters().containsKey("kafka_request")) { isInputParameterSet = true; } TaskDef taskDef = Optional.ofNullable(workflowTask.getTaskDefinition()) .orElse( ValidationContext.getMetadataDAO() .getTaskDef(workflowTask.getName())); if (taskDef != null && taskDef.getInputTemplate() != null && taskDef.getInputTemplate().containsKey("kafka_request")) { isInputTemplateSet = true; } if (!(isInputParameterSet || isInputTemplateSet)) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "inputParameters.kafka_request", TaskType.KAFKA_PUBLISH, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isSubWorkflowTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; if (workflowTask.getSubWorkflowParam() == null) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "subWorkflowParam", TaskType.SUB_WORKFLOW, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } private boolean isJSONJQTransformTaskValid( WorkflowTask workflowTask, ConstraintValidatorContext context) { boolean valid = true; boolean isInputParameterSet = false; boolean isInputTemplateSet = false; // Either queryExpression in WorkflowTask inputParam should be set or in inputTemplate // Taskdef should be set if (workflowTask.getInputParameters() != null && workflowTask.getInputParameters().containsKey("queryExpression")) { isInputParameterSet = true; } TaskDef taskDef = Optional.ofNullable(workflowTask.getTaskDefinition()) .orElse( ValidationContext.getMetadataDAO() .getTaskDef(workflowTask.getName())); if (taskDef != null && taskDef.getInputTemplate() != null && taskDef.getInputTemplate().containsKey("queryExpression")) { isInputTemplateSet = true; } if (!(isInputParameterSet || isInputTemplateSet)) { String message = String.format( PARAM_REQUIRED_STRING_FORMAT, "inputParameters.queryExpression", TaskType.JSON_JQ_TRANSFORM, workflowTask.getName()); context.buildConstraintViolationWithTemplate(message).addConstraintViolation(); valid = false; } return valid; } } } ================================================ FILE: core/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.workflow-reconciler.enabled", "type": "java.lang.Boolean", "description": "Enables the workflow reconciliation mechanism.", "sourceType": "com.netflix.conductor.core.reconciliation.WorkflowReconciler", "defaultValue": true }, { "name": "conductor.sweep-frequency.millis", "type": "java.lang.Integer", "description": "The frequency in milliseconds, at which the workflow sweeper should evaluate active workflows.", "sourceType": "com.netflix.conductor.core.reconciliation.WorkflowReconciler", "defaultValue": 500 }, { "name": "conductor.workflow-repair-service.enabled", "type": "java.lang.Boolean", "description": "Configuration to enable WorkflowRepairService, that tries to keep ExecutionDAO and QueueDAO in sync, based on the task or workflow state. This is disabled by default; To enable, the Queueing layer must implement QueueDAO.containsMessage method.", "sourceType": "com.netflix.conductor.core.reconciliation.WorkflowRepairService" }, { "name": "conductor.system-task-workers.enabled", "type": "java.lang.Boolean", "description": "Configuration to enable SystemTaskWorkerCoordinator, that polls and executes the asynchronous system tasks.", "sourceType": "com.netflix.conductor.core.execution.tasks.SystemTaskWorkerCoordinator", "defaultValue": true }, { "name": "conductor.app.isolated-system-task-enabled", "type": "java.lang.Boolean", "description": "Used to enable/disable use of isolation groups for system task workers." }, { "name": "conductor.app.isolatedSystemTaskPollIntervalSecs", "type": "java.lang.Integer", "description": "The time interval (in seconds) at which new isolated task queues will be polled and added to the system task queue repository." }, { "name": "conductor.app.taskPendingTimeThresholdMins", "type": "java.lang.Long", "description": "The time threshold (in minutes) beyond which a warning log will be emitted for a task if it stays in the same state for this duration." }, { "name": "conductor.workflow-monitor.enabled", "type": "java.lang.Boolean", "description": "Enables the workflow monitor that publishes workflow and task metrics.", "defaultValue": "true", "sourceType": "com.netflix.conductor.metrics.WorkflowMonitor" }, { "name": "conductor.workflow-monitor.stats.initial-delay", "type": "java.lang.Integer", "description": "The initial delay (in milliseconds) at which the workflow monitor publishes workflow and task metrics." }, { "name": "conductor.workflow-monitor.metadata-refresh-interval", "type": "java.lang.Integer", "description": "The interval (counter) after which the workflow monitor refreshes the metadata definitions from the datastore.", "defaultValue": "10" }, { "name": "conductor.workflow-monitor.stats.delay", "type": "java.lang.Integer", "description": "The delay (in milliseconds) at which the workflow monitor publishes workflow and task metrics." }, { "name": "conductor.external-payload-storage.type", "type": "java.lang.String", "description": "The type of payload storage to be used for externalizing large payloads." }, { "name": "conductor.default-event-processor.enabled", "type": "java.lang.Boolean", "description": "Enables the default event processor for handling events.", "sourceType": "com.netflix.conductor.core.events.DefaultEventProcessor", "defaultValue": "true" }, { "name": "conductor.event-queues.default.enabled", "type": "java.lang.Boolean", "description": "Enables the use of the underlying queue implementation to provide queues for consuming events.", "sourceType": "com.netflix.conductor.core.events.queue.ConductorEventQueueProvider", "defaultValue": "true" }, { "name": "conductor.default-event-queue-processor.enabled", "type": "java.lang.Boolean", "description": "Enables the processor for the default event queues that conductor is configured to listen on.", "sourceType": "com.netflix.conductor.core.events.queue.DefaultEventQueueProcessor", "defaultValue": "true" }, { "name": "conductor.workflow-status-listener.type", "type": "java.lang.String", "description": "The implementation of the workflow status listener to be used." }, { "name": "conductor.task-status-listener.type", "type": "java.lang.String", "description": "The implementation of the task status listener to be used." }, { "name": "conductor.workflow-execution-lock.type", "type": "java.lang.String", "description": "The implementation of the workflow execution lock to be used.", "defaultValue": "noop_lock" } ], "hints": [ { "name": "conductor.external-payload-storage.type", "values": [ { "value": "dummy", "description": "Use the dummy no-op implementation as the external payload storage." } ] }, { "name": "conductor.workflow-status-listener.type", "values": [ { "value": "stub", "description": "Use the no-op implementation of the workflow status listener." } ] }, { "name": "conductor.workflow-execution-lock.type", "values": [ { "value": "noop_lock", "description": "Use the no-op implementation as the lock provider." }, { "value": "local_only", "description": "Use the local in-memory cache based implementation as the lock provider." } ] } ] } ================================================ FILE: core/src/main/resources/META-INF/validation/constraints.xml ================================================ com.netflix.conductor.common.metadata.workflow ================================================ FILE: core/src/main/resources/META-INF/validation.xml ================================================ META-INF/validation/constraints.xml ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/execution/AsyncSystemTaskExecutorTest.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution import java.time.Duration import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.core.config.ConductorProperties import com.netflix.conductor.core.dal.ExecutionDAOFacade import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask import com.netflix.conductor.core.operation.StartWorkflowOperation import com.netflix.conductor.core.utils.IDGenerator import com.netflix.conductor.core.utils.QueueUtils import com.netflix.conductor.dao.MetadataDAO import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.model.TaskModel import com.netflix.conductor.model.WorkflowModel import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Subject import static com.netflix.conductor.common.metadata.tasks.TaskType.SUB_WORKFLOW class AsyncSystemTaskExecutorTest extends Specification { ExecutionDAOFacade executionDAOFacade QueueDAO queueDAO MetadataDAO metadataDAO WorkflowExecutor workflowExecutor StartWorkflowOperation startWorkflowOperation @Subject AsyncSystemTaskExecutor executor WorkflowSystemTask workflowSystemTask ConductorProperties properties = new ConductorProperties() def setup() { executionDAOFacade = Mock(ExecutionDAOFacade.class) queueDAO = Mock(QueueDAO.class) metadataDAO = Mock(MetadataDAO.class) workflowExecutor = Mock(WorkflowExecutor.class) startWorkflowOperation = Mock(StartWorkflowOperation.class) workflowSystemTask = Mock(WorkflowSystemTask.class) { isTaskRetrievalRequired() >> true } properties.taskExecutionPostponeDuration = Duration.ofSeconds(1) properties.systemTaskWorkerCallbackDuration = Duration.ofSeconds(1) executor = new AsyncSystemTaskExecutor(executionDAOFacade, queueDAO, metadataDAO, properties, workflowExecutor) } // this is not strictly a unit test, but its essential to test AsyncSystemTaskExecutor with SubWorkflow def "Execute SubWorkflow task"() { given: String workflowId = "workflowId" String subWorkflowId = "subWorkflowId" SubWorkflow subWorkflowTask = new SubWorkflow(new ObjectMapper(), startWorkflowOperation) String task1Id = new IDGenerator().generate() TaskModel task1 = new TaskModel() task1.setTaskType(SUB_WORKFLOW.name()) task1.setReferenceTaskName("waitTask") task1.setWorkflowInstanceId(workflowId) task1.setScheduledTime(System.currentTimeMillis()) task1.setTaskId(task1Id) task1.getInputData().put("asyncComplete", true) task1.getInputData().put("subWorkflowName", "junit1") task1.getInputData().put("subWorkflowVersion", 1) task1.setStatus(TaskModel.Status.SCHEDULED) String queueName = QueueUtils.getQueueName(task1) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) WorkflowModel subWorkflow = new WorkflowModel(workflowId: subWorkflowId, status: WorkflowModel.Status.RUNNING) when: executor.execute(subWorkflowTask, task1Id) then: 1 * executionDAOFacade.getTaskModel(task1Id) >> task1 1 * executionDAOFacade.getWorkflowModel(workflowId, subWorkflowTask.isTaskRetrievalRequired()) >> workflow 1 * startWorkflowOperation.execute(*_) >> subWorkflowId 1 * workflowExecutor.getWorkflow(subWorkflowId, false) >> subWorkflow // SUB_WORKFLOW is asyncComplete so its removed from the queue 1 * queueDAO.remove(queueName, task1Id) task1.status == TaskModel.Status.IN_PROGRESS task1.subWorkflowId == subWorkflowId task1.startTime != 0 } def "Execute with a non-existing task id"() { given: String taskId = "taskId" when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> null 0 * workflowSystemTask.start(*_) 0 * executionDAOFacade.updateTask(_) } def "Execute with a task id that fails to load"() { given: String taskId = "taskId" when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> { throw new RuntimeException("datastore unavailable") } 0 * workflowSystemTask.start(*_) 0 * executionDAOFacade.updateTask(_) } def "Execute with a task id that is in terminal state"() { given: String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.COMPLETED, taskId: taskId) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * queueDAO.remove(task.taskType, taskId) 0 * workflowSystemTask.start(*_) 0 * executionDAOFacade.updateTask(_) } def "Execute with a task id that is part of a workflow in terminal state"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.COMPLETED) String queueName = QueueUtils.getQueueName(task) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * queueDAO.remove(queueName, taskId) task.status == TaskModel.Status.CANCELED task.startTime == 0 } def "Execute with a task id that exceeds in-progress limit"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, workflowPriority: 10) String queueName = QueueUtils.getQueueName(task) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.exceedsInProgressLimit(task) >> true 1 * queueDAO.postpone(queueName, taskId, task.workflowPriority, properties.taskExecutionPostponeDuration.seconds) task.status == TaskModel.Status.SCHEDULED task.startTime == 0 } def "Execute with a task id that is rate limited"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, rateLimitPerFrequency: 1, taskDefName: "taskDefName", workflowPriority: 10) String queueName = QueueUtils.getQueueName(task) TaskDef taskDef = new TaskDef() when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * metadataDAO.getTaskDef(task.taskDefName) >> taskDef 1 * executionDAOFacade.exceedsRateLimitPerFrequency(task, taskDef) >> taskDef 1 * queueDAO.postpone(queueName, taskId, task.workflowPriority, properties.taskExecutionPostponeDuration.seconds) task.status == TaskModel.Status.SCHEDULED task.startTime == 0 } def "Execute with a task id that is rate limited but postpone fails"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, rateLimitPerFrequency: 1, taskDefName: "taskDefName", workflowPriority: 10) String queueName = QueueUtils.getQueueName(task) TaskDef taskDef = new TaskDef() when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * metadataDAO.getTaskDef(task.taskDefName) >> taskDef 1 * executionDAOFacade.exceedsRateLimitPerFrequency(task, taskDef) >> taskDef 1 * queueDAO.postpone(queueName, taskId, task.workflowPriority, properties.taskExecutionPostponeDuration.seconds) >> { throw new RuntimeException("queue unavailable") } task.status == TaskModel.Status.SCHEDULED task.startTime == 0 } def "Execute with a task id that is in SCHEDULED state"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, taskDefName: "taskDefName", workflowPriority: 10) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) String queueName = QueueUtils.getQueueName(task) workflowSystemTask.getEvaluationOffset(task, 1) >> Optional.empty(); when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) 1 * queueDAO.postpone(queueName, taskId, task.workflowPriority, properties.systemTaskWorkerCallbackDuration.seconds) 1 * workflowSystemTask.start(workflow, task, workflowExecutor) >> { task.status = TaskModel.Status.IN_PROGRESS } 0 * workflowExecutor.decide(workflowId) // verify that workflow is NOT decided task.status == TaskModel.Status.IN_PROGRESS task.startTime != 0 // verify that startTime is set task.endTime == 0 // verify that endTime is not set task.pollCount == 1 // verify that poll count is incremented task.callbackAfterSeconds == properties.systemTaskWorkerCallbackDuration.seconds } def "Execute with a task id that is in SCHEDULED state and WorkflowSystemTask.start sets the task in a terminal state"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, taskDefName: "taskDefName", workflowPriority: 10) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) String queueName = QueueUtils.getQueueName(task) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) 1 * workflowSystemTask.start(workflow, task, workflowExecutor) >> { task.status = TaskModel.Status.COMPLETED } 1 * queueDAO.remove(queueName, taskId) 1 * workflowExecutor.decide(workflowId) // verify that workflow is decided task.status == TaskModel.Status.COMPLETED task.startTime != 0 // verify that startTime is set task.endTime != 0 // verify that endTime is set task.pollCount == 1 // verify that poll count is incremented } def "Execute with a task id that is in SCHEDULED state but WorkflowSystemTask.start fails"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, taskDefName: "taskDefName", workflowPriority: 10) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) // simulating a "start" failure that happens after the Task object is modified // the modification will be persisted 1 * workflowSystemTask.start(workflow, task, workflowExecutor) >> { task.status = TaskModel.Status.IN_PROGRESS throw new RuntimeException("unknown system task failure") } 0 * workflowExecutor.decide(workflowId) // verify that workflow is NOT decided task.status == TaskModel.Status.IN_PROGRESS task.startTime != 0 // verify that startTime is set task.endTime == 0 // verify that endTime is not set task.pollCount == 1 // verify that poll count is incremented } def "Execute with a task id that is in SCHEDULED state and is set to asyncComplete"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.SCHEDULED, taskId: taskId, workflowInstanceId: workflowId, taskDefName: "taskDefName", workflowPriority: 10) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) String queueName = QueueUtils.getQueueName(task) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) // 1st call for pollCount, 2nd call for status update 1 * workflowSystemTask.isAsyncComplete(task) >> true 1 * workflowSystemTask.start(workflow, task, workflowExecutor) >> { task.status = TaskModel.Status.IN_PROGRESS } 1 * queueDAO.remove(queueName, taskId) 1 * workflowExecutor.decide(workflowId) // verify that workflow is decided task.status == TaskModel.Status.IN_PROGRESS task.startTime != 0 // verify that startTime is set task.endTime == 0 // verify that endTime is not set task.pollCount == 1 // verify that poll count is incremented } def "Execute with a task id that is in IN_PROGRESS state"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.IN_PROGRESS, taskId: taskId, workflowInstanceId: workflowId, rateLimitPerFrequency: 1, taskDefName: "taskDefName", workflowPriority: 10, pollCount: 1) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) // 1st call for pollCount, 2nd call for status update 0 * workflowSystemTask.start(workflow, task, workflowExecutor) 1 * workflowSystemTask.execute(workflow, task, workflowExecutor) task.status == TaskModel.Status.IN_PROGRESS task.endTime == 0 // verify that endTime is not set task.pollCount == 2 // verify that poll count is incremented } def "Execute with a task id that is in IN_PROGRESS state and is set to asyncComplete"() { given: String workflowId = "workflowId" String taskId = "taskId" TaskModel task = new TaskModel(taskType: "type1", status: TaskModel.Status.IN_PROGRESS, taskId: taskId, workflowInstanceId: workflowId, rateLimitPerFrequency: 1, taskDefName: "taskDefName", workflowPriority: 10, pollCount: 1) WorkflowModel workflow = new WorkflowModel(workflowId: workflowId, status: WorkflowModel.Status.RUNNING) when: executor.execute(workflowSystemTask, taskId) then: 1 * executionDAOFacade.getTaskModel(taskId) >> task 1 * executionDAOFacade.getWorkflowModel(workflowId, true) >> workflow 1 * executionDAOFacade.updateTask(task) // only one call since pollCount is not incremented 1 * workflowSystemTask.isAsyncComplete(task) >> true 0 * workflowSystemTask.start(workflow, task, workflowExecutor) 1 * workflowSystemTask.execute(workflow, task, workflowExecutor) task.status == TaskModel.Status.IN_PROGRESS task.endTime == 0 // verify that endTime is not set task.pollCount == 1 // verify that poll count is NOT incremented } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/execution/tasks/DoWhileSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.common.utils.TaskUtils import com.netflix.conductor.core.exception.TerminateWorkflowException import com.netflix.conductor.core.execution.WorkflowExecutor import com.netflix.conductor.core.utils.ParametersUtils import com.netflix.conductor.model.TaskModel import com.netflix.conductor.model.WorkflowModel import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Subject import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DO_WHILE import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HTTP class DoWhileSpec extends Specification { @Subject DoWhile doWhile WorkflowExecutor workflowExecutor ObjectMapper objectMapper ParametersUtils parametersUtils TaskModel doWhileTaskModel WorkflowTask task1, task2 TaskModel taskModel1, taskModel2 def setup() { objectMapper = new ObjectMapper(); workflowExecutor = Mock(WorkflowExecutor.class) parametersUtils = new ParametersUtils(objectMapper) task1 = new WorkflowTask(name: 'task1', taskReferenceName: 'task1') task2 = new WorkflowTask(name: 'task2', taskReferenceName: 'task2') doWhile = new DoWhile(parametersUtils) } def "first iteration"() { given: WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 1) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) def workflowModel = new WorkflowModel() workflowModel.tasks = [doWhileTaskModel] when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that return value is true, iteration value is updated in DO_WHILE TaskModel" retVal and: "verify the iteration value" doWhileTaskModel.iteration == 1 doWhileTaskModel.outputData['iteration'] == 1 and: "verify whether the first task is scheduled" 1 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "an iteration - one task is complete and other is not scheduled"() { given: "WorkflowModel consists of one iteration of one task inside DO_WHILE already completed" taskModel1 = createTaskModel(task1) and: "loop over contains two tasks" WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 2) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] // two tasks doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that the return value is false, since the iteration is not complete" !retVal and: "verify that the next iteration is NOT scheduled" 0 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "next iteration - one iteration of all tasks inside DO_WHILE are complete"() { given: "WorkflowModel consists of one iteration of tasks inside DO_WHILE already completed" taskModel1 = createTaskModel(task1) taskModel2 = createTaskModel(task2) WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 2) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1, taskModel2] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData iteration1OutputData[task2.taskReferenceName] = taskModel2.outputData when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that the return value is true, since the iteration is updated" retVal and: "verify that the DO_WHILE TaskModel is correct" doWhileTaskModel.iteration == 2 doWhileTaskModel.outputData['iteration'] == 2 doWhileTaskModel.outputData['1'] == iteration1OutputData doWhileTaskModel.status == TaskModel.Status.IN_PROGRESS and: "verify whether the first task in the next iteration is scheduled" 1 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "next iteration - a task failed in the previous iteration"() { given: "WorkflowModel consists of one iteration of tasks one of which is FAILED" taskModel1 = createTaskModel(task1) taskModel2 = createTaskModel(task2, TaskModel.Status.FAILED) taskModel2.reasonForIncompletion = 'no specific reason, i am tired of success' WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 2) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1, taskModel2] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData iteration1OutputData[task2.taskReferenceName] = taskModel2.outputData when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that return value is true, status is updated" retVal and: "verify the status and reasonForIncompletion fields" doWhileTaskModel.iteration == 1 doWhileTaskModel.outputData['iteration'] == 1 doWhileTaskModel.outputData['1'] == iteration1OutputData doWhileTaskModel.status == TaskModel.Status.FAILED doWhileTaskModel.reasonForIncompletion && doWhileTaskModel.reasonForIncompletion.contains(taskModel2.reasonForIncompletion) and: "verify that next iteration is NOT scheduled" 0 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "next iteration - a task is in progress in the previous iteration"() { given: "WorkflowModel consists of one iteration of tasks inside DO_WHILE already completed" taskModel1 = createTaskModel(task1) taskModel2 = createTaskModel(task2, TaskModel.Status.IN_PROGRESS) taskModel2.outputData = [:] // no output data, task is in progress WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 2) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1, taskModel2] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData iteration1OutputData[task2.taskReferenceName] = [:] when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that return value is false, since the DO_WHILE task model is not updated" !retVal and: "verify that DO_WHILE task model is not modified" doWhileTaskModel.iteration == 1 doWhileTaskModel.outputData['iteration'] == 1 doWhileTaskModel.outputData['1'] == iteration1OutputData doWhileTaskModel.status == TaskModel.Status.IN_PROGRESS and: "verify that next iteration is NOT scheduled" 0 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "final step - all iterations are complete and all tasks in them are successful"() { given: "WorkflowModel consists of one iteration of tasks inside DO_WHILE already completed" taskModel1 = createTaskModel(task1) taskModel2 = createTaskModel(task2) WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) doWhileWorkflowTask.loopCondition = "if (\$.doWhileTask['iteration'] < 1) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1, taskModel2] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData iteration1OutputData[task2.taskReferenceName] = taskModel2.outputData when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that the return value is true, DO_WHILE TaskModel is updated" retVal and: "verify the status and other fields are set correctly" doWhileTaskModel.iteration == 1 doWhileTaskModel.outputData['iteration'] == 1 doWhileTaskModel.outputData['1'] == iteration1OutputData doWhileTaskModel.status == TaskModel.Status.COMPLETED and: "verify that next iteration is not scheduled" 0 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "next iteration - one iteration of all tasks inside DO_WHILE are complete, but the condition is incorrect"() { given: "WorkflowModel consists of one iteration of tasks inside DO_WHILE already completed" taskModel1 = createTaskModel(task1) taskModel2 = createTaskModel(task2) WorkflowTask doWhileWorkflowTask = new WorkflowTask(taskReferenceName: 'doWhileTask', type: TASK_TYPE_DO_WHILE) // condition will produce a ScriptException doWhileWorkflowTask.loopCondition = "if (dollar_sign_goes_here.doWhileTask['iteration'] < 2) { true; } else { false; }" doWhileWorkflowTask.loopOver = [task1, task2] doWhileTaskModel = new TaskModel(workflowTask: doWhileWorkflowTask, taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE, referenceTaskName: doWhileWorkflowTask.taskReferenceName) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS def workflowModel = new WorkflowModel(workflowDefinition: new WorkflowDef(name: 'test_workflow')) // setup the WorkflowModel workflowModel.tasks = [doWhileTaskModel, taskModel1, taskModel2] // this is the expected format of iteration 1's output data def iteration1OutputData = [:] iteration1OutputData[task1.taskReferenceName] = taskModel1.outputData iteration1OutputData[task2.taskReferenceName] = taskModel2.outputData when: def retVal = doWhile.execute(workflowModel, doWhileTaskModel, workflowExecutor) then: "verify that the return value is true since DO_WHILE TaskModel is updated" retVal and: "verify the status of DO_WHILE TaskModel" doWhileTaskModel.iteration == 1 doWhileTaskModel.outputData['iteration'] == 1 doWhileTaskModel.outputData['1'] == iteration1OutputData doWhileTaskModel.status == TaskModel.Status.FAILED_WITH_TERMINAL_ERROR doWhileTaskModel.reasonForIncompletion != null and: "verify that next iteration is not scheduled" 0 * workflowExecutor.scheduleNextIteration(doWhileTaskModel, workflowModel) } def "cancel sets the status as CANCELED"() { given: doWhileTaskModel = new TaskModel(taskId: UUID.randomUUID().toString(), taskType: TASK_TYPE_DO_WHILE) doWhileTaskModel.iteration = 1 doWhileTaskModel.outputData['iteration'] = 1 doWhileTaskModel.status = TaskModel.Status.IN_PROGRESS when: "cancel is called with null for WorkflowModel and WorkflowExecutor" // null is used to note that those arguments are not intended to be used by this method doWhile.cancel(null, doWhileTaskModel, null) then: doWhileTaskModel.status == TaskModel.Status.CANCELED } private static createTaskModel(WorkflowTask workflowTask, TaskModel.Status status = TaskModel.Status.COMPLETED, int iteration = 1) { TaskModel taskModel1 = new TaskModel(workflowTask: workflowTask, taskType: TASK_TYPE_HTTP) taskModel1.status = status taskModel1.outputData = ['k1': 'v1'] taskModel1.iteration = iteration taskModel1.referenceTaskName = TaskUtils.appendIteration(workflowTask.taskReferenceName, iteration) return taskModel1 } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/execution/tasks/EventSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.core.events.EventQueues import com.netflix.conductor.core.events.queue.Message import com.netflix.conductor.core.events.queue.ObservableQueue import com.netflix.conductor.core.exception.NonTransientException import com.netflix.conductor.core.exception.TransientException import com.netflix.conductor.core.utils.ParametersUtils import com.netflix.conductor.model.TaskModel import com.netflix.conductor.model.WorkflowModel import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Subject class EventSpec extends Specification { EventQueues eventQueues ParametersUtils parametersUtils ObjectMapper objectMapper ObservableQueue observableQueue String payloadJSON = "payloadJSON" WorkflowDef testWorkflowDefinition WorkflowModel workflow @Subject Event event def setup() { parametersUtils = Mock(ParametersUtils.class) eventQueues = Mock(EventQueues.class) observableQueue = Mock(ObservableQueue.class) objectMapper = Mock(ObjectMapper.class) { writeValueAsString(_) >> payloadJSON } testWorkflowDefinition = new WorkflowDef(name: "testWorkflow", version: 2) workflow = new WorkflowModel(workflowDefinition: testWorkflowDefinition, workflowId: 'workflowId', correlationId: 'corrId') event = new Event(eventQueues, parametersUtils, objectMapper) } def "verify that event task is NOT async"() { when: def async = event.isAsync() then: !async } def "event cancel calls ack on the queue"() { given: // status is intentionally left as null TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': 'conductor']) String queueName = "conductor:${workflow.workflowName}:${task.referenceTaskName}" when: event.cancel(workflow, task, null) then: task.status == null // task status is NOT updated by the cancel method 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': 'conductor'] 1 * eventQueues.getQueue(queueName) >> observableQueue // Event.cancel sends a list with one Message object to ack 1 * observableQueue.ack({it.size() == 1}) } def "event task with 'conductor' as sink"() { given: TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': 'conductor']) String queueName = "conductor:${workflow.workflowName}:${task.referenceTaskName}" Message expectedMessage when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS verifyOutputData(task, queueName) 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': 'conductor'] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.COMPLETED verifyOutputData(task, queueName) 1 * eventQueues.getQueue(queueName) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish({ it.size() == 1 }) >> { it -> expectedMessage = it[0][0] as Message } verifyMessage(expectedMessage, task) } def "event task with 'conductor:' as sink"() { given: String eventName = 'testEvent' String sinkValue = "conductor:$eventName".toString() TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': sinkValue]) String queueName = "conductor:${workflow.workflowName}:$eventName" Message expectedMessage when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS verifyOutputData(task, queueName) 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.COMPLETED verifyOutputData(task, queueName) 1 * eventQueues.getQueue(queueName) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish({ it.size() == 1 }) >> { it -> expectedMessage = it[0][0] as Message } verifyMessage(expectedMessage, task) } def "event task with 'sqs' as sink"() { given: String eventName = 'testEvent' String sinkValue = "sqs:$eventName".toString() TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': sinkValue]) // for non conductor queues, queueName is the same as the value of the 'sink' field in the inputData String queueName = sinkValue Message expectedMessage when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS verifyOutputData(task, queueName) 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.COMPLETED verifyOutputData(task, queueName) 1 * eventQueues.getQueue(queueName) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish({ it.size() == 1 }) >> { it -> expectedMessage = it[0][0] as Message } verifyMessage(expectedMessage, task) } def "event task with 'conductor' as sink and async complete"() { given: TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': 'conductor', 'asyncComplete': true]) String queueName = "conductor:${workflow.workflowName}:${task.referenceTaskName}" Message expectedMessage when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS verifyOutputData(task, queueName) 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': 'conductor'] when: boolean isTaskUpdateRequired = event.execute(workflow, task, null) then: !isTaskUpdateRequired task.status == TaskModel.Status.IN_PROGRESS verifyOutputData(task, queueName) 1 * eventQueues.getQueue(queueName) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish({ it.size() == 1 }) >> { args -> expectedMessage = args[0][0] as Message } verifyMessage(expectedMessage, task) } def "event task with incorrect 'conductor' sink value"() { given: String sinkValue = 'conductorinvalidsink' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': sinkValue]) when: event.start(workflow, task, null) then: task.status == TaskModel.Status.FAILED task.reasonForIncompletion != null task.reasonForIncompletion.contains('Invalid / Unsupported sink specified:') 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] } def "event task with sink value that does not resolve to a queue"() { given: String sinkValue = 'rabbitmq:abc_123' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', inputData: ['sink': sinkValue]) // for non conductor queues, queueName is the same as the value of the 'sink' field in the inputData String queueName = sinkValue when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.FAILED task.reasonForIncompletion != null 1 * eventQueues.getQueue(queueName) >> {throw new IllegalArgumentException() } } def "publishing to a queue throws a TransientException"() { given: String sinkValue = 'conductor' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', status: TaskModel.Status.SCHEDULED, inputData: ['sink': sinkValue]) when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.FAILED 1 * eventQueues.getQueue(_) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish(_) >> { throw new TransientException("transient error") } } def "publishing to a queue throws a NonTransientException"() { given: String sinkValue = 'conductor' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', status: TaskModel.Status.SCHEDULED, inputData: ['sink': sinkValue]) when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.FAILED task.reasonForIncompletion != null 1 * eventQueues.getQueue(_) >> observableQueue // capture the Message object sent to the publish method. Event.start sends a list with one Message object 1 * observableQueue.publish(_) >> { throw new NonTransientException("fatal error") } } def "event task fails to convert the payload to json"() { given: String sinkValue = 'conductor' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', status: TaskModel.Status.SCHEDULED, inputData: ['sink': sinkValue]) when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.FAILED task.reasonForIncompletion != null 1 * objectMapper.writeValueAsString(_ as Map) >> { throw new JsonParseException(null, "invalid json") } } def "event task fails with an unexpected exception"() { given: String sinkValue = 'conductor' TaskModel task = new TaskModel(referenceTaskName: 'task0', taskId: 'task_id_0', status: TaskModel.Status.SCHEDULED, inputData: ['sink': sinkValue]) when: event.start(workflow, task, null) then: task.status == TaskModel.Status.IN_PROGRESS 1 * parametersUtils.getTaskInputV2(_, workflow, task.taskId, _) >> ['sink': sinkValue] when: event.execute(workflow, task, null) then: task.status == TaskModel.Status.FAILED task.reasonForIncompletion != null 1 * eventQueues.getQueue(_) >> { throw new NullPointerException("some object is null") } } private void verifyOutputData(TaskModel task, String queueName) { assert task.outputData != null assert task.outputData['event_produced'] == queueName assert task.outputData['workflowInstanceId'] == workflow.workflowId assert task.outputData['workflowVersion'] == workflow.workflowVersion assert task.outputData['workflowType'] == workflow.workflowName assert task.outputData['correlationId'] == workflow.correlationId } private void verifyMessage(Message expectedMessage, TaskModel task) { assert expectedMessage != null assert expectedMessage.id == task.taskId assert expectedMessage.receipt == task.taskId assert expectedMessage.payload == payloadJSON } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducerSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks import java.time.Duration import org.junit.Test import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.service.MetadataService import spock.lang.Specification import spock.lang.Subject class IsolatedTaskQueueProducerSpec extends Specification { SystemTaskWorker systemTaskWorker MetadataService metadataService @Subject IsolatedTaskQueueProducer isolatedTaskQueueProducer def asyncSystemTask = new WorkflowSystemTask("asyncTask") { @Override boolean isAsync() { return true } } def setup() { systemTaskWorker = Mock(SystemTaskWorker.class) metadataService = Mock(MetadataService.class) isolatedTaskQueueProducer = new IsolatedTaskQueueProducer(metadataService, [asyncSystemTask] as Set, systemTaskWorker, false, Duration.ofSeconds(10)) } def "addTaskQueuesAddsElementToQueue"() { given: TaskDef taskDef = new TaskDef(isolationGroupId: "isolated") when: isolatedTaskQueueProducer.addTaskQueues() then: 1 * systemTaskWorker.startPolling(asyncSystemTask, "${asyncSystemTask.taskType}-${taskDef.isolationGroupId}") 1 * metadataService.getTaskDefs() >> Collections.singletonList(taskDef) } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/execution/tasks/StartWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks import javax.validation.ConstraintViolation import javax.validation.Validator import com.netflix.conductor.common.config.ObjectMapperProvider import com.netflix.conductor.core.exception.NotFoundException import com.netflix.conductor.core.exception.TransientException import com.netflix.conductor.core.execution.WorkflowExecutor import com.netflix.conductor.core.operation.StartWorkflowOperation import com.netflix.conductor.model.TaskModel import com.netflix.conductor.model.WorkflowModel import spock.lang.Specification import spock.lang.Subject import static com.netflix.conductor.core.execution.tasks.StartWorkflow.START_WORKFLOW_PARAMETER import static com.netflix.conductor.model.TaskModel.Status.FAILED import static com.netflix.conductor.model.TaskModel.Status.SCHEDULED /** * Unit test for StartWorkflow. Success and Javax validation cases are covered by the StartWorkflowSpec in test-harness module. */ class StartWorkflowSpec extends Specification { @Subject StartWorkflow startWorkflow WorkflowExecutor workflowExecutor Validator validator WorkflowModel workflowModel TaskModel taskModel StartWorkflowOperation startWorkflowOperation def setup() { workflowExecutor = Mock(WorkflowExecutor.class) validator = Mock(Validator.class) { validate(_) >> new HashSet>() } startWorkflowOperation = Mock(StartWorkflowOperation.class) def inputData = [:] inputData[START_WORKFLOW_PARAMETER] = ['name': 'some_workflow'] taskModel = new TaskModel(status: SCHEDULED, inputData: inputData) workflowModel = new WorkflowModel() startWorkflow = new StartWorkflow(new ObjectMapperProvider().getObjectMapper(), validator, startWorkflowOperation) } def "StartWorkflow task is asynchronous"() { expect: startWorkflow.isAsync() } def "startWorkflow parameter is missing"() { given: "a task with no start_workflow in input" taskModel.inputData = [:] when: startWorkflow.start(workflowModel, taskModel, workflowExecutor) then: taskModel.status == FAILED taskModel.reasonForIncompletion != null } def "ObjectMapper throws an IllegalArgumentException"() { given: "a task with no start_workflow in input" taskModel.inputData[START_WORKFLOW_PARAMETER] = "I can't be converted to StartWorkflowRequest" when: startWorkflow.start(workflowModel, taskModel, workflowExecutor) then: taskModel.status == FAILED taskModel.reasonForIncompletion != null } def "WorkflowExecutor throws a retryable exception"() { when: startWorkflow.start(workflowModel, taskModel, workflowExecutor) then: taskModel.status == SCHEDULED 1 * startWorkflowOperation.execute(*_) >> { throw new TransientException("") } } def "WorkflowExecutor throws a NotFoundException"() { when: startWorkflow.start(workflowModel, taskModel, workflowExecutor) then: taskModel.status == FAILED taskModel.reasonForIncompletion != null 1 * startWorkflowOperation.execute(*_) >> { throw new NotFoundException("") } } def "WorkflowExecutor throws a RuntimeException"() { when: startWorkflow.start(workflowModel, taskModel, workflowExecutor) then: taskModel.status == FAILED taskModel.reasonForIncompletion != null 1 * startWorkflowOperation.execute(*_) >> { throw new RuntimeException("I am an unexpected exception") } } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/core/operation/StartWorkflowOperationSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.operation import org.springframework.context.ApplicationEventPublisher import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.core.dal.ExecutionDAOFacade import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.metadata.MetadataMapperService import com.netflix.conductor.core.utils.IDGenerator import com.netflix.conductor.core.utils.ParametersUtils import com.netflix.conductor.service.ExecutionLockService import spock.lang.Specification import spock.lang.Subject class StartWorkflowOperationSpec extends Specification { @Subject StartWorkflowOperation startWorkflowOperation MetadataMapperService metadataMapperService IDGenerator idGenerator ParametersUtils parametersUtils ExecutionDAOFacade executionDAOFacade ExecutionLockService executionLockService ApplicationEventPublisher eventPublisher def setup() { metadataMapperService = Mock(MetadataMapperService.class) idGenerator = Mock(IDGenerator.class) parametersUtils = Mock(ParametersUtils.class) executionDAOFacade = Mock(ExecutionDAOFacade.class) executionLockService = Mock(ExecutionLockService.class) eventPublisher = Mock(ApplicationEventPublisher.class) startWorkflowOperation = new StartWorkflowOperation(metadataMapperService, idGenerator, parametersUtils, executionDAOFacade, executionLockService, eventPublisher) } def "simple start workflow"() { given: def workflowDef = new WorkflowDef(name: 'test') def generatedWorkflowId = UUID.randomUUID().toString() StartWorkflowInput startWorkflowInput = new StartWorkflowInput(workflowDefinition: workflowDef, workflowInput: [:]) when: def workflowId = startWorkflowOperation.execute(startWorkflowInput) then: workflowId == generatedWorkflowId 1 * idGenerator.generate() >> generatedWorkflowId 1 * metadataMapperService.populateTaskDefinitions(workflowDef) >> workflowDef 1 * executionLockService.acquireLock(generatedWorkflowId) >> true 1 * executionDAOFacade.createWorkflow(_) 1 * eventPublisher.publishEvent(_) } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/model/TaskModelSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.model import com.netflix.conductor.common.config.ObjectMapperProvider import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Subject class TaskModelSpec extends Specification { @Subject TaskModel taskModel private static final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper() def setup() { taskModel = new TaskModel() } def "check inputData serialization"() { given: String path = "task/input/${UUID.randomUUID()}.json" taskModel.addInput(['key1': 'value1', 'key2': 'value2']) taskModel.externalizeInput(path) when: def json = objectMapper.writeValueAsString(taskModel) println(json) then: json != null JsonNode node = objectMapper.readTree(json) node.path("inputData").isEmpty() node.path("externalInputPayloadStoragePath").isTextual() } def "check outputData serialization"() { given: String path = "task/output/${UUID.randomUUID()}.json" taskModel.addOutput(['key1': 'value1', 'key2': 'value2']) taskModel.externalizeOutput(path) when: def json = objectMapper.writeValueAsString(taskModel) println(json) then: json != null JsonNode node = objectMapper.readTree(json) node.path("outputData").isEmpty() node.path("externalOutputPayloadStoragePath").isTextual() } } ================================================ FILE: core/src/test/groovy/com/netflix/conductor/model/WorkflowModelSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.model import com.netflix.conductor.common.config.ObjectMapperProvider import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Subject class WorkflowModelSpec extends Specification { @Subject WorkflowModel workflowModel private static final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper() def setup() { def workflowDef = new WorkflowDef(name: "test def name", version: 1) workflowModel = new WorkflowModel(workflowDefinition: workflowDef) } def "check input serialization"() { given: String path = "task/input/${UUID.randomUUID()}.json" workflowModel.input = ['key1': 'value1', 'key2': 'value2'] workflowModel.externalizeInput(path) when: def json = objectMapper.writeValueAsString(workflowModel) println(json) then: json != null JsonNode node = objectMapper.readTree(json) node.path("input").isEmpty() node.path("externalInputPayloadStoragePath").isTextual() } def "check output serialization"() { given: String path = "task/output/${UUID.randomUUID()}.json" workflowModel.output = ['key1': 'value1', 'key2': 'value2'] workflowModel.externalizeOutput(path) when: def json = objectMapper.writeValueAsString(workflowModel) println(json) then: json != null JsonNode node = objectMapper.readTree(json) node.path("output").isEmpty() node.path("externalOutputPayloadStoragePath").isTextual() } } ================================================ FILE: core/src/test/java/com/netflix/conductor/TestUtils.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import javax.validation.ConstraintViolation; public class TestUtils { public static Set getConstraintViolationMessages( Set> constraintViolations) { Set messages = new HashSet<>(constraintViolations.size()); messages.addAll( constraintViolations.stream() .map(ConstraintViolation::getMessage) .collect(Collectors.toList())); return messages; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/dal/ExecutionDAOFacadeTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.dal; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.UUID; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.TestDeciderService; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.dao.*; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class ExecutionDAOFacadeTest { private ExecutionDAO executionDAO; private IndexDAO indexDAO; private ExecutionDAOFacade executionDAOFacade; private ExternalPayloadStorageUtils externalPayloadStorageUtils; @Autowired private ObjectMapper objectMapper; @Before public void setUp() { executionDAO = mock(ExecutionDAO.class); QueueDAO queueDAO = mock(QueueDAO.class); indexDAO = mock(IndexDAO.class); externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); RateLimitingDAO rateLimitingDao = mock(RateLimitingDAO.class); ConcurrentExecutionLimitDAO concurrentExecutionLimitDAO = mock(ConcurrentExecutionLimitDAO.class); PollDataDAO pollDataDAO = mock(PollDataDAO.class); ConductorProperties properties = mock(ConductorProperties.class); when(properties.isEventExecutionIndexingEnabled()).thenReturn(true); when(properties.isAsyncIndexingEnabled()).thenReturn(true); executionDAOFacade = new ExecutionDAOFacade( executionDAO, queueDAO, indexDAO, rateLimitingDao, concurrentExecutionLimitDAO, pollDataDAO, objectMapper, properties, externalPayloadStorageUtils); } @Test public void testGetWorkflow() throws Exception { when(executionDAO.getWorkflow(any(), anyBoolean())).thenReturn(new WorkflowModel()); Workflow workflow = executionDAOFacade.getWorkflow("workflowId", true); assertNotNull(workflow); verify(indexDAO, never()).get(any(), any()); } @Test public void testGetWorkflowModel() throws Exception { when(executionDAO.getWorkflow(any(), anyBoolean())).thenReturn(new WorkflowModel()); WorkflowModel workflowModel = executionDAOFacade.getWorkflowModel("workflowId", true); assertNotNull(workflowModel); verify(indexDAO, never()).get(any(), any()); when(executionDAO.getWorkflow(any(), anyBoolean())).thenReturn(null); InputStream stream = ExecutionDAOFacadeTest.class.getResourceAsStream("/test.json"); byte[] bytes = IOUtils.toByteArray(stream); String jsonString = new String(bytes); when(indexDAO.get(any(), any())).thenReturn(jsonString); workflowModel = executionDAOFacade.getWorkflowModel("wokflowId", true); assertNotNull(workflowModel); verify(indexDAO, times(1)).get(any(), any()); } @Test public void testGetWorkflowsByCorrelationId() { when(executionDAO.canSearchAcrossWorkflows()).thenReturn(true); when(executionDAO.getWorkflowsByCorrelationId(any(), any(), anyBoolean())) .thenReturn(Collections.singletonList(new WorkflowModel())); List workflows = executionDAOFacade.getWorkflowsByCorrelationId( "workflowName", "correlationId", true); assertNotNull(workflows); assertEquals(1, workflows.size()); verify(indexDAO, never()) .searchWorkflows(anyString(), anyString(), anyInt(), anyInt(), any()); when(executionDAO.canSearchAcrossWorkflows()).thenReturn(false); List workflowIds = new ArrayList<>(); workflowIds.add("workflowId"); SearchResult searchResult = new SearchResult<>(); searchResult.setResults(workflowIds); when(indexDAO.searchWorkflows(anyString(), anyString(), anyInt(), anyInt(), any())) .thenReturn(searchResult); when(executionDAO.getWorkflow("workflowId", true)).thenReturn(new WorkflowModel()); workflows = executionDAOFacade.getWorkflowsByCorrelationId( "workflowName", "correlationId", true); assertNotNull(workflows); assertEquals(1, workflows.size()); } @Test public void testRemoveWorkflow() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("workflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); TaskModel task = new TaskModel(); task.setTaskId("taskId"); workflow.setTasks(Collections.singletonList(task)); when(executionDAO.getWorkflow(anyString(), anyBoolean())).thenReturn(workflow); executionDAOFacade.removeWorkflow("workflowId", false); verify(executionDAO, times(1)).removeWorkflow(anyString()); verify(executionDAO, never()).removeTask(anyString()); verify(indexDAO, never()).updateWorkflow(anyString(), any(), any()); verify(indexDAO, never()).updateTask(anyString(), anyString(), any(), any()); verify(indexDAO, times(1)).asyncRemoveWorkflow(anyString()); verify(indexDAO, times(1)).asyncRemoveTask(anyString(), anyString()); } @Test public void testArchiveWorkflow() throws Exception { InputStream stream = TestDeciderService.class.getResourceAsStream("/completed.json"); WorkflowModel workflow = objectMapper.readValue(stream, WorkflowModel.class); when(executionDAO.getWorkflow(anyString(), anyBoolean())).thenReturn(workflow); executionDAOFacade.removeWorkflow("workflowId", true); verify(executionDAO, times(1)).removeWorkflow(anyString()); verify(executionDAO, never()).removeTask(anyString()); verify(indexDAO, times(1)).updateWorkflow(anyString(), any(), any()); verify(indexDAO, times(15)).updateTask(anyString(), anyString(), any(), any()); verify(indexDAO, never()).removeWorkflow(anyString()); verify(indexDAO, never()).removeTask(anyString(), anyString()); } @Test public void testAddEventExecution() { when(executionDAO.addEventExecution(any())).thenReturn(false); boolean added = executionDAOFacade.addEventExecution(new EventExecution()); assertFalse(added); verify(indexDAO, never()).addEventExecution(any()); when(executionDAO.addEventExecution(any())).thenReturn(true); added = executionDAOFacade.addEventExecution(new EventExecution()); assertTrue(added); verify(indexDAO, times(1)).asyncAddEventExecution(any()); } @Test(expected = TerminateWorkflowException.class) public void testUpdateTaskThrowsTerminateWorkflowException() { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName("task1"); doThrow(new TerminateWorkflowException("failed")) .when(externalPayloadStorageUtils) .verifyAndUpload(task, ExternalPayloadStorage.PayloadType.TASK_OUTPUT); executionDAOFacade.updateTask(task); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/events/MockObservableQueue.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import rx.Observable; public class MockObservableQueue implements ObservableQueue { private final String uri; private final String name; private final String type; private final Set messages = new TreeSet<>(Comparator.comparing(Message::getId)); public MockObservableQueue(String uri, String name, String type) { this.uri = uri; this.name = name; this.type = type; } @Override public Observable observe() { return Observable.from(messages); } public String getType() { return type; } @Override public String getName() { return name; } @Override public String getURI() { return uri; } @Override public List ack(List msgs) { messages.removeAll(msgs); return msgs.stream().map(Message::getId).collect(Collectors.toList()); } @Override public void publish(List messages) { this.messages.addAll(messages); } @Override public void setUnackTimeout(Message message, long unackTimeout) {} @Override public long size() { return messages.size(); } @Override public String toString() { return "MockObservableQueue [uri=" + uri + ", name=" + name + ", type=" + type + "]"; } @Override public void start() {} @Override public void stop() {} @Override public boolean isRunning() { return false; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/events/MockQueueProvider.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import org.springframework.lang.NonNull; import com.netflix.conductor.core.events.queue.ObservableQueue; public class MockQueueProvider implements EventQueueProvider { private final String type; public MockQueueProvider(String type) { this.type = type; } @Override public String getQueueType() { return "mock"; } @Override @NonNull public ObservableQueue getQueue(String queueURI) { return new MockObservableQueue(queueURI, queueURI, type); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/events/TestDefaultEventProcessor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.retry.support.RetryTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.events.EventHandler.Action; import com.netflix.conductor.common.metadata.events.EventHandler.Action.Type; import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; import com.netflix.conductor.common.metadata.events.EventHandler.TaskDetails; import com.netflix.conductor.core.config.ConductorCoreConfiguration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.execution.evaluators.JavascriptEvaluator; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.JsonUtils; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionService; import com.netflix.conductor.service.MetadataService; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, TestDefaultEventProcessor.TestConfiguration.class, ConductorCoreConfiguration.class }) @RunWith(SpringRunner.class) public class TestDefaultEventProcessor { private String event; private ObservableQueue queue; private MetadataService metadataService; private ExecutionService executionService; private WorkflowExecutor workflowExecutor; private StartWorkflowOperation startWorkflowOperation; private ExternalPayloadStorageUtils externalPayloadStorageUtils; private SimpleActionProcessor actionProcessor; private ParametersUtils parametersUtils; private JsonUtils jsonUtils; private ConductorProperties properties; private Message message; @Autowired private Map evaluators; @Autowired private ObjectMapper objectMapper; @Autowired private @Qualifier("onTransientErrorRetryTemplate") RetryTemplate retryTemplate; @Configuration @ComponentScan(basePackageClasses = {Evaluator.class}) // load all Evaluator beans public static class TestConfiguration {} @Before public void setup() { event = "sqs:arn:account090:sqstest1"; String queueURI = "arn:account090:sqstest1"; metadataService = mock(MetadataService.class); executionService = mock(ExecutionService.class); workflowExecutor = mock(WorkflowExecutor.class); startWorkflowOperation = mock(StartWorkflowOperation.class); externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); actionProcessor = mock(SimpleActionProcessor.class); parametersUtils = new ParametersUtils(objectMapper); jsonUtils = new JsonUtils(objectMapper); queue = mock(ObservableQueue.class); message = new Message( "t0", "{\"Type\":\"Notification\",\"MessageId\":\"7e4e6415-01e9-5caf-abaa-37fd05d446ff\",\"Message\":\"{\\n \\\"testKey1\\\": \\\"level1\\\",\\n \\\"metadata\\\": {\\n \\\"testKey2\\\": 123456 }\\n }\",\"Timestamp\":\"2018-08-10T21:22:05.029Z\",\"SignatureVersion\":\"1\"}", "t0"); when(queue.getURI()).thenReturn(queueURI); when(queue.getName()).thenReturn(queueURI); when(queue.getType()).thenReturn("sqs"); properties = mock(ConductorProperties.class); when(properties.isEventMessageIndexingEnabled()).thenReturn(true); when(properties.getEventProcessorThreadCount()).thenReturn(2); } @Test public void testEventProcessor() { // setup event handler EventHandler eventHandler = new EventHandler(); eventHandler.setName(UUID.randomUUID().toString()); eventHandler.setActive(true); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "dev"); Action startWorkflowAction = new Action(); startWorkflowAction.setAction(Type.start_workflow); startWorkflowAction.setStart_workflow(new StartWorkflow()); startWorkflowAction.getStart_workflow().setName("workflow_x"); startWorkflowAction.getStart_workflow().setVersion(1); startWorkflowAction.getStart_workflow().setTaskToDomain(taskToDomain); eventHandler.getActions().add(startWorkflowAction); Action completeTaskAction = new Action(); completeTaskAction.setAction(Type.complete_task); completeTaskAction.setComplete_task(new TaskDetails()); completeTaskAction.getComplete_task().setTaskRefName("task_x"); completeTaskAction.getComplete_task().setWorkflowId(UUID.randomUUID().toString()); completeTaskAction.getComplete_task().setOutput(new HashMap<>()); eventHandler.getActions().add(completeTaskAction); eventHandler.setEvent(event); when(metadataService.getEventHandlersForEvent(event, true)) .thenReturn(Collections.singletonList(eventHandler)); when(executionService.addEventExecution(any())).thenReturn(true); when(queue.rePublishIfNoAck()).thenReturn(false); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName(startWorkflowAction.getStart_workflow().getName()); startWorkflowInput.setVersion(startWorkflowAction.getStart_workflow().getVersion()); startWorkflowInput.setCorrelationId( startWorkflowAction.getStart_workflow().getCorrelationId()); startWorkflowInput.setEvent(event); String id = UUID.randomUUID().toString(); AtomicBoolean started = new AtomicBoolean(false); doAnswer( (Answer) invocation -> { started.set(true); return id; }) .when(startWorkflowOperation) .execute( argThat( argument -> startWorkflowAction .getStart_workflow() .getName() .equals(argument.getName()) && startWorkflowAction .getStart_workflow() .getVersion() .equals(argument.getVersion()) && event.equals(argument.getEvent()))); AtomicBoolean completed = new AtomicBoolean(false); doAnswer( (Answer) invocation -> { completed.set(true); return null; }) .when(workflowExecutor) .updateTask(any()); TaskModel task = new TaskModel(); task.setReferenceTaskName(completeTaskAction.getComplete_task().getTaskRefName()); WorkflowModel workflow = new WorkflowModel(); workflow.setTasks(Collections.singletonList(task)); when(workflowExecutor.getWorkflow( completeTaskAction.getComplete_task().getWorkflowId(), true)) .thenReturn(workflow); doNothing().when(externalPayloadStorageUtils).verifyAndUpload(any(), any()); SimpleActionProcessor actionProcessor = new SimpleActionProcessor( workflowExecutor, parametersUtils, jsonUtils, startWorkflowOperation); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); assertTrue(completed.get()); verify(queue, atMost(1)).ack(any()); verify(queue, never()).nack(any()); verify(queue, never()).publish(any()); } @Test public void testEventHandlerWithCondition() { EventHandler eventHandler = new EventHandler(); eventHandler.setName("cms_intermediate_video_ingest_handler"); eventHandler.setActive(true); eventHandler.setEvent("sqs:dev_cms_asset_ingest_queue"); eventHandler.setCondition( "$.Message.testKey1 == 'level1' && $.Message.metadata.testKey2 == 123456"); Map workflowInput = new LinkedHashMap<>(); workflowInput.put("param1", "${Message.metadata.testKey2}"); workflowInput.put("param2", "SQS-${MessageId}"); Action startWorkflowAction = new Action(); startWorkflowAction.setAction(Type.start_workflow); startWorkflowAction.setStart_workflow(new StartWorkflow()); startWorkflowAction.getStart_workflow().setName("cms_artwork_automation"); startWorkflowAction.getStart_workflow().setVersion(1); startWorkflowAction.getStart_workflow().setInput(workflowInput); startWorkflowAction.setExpandInlineJSON(true); eventHandler.getActions().add(startWorkflowAction); eventHandler.setEvent(event); when(metadataService.getEventHandlersForEvent(event, true)) .thenReturn(Collections.singletonList(eventHandler)); when(executionService.addEventExecution(any())).thenReturn(true); when(queue.rePublishIfNoAck()).thenReturn(false); String id = UUID.randomUUID().toString(); AtomicBoolean started = new AtomicBoolean(false); doAnswer( (Answer) invocation -> { started.set(true); return id; }) .when(startWorkflowOperation) .execute( argThat( argument -> startWorkflowAction .getStart_workflow() .getName() .equals(argument.getName()) && startWorkflowAction .getStart_workflow() .getVersion() .equals(argument.getVersion()) && event.equals(argument.getEvent()))); SimpleActionProcessor actionProcessor = new SimpleActionProcessor( workflowExecutor, parametersUtils, jsonUtils, startWorkflowOperation); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); } @Test public void testEventHandlerWithConditionEvaluator() { EventHandler eventHandler = new EventHandler(); eventHandler.setName("cms_intermediate_video_ingest_handler"); eventHandler.setActive(true); eventHandler.setEvent("sqs:dev_cms_asset_ingest_queue"); eventHandler.setEvaluatorType(JavascriptEvaluator.NAME); eventHandler.setCondition( "$.Message.testKey1 == 'level1' && $.Message.metadata.testKey2 == 123456"); Map workflowInput = new LinkedHashMap<>(); workflowInput.put("param1", "${Message.metadata.testKey2}"); workflowInput.put("param2", "SQS-${MessageId}"); Action startWorkflowAction = new Action(); startWorkflowAction.setAction(Type.start_workflow); startWorkflowAction.setStart_workflow(new StartWorkflow()); startWorkflowAction.getStart_workflow().setName("cms_artwork_automation"); startWorkflowAction.getStart_workflow().setVersion(1); startWorkflowAction.getStart_workflow().setInput(workflowInput); startWorkflowAction.setExpandInlineJSON(true); eventHandler.getActions().add(startWorkflowAction); eventHandler.setEvent(event); when(metadataService.getEventHandlersForEvent(event, true)) .thenReturn(Collections.singletonList(eventHandler)); when(executionService.addEventExecution(any())).thenReturn(true); when(queue.rePublishIfNoAck()).thenReturn(false); String id = UUID.randomUUID().toString(); AtomicBoolean started = new AtomicBoolean(false); doAnswer( (Answer) invocation -> { started.set(true); return id; }) .when(startWorkflowOperation) .execute( argThat( argument -> startWorkflowAction .getStart_workflow() .getName() .equals(argument.getName()) && startWorkflowAction .getStart_workflow() .getVersion() .equals(argument.getVersion()) && event.equals(argument.getEvent()))); SimpleActionProcessor actionProcessor = new SimpleActionProcessor( workflowExecutor, parametersUtils, jsonUtils, startWorkflowOperation); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); } @Test public void testEventProcessorWithRetriableError() { EventHandler eventHandler = new EventHandler(); eventHandler.setName(UUID.randomUUID().toString()); eventHandler.setActive(true); eventHandler.setEvent(event); Action completeTaskAction = new Action(); completeTaskAction.setAction(Type.complete_task); completeTaskAction.setComplete_task(new TaskDetails()); completeTaskAction.getComplete_task().setTaskRefName("task_x"); completeTaskAction.getComplete_task().setWorkflowId(UUID.randomUUID().toString()); completeTaskAction.getComplete_task().setOutput(new HashMap<>()); eventHandler.getActions().add(completeTaskAction); when(queue.rePublishIfNoAck()).thenReturn(false); when(metadataService.getEventHandlersForEvent(event, true)) .thenReturn(Collections.singletonList(eventHandler)); when(executionService.addEventExecution(any())).thenReturn(true); when(actionProcessor.execute(any(), any(), any(), any())) .thenThrow(new TransientException("some retriable error")); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); eventProcessor.handle(queue, message); verify(queue, never()).ack(any()); verify(queue, never()).nack(any()); verify(queue, atLeastOnce()).publish(any()); } @Test public void testEventProcessorWithNonRetriableError() { EventHandler eventHandler = new EventHandler(); eventHandler.setName(UUID.randomUUID().toString()); eventHandler.setActive(true); eventHandler.setEvent(event); Action completeTaskAction = new Action(); completeTaskAction.setAction(Type.complete_task); completeTaskAction.setComplete_task(new TaskDetails()); completeTaskAction.getComplete_task().setTaskRefName("task_x"); completeTaskAction.getComplete_task().setWorkflowId(UUID.randomUUID().toString()); completeTaskAction.getComplete_task().setOutput(new HashMap<>()); eventHandler.getActions().add(completeTaskAction); when(metadataService.getEventHandlersForEvent(event, true)) .thenReturn(Collections.singletonList(eventHandler)); when(executionService.addEventExecution(any())).thenReturn(true); when(actionProcessor.execute(any(), any(), any(), any())) .thenThrow(new IllegalArgumentException("some non-retriable error")); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); eventProcessor.handle(queue, message); verify(queue, atMost(1)).ack(any()); verify(queue, never()).publish(any()); } @Test public void testExecuteInvalidAction() { AtomicInteger executeInvoked = new AtomicInteger(0); doAnswer( (Answer>) invocation -> { executeInvoked.incrementAndGet(); throw new UnsupportedOperationException("error"); }) .when(actionProcessor) .execute(any(), any(), any(), any()); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setName("handler"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); eventExecution.setEvent("event"); Action action = new Action(); eventExecution.setAction(Type.start_workflow); eventProcessor.execute(eventExecution, action, "payload"); assertEquals(1, executeInvoked.get()); assertEquals(EventExecution.Status.FAILED, eventExecution.getStatus()); assertNotNull(eventExecution.getOutput().get("exception")); } @Test public void testExecuteNonRetriableException() { AtomicInteger executeInvoked = new AtomicInteger(0); doAnswer( (Answer>) invocation -> { executeInvoked.incrementAndGet(); throw new IllegalArgumentException("some non-retriable error"); }) .when(actionProcessor) .execute(any(), any(), any(), any()); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); eventExecution.setEvent("event"); eventExecution.setName("handler"); Action action = new Action(); action.setAction(Type.start_workflow); eventExecution.setAction(Type.start_workflow); eventProcessor.execute(eventExecution, action, "payload"); assertEquals(1, executeInvoked.get()); assertEquals(EventExecution.Status.FAILED, eventExecution.getStatus()); assertNotNull(eventExecution.getOutput().get("exception")); } @Test public void testExecuteTransientException() { AtomicInteger executeInvoked = new AtomicInteger(0); doAnswer( (Answer>) invocation -> { executeInvoked.incrementAndGet(); throw new TransientException("some retriable error"); }) .when(actionProcessor) .execute(any(), any(), any(), any()); DefaultEventProcessor eventProcessor = new DefaultEventProcessor( executionService, metadataService, actionProcessor, jsonUtils, properties, objectMapper, evaluators, retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); eventExecution.setEvent("event"); Action action = new Action(); action.setAction(Type.start_workflow); eventProcessor.execute(eventExecution, action, "payload"); assertEquals(3, executeInvoked.get()); assertNull(eventExecution.getOutput().get("exception")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/events/TestScriptEval.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.HashMap; import java.util.Map; import org.junit.Test; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class TestScriptEval { @Test public void testScript() throws Exception { Map payload = new HashMap<>(); Map app = new HashMap<>(); app.put("name", "conductor"); app.put("version", 2.0); app.put("license", "Apache 2.0"); payload.put("app", app); payload.put("author", "Netflix"); payload.put("oss", true); String script1 = "$.app.name == 'conductor'"; // true String script2 = "$.version > 3"; // false String script3 = "$.oss"; // true String script4 = "$.author == 'me'"; // false assertTrue(ScriptEvaluator.evalBool(script1, payload)); assertFalse(ScriptEvaluator.evalBool(script2, payload)); assertTrue(ScriptEvaluator.evalBool(script3, payload)); assertFalse(ScriptEvaluator.evalBool(script4, payload)); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/events/TestSimpleActionProcessor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.events; import java.util.HashMap; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.events.EventHandler.Action; import com.netflix.conductor.common.metadata.events.EventHandler.Action.Type; import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; import com.netflix.conductor.common.metadata.events.EventHandler.TaskDetails; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskResult.Status; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.JsonUtils; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class TestSimpleActionProcessor { private WorkflowExecutor workflowExecutor; private ExternalPayloadStorageUtils externalPayloadStorageUtils; private SimpleActionProcessor actionProcessor; private StartWorkflowOperation startWorkflowOperation; @Autowired private ObjectMapper objectMapper; @Before public void setup() { externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); workflowExecutor = mock(WorkflowExecutor.class); startWorkflowOperation = mock(StartWorkflowOperation.class); actionProcessor = new SimpleActionProcessor( workflowExecutor, new ParametersUtils(objectMapper), new JsonUtils(objectMapper), startWorkflowOperation); } @SuppressWarnings({"unchecked", "rawtypes"}) @Test public void testStartWorkflow_correlationId() throws Exception { StartWorkflow startWorkflow = new StartWorkflow(); startWorkflow.setName("testWorkflow"); startWorkflow.getInput().put("testInput", "${testId}"); startWorkflow.setCorrelationId("${correlationId}"); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "dev"); startWorkflow.setTaskToDomain(taskToDomain); Action action = new Action(); action.setAction(Type.start_workflow); action.setStart_workflow(startWorkflow); Object payload = objectMapper.readValue( "{\"correlationId\":\"test-id\", \"testId\":\"test_1\"}", Object.class); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testWorkflow"); workflowDef.setVersion(1); when(startWorkflowOperation.execute(any())).thenReturn("workflow_1"); Map output = actionProcessor.execute(action, payload, "testEvent", "testMessage"); assertNotNull(output); assertEquals("workflow_1", output.get("workflowId")); ArgumentCaptor startWorkflowInputArgumentCaptor = ArgumentCaptor.forClass(StartWorkflowInput.class); verify(startWorkflowOperation).execute(startWorkflowInputArgumentCaptor.capture()); StartWorkflowInput capturedValue = startWorkflowInputArgumentCaptor.getValue(); assertEquals("test_1", capturedValue.getWorkflowInput().get("testInput")); assertEquals("test-id", capturedValue.getCorrelationId()); assertEquals( "testMessage", capturedValue.getWorkflowInput().get("conductor.event.messageId")); assertEquals("testEvent", capturedValue.getWorkflowInput().get("conductor.event.name")); assertEquals(taskToDomain, capturedValue.getTaskToDomain()); } @SuppressWarnings({"unchecked", "rawtypes"}) @Test public void testStartWorkflow() throws Exception { StartWorkflow startWorkflow = new StartWorkflow(); startWorkflow.setName("testWorkflow"); startWorkflow.getInput().put("testInput", "${testId}"); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "dev"); startWorkflow.setTaskToDomain(taskToDomain); Action action = new Action(); action.setAction(Type.start_workflow); action.setStart_workflow(startWorkflow); Object payload = objectMapper.readValue("{\"testId\":\"test_1\"}", Object.class); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testWorkflow"); workflowDef.setVersion(1); when(startWorkflowOperation.execute(any())).thenReturn("workflow_1"); Map output = actionProcessor.execute(action, payload, "testEvent", "testMessage"); assertNotNull(output); assertEquals("workflow_1", output.get("workflowId")); ArgumentCaptor startWorkflowInputArgumentCaptor = ArgumentCaptor.forClass(StartWorkflowInput.class); verify(startWorkflowOperation).execute(startWorkflowInputArgumentCaptor.capture()); StartWorkflowInput capturedArgument = startWorkflowInputArgumentCaptor.getValue(); assertEquals("test_1", capturedArgument.getWorkflowInput().get("testInput")); assertNull(capturedArgument.getCorrelationId()); assertEquals( "testMessage", capturedArgument.getWorkflowInput().get("conductor.event.messageId")); assertEquals("testEvent", capturedArgument.getWorkflowInput().get("conductor.event.name")); assertEquals(taskToDomain, capturedArgument.getTaskToDomain()); } @Test public void testCompleteTask() throws Exception { TaskDetails taskDetails = new TaskDetails(); taskDetails.setWorkflowId("${workflowId}"); taskDetails.setTaskRefName("testTask"); taskDetails.getOutput().put("someNEKey", "${Message.someNEKey}"); taskDetails.getOutput().put("someKey", "${Message.someKey}"); taskDetails.getOutput().put("someNullKey", "${Message.someNullKey}"); Action action = new Action(); action.setAction(Type.complete_task); action.setComplete_task(taskDetails); String payloadJson = "{\"workflowId\":\"workflow_1\",\"Message\":{\"someKey\":\"someData\",\"someNullKey\":null}}"; Object payload = objectMapper.readValue(payloadJson, Object.class); TaskModel task = new TaskModel(); task.setReferenceTaskName("testTask"); WorkflowModel workflow = new WorkflowModel(); workflow.getTasks().add(task); when(workflowExecutor.getWorkflow(eq("workflow_1"), anyBoolean())).thenReturn(workflow); doNothing().when(externalPayloadStorageUtils).verifyAndUpload(any(), any()); actionProcessor.execute(action, payload, "testEvent", "testMessage"); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); verify(workflowExecutor).updateTask(argumentCaptor.capture()); assertEquals(Status.COMPLETED, argumentCaptor.getValue().getStatus()); assertEquals( "testMessage", argumentCaptor.getValue().getOutputData().get("conductor.event.messageId")); assertEquals( "testEvent", argumentCaptor.getValue().getOutputData().get("conductor.event.name")); assertEquals("workflow_1", argumentCaptor.getValue().getOutputData().get("workflowId")); assertEquals("testTask", argumentCaptor.getValue().getOutputData().get("taskRefName")); assertEquals("someData", argumentCaptor.getValue().getOutputData().get("someKey")); // Assert values not in message are evaluated to null assertTrue("testTask", argumentCaptor.getValue().getOutputData().containsKey("someNEKey")); // Assert null values from message are kept assertTrue( "testTask", argumentCaptor.getValue().getOutputData().containsKey("someNullKey")); assertNull("testTask", argumentCaptor.getValue().getOutputData().get("someNullKey")); } @Test public void testCompleteLoopOverTask() throws Exception { TaskDetails taskDetails = new TaskDetails(); taskDetails.setWorkflowId("${workflowId}"); taskDetails.setTaskRefName("testTask"); taskDetails.getOutput().put("someNEKey", "${Message.someNEKey}"); taskDetails.getOutput().put("someKey", "${Message.someKey}"); taskDetails.getOutput().put("someNullKey", "${Message.someNullKey}"); Action action = new Action(); action.setAction(Type.complete_task); action.setComplete_task(taskDetails); String payloadJson = "{\"workflowId\":\"workflow_1\", \"taskRefName\":\"testTask\", \"Message\":{\"someKey\":\"someData\",\"someNullKey\":null}}"; Object payload = objectMapper.readValue(payloadJson, Object.class); TaskModel task = new TaskModel(); task.setIteration(1); task.setReferenceTaskName("testTask__1"); WorkflowModel workflow = new WorkflowModel(); workflow.getTasks().add(task); when(workflowExecutor.getWorkflow(eq("workflow_1"), anyBoolean())).thenReturn(workflow); doNothing().when(externalPayloadStorageUtils).verifyAndUpload(any(), any()); actionProcessor.execute(action, payload, "testEvent", "testMessage"); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); verify(workflowExecutor).updateTask(argumentCaptor.capture()); assertEquals(Status.COMPLETED, argumentCaptor.getValue().getStatus()); assertEquals( "testMessage", argumentCaptor.getValue().getOutputData().get("conductor.event.messageId")); assertEquals( "testEvent", argumentCaptor.getValue().getOutputData().get("conductor.event.name")); assertEquals("workflow_1", argumentCaptor.getValue().getOutputData().get("workflowId")); assertEquals("testTask", argumentCaptor.getValue().getOutputData().get("taskRefName")); assertEquals("someData", argumentCaptor.getValue().getOutputData().get("someKey")); // Assert values not in message are evaluated to null assertTrue("testTask", argumentCaptor.getValue().getOutputData().containsKey("someNEKey")); // Assert null values from message are kept assertTrue( "testTask", argumentCaptor.getValue().getOutputData().containsKey("someNullKey")); assertNull("testTask", argumentCaptor.getValue().getOutputData().get("someNullKey")); } @Test public void testCompleteTaskByTaskId() throws Exception { TaskDetails taskDetails = new TaskDetails(); taskDetails.setWorkflowId("${workflowId}"); taskDetails.setTaskId("${taskId}"); Action action = new Action(); action.setAction(Type.complete_task); action.setComplete_task(taskDetails); Object payload = objectMapper.readValue( "{\"workflowId\":\"workflow_1\", \"taskId\":\"task_1\"}", Object.class); TaskModel task = new TaskModel(); task.setTaskId("task_1"); task.setReferenceTaskName("testTask"); when(workflowExecutor.getTask(eq("task_1"))).thenReturn(task); doNothing().when(externalPayloadStorageUtils).verifyAndUpload(any(), any()); actionProcessor.execute(action, payload, "testEvent", "testMessage"); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); verify(workflowExecutor).updateTask(argumentCaptor.capture()); assertEquals(Status.COMPLETED, argumentCaptor.getValue().getStatus()); assertEquals( "testMessage", argumentCaptor.getValue().getOutputData().get("conductor.event.messageId")); assertEquals( "testEvent", argumentCaptor.getValue().getOutputData().get("conductor.event.name")); assertEquals("workflow_1", argumentCaptor.getValue().getOutputData().get("workflowId")); assertEquals("task_1", argumentCaptor.getValue().getOutputData().get("taskId")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/TestDeciderOutcomes.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.io.InputStream; import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.unit.DataSize; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.DeciderService.DeciderOutcome; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.execution.mapper.DecisionTaskMapper; import com.netflix.conductor.core.execution.mapper.DynamicTaskMapper; import com.netflix.conductor.core.execution.mapper.EventTaskMapper; import com.netflix.conductor.core.execution.mapper.ForkJoinDynamicTaskMapper; import com.netflix.conductor.core.execution.mapper.ForkJoinTaskMapper; import com.netflix.conductor.core.execution.mapper.HTTPTaskMapper; import com.netflix.conductor.core.execution.mapper.JoinTaskMapper; import com.netflix.conductor.core.execution.mapper.SimpleTaskMapper; import com.netflix.conductor.core.execution.mapper.SubWorkflowTaskMapper; import com.netflix.conductor.core.execution.mapper.SwitchTaskMapper; import com.netflix.conductor.core.execution.mapper.TaskMapper; import com.netflix.conductor.core.execution.mapper.UserDefinedTaskMapper; import com.netflix.conductor.core.execution.mapper.WaitTaskMapper; import com.netflix.conductor.core.execution.tasks.Decision; import com.netflix.conductor.core.execution.tasks.Join; import com.netflix.conductor.core.execution.tasks.Switch; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.DECISION; import static com.netflix.conductor.common.metadata.tasks.TaskType.DYNAMIC; import static com.netflix.conductor.common.metadata.tasks.TaskType.EVENT; import static com.netflix.conductor.common.metadata.tasks.TaskType.FORK_JOIN; import static com.netflix.conductor.common.metadata.tasks.TaskType.FORK_JOIN_DYNAMIC; import static com.netflix.conductor.common.metadata.tasks.TaskType.HTTP; import static com.netflix.conductor.common.metadata.tasks.TaskType.JOIN; import static com.netflix.conductor.common.metadata.tasks.TaskType.SIMPLE; import static com.netflix.conductor.common.metadata.tasks.TaskType.SUB_WORKFLOW; import static com.netflix.conductor.common.metadata.tasks.TaskType.SWITCH; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DECISION; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SWITCH; import static com.netflix.conductor.common.metadata.tasks.TaskType.USER_DEFINED; import static com.netflix.conductor.common.metadata.tasks.TaskType.WAIT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, TestDeciderOutcomes.TestConfiguration.class }) @RunWith(SpringRunner.class) public class TestDeciderOutcomes { private DeciderService deciderService; @Autowired private Map evaluators; @Autowired private ObjectMapper objectMapper; @Autowired private SystemTaskRegistry systemTaskRegistry; @Configuration @ComponentScan(basePackageClasses = {Evaluator.class}) // load all Evaluator beans. public static class TestConfiguration { @Bean(TASK_TYPE_DECISION) public Decision decision() { return new Decision(); } @Bean(TASK_TYPE_SWITCH) public Switch switchTask() { return new Switch(); } @Bean(TASK_TYPE_JOIN) public Join join() { return new Join(); } @Bean public SystemTaskRegistry systemTaskRegistry(Set tasks) { return new SystemTaskRegistry(tasks); } } @Before public void init() { MetadataDAO metadataDAO = mock(MetadataDAO.class); ExternalPayloadStorageUtils externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); ConductorProperties properties = mock(ConductorProperties.class); when(properties.getTaskInputPayloadSizeThreshold()).thenReturn(DataSize.ofKilobytes(10L)); when(properties.getMaxTaskInputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10240L)); TaskDef taskDef = new TaskDef(); taskDef.setRetryCount(1); taskDef.setName("mockTaskDef"); taskDef.setResponseTimeoutSeconds(60 * 60); when(metadataDAO.getTaskDef(anyString())).thenReturn(taskDef); ParametersUtils parametersUtils = new ParametersUtils(objectMapper); Map taskMappers = new HashMap<>(); taskMappers.put(DECISION.name(), new DecisionTaskMapper()); taskMappers.put(SWITCH.name(), new SwitchTaskMapper(evaluators)); taskMappers.put(DYNAMIC.name(), new DynamicTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(FORK_JOIN.name(), new ForkJoinTaskMapper()); taskMappers.put(JOIN.name(), new JoinTaskMapper()); taskMappers.put( FORK_JOIN_DYNAMIC.name(), new ForkJoinDynamicTaskMapper( new IDGenerator(), parametersUtils, objectMapper, metadataDAO)); taskMappers.put( USER_DEFINED.name(), new UserDefinedTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(SIMPLE.name(), new SimpleTaskMapper(parametersUtils)); taskMappers.put( SUB_WORKFLOW.name(), new SubWorkflowTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(EVENT.name(), new EventTaskMapper(parametersUtils)); taskMappers.put(WAIT.name(), new WaitTaskMapper(parametersUtils)); taskMappers.put(HTTP.name(), new HTTPTaskMapper(parametersUtils, metadataDAO)); this.deciderService = new DeciderService( new IDGenerator(), parametersUtils, metadataDAO, externalPayloadStorageUtils, systemTaskRegistry, taskMappers, Duration.ofMinutes(60)); } @Test public void testWorkflowWithNoTasks() throws Exception { InputStream stream = new ClassPathResource("./conditional_flow.json").getInputStream(); WorkflowDef def = objectMapper.readValue(stream, WorkflowDef.class); assertNotNull(def); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCreateTime(0L); workflow.getInput().put("param1", "nested"); workflow.getInput().put("param2", "one"); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertFalse(outcome.isComplete); assertTrue(outcome.tasksToBeUpdated.isEmpty()); assertEquals(3, outcome.tasksToBeScheduled.size()); outcome.tasksToBeScheduled.forEach(t -> t.setStatus(TaskModel.Status.COMPLETED)); workflow.getTasks().addAll(outcome.tasksToBeScheduled); outcome = deciderService.decide(workflow); assertFalse(outcome.isComplete); assertEquals(outcome.tasksToBeUpdated.toString(), 3, outcome.tasksToBeUpdated.size()); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals("junit_task_3", outcome.tasksToBeScheduled.get(0).getTaskDefName()); } @Test public void testWorkflowWithNoTasksWithSwitch() throws Exception { InputStream stream = new ClassPathResource("./conditional_flow_with_switch.json").getInputStream(); WorkflowDef def = objectMapper.readValue(stream, WorkflowDef.class); assertNotNull(def); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCreateTime(0L); workflow.getInput().put("param1", "nested"); workflow.getInput().put("param2", "one"); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertFalse(outcome.isComplete); assertTrue(outcome.tasksToBeUpdated.isEmpty()); assertEquals(3, outcome.tasksToBeScheduled.size()); outcome.tasksToBeScheduled.forEach(t -> t.setStatus(TaskModel.Status.COMPLETED)); workflow.getTasks().addAll(outcome.tasksToBeScheduled); outcome = deciderService.decide(workflow); assertFalse(outcome.isComplete); assertEquals(outcome.tasksToBeUpdated.toString(), 3, outcome.tasksToBeUpdated.size()); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals("junit_task_3", outcome.tasksToBeScheduled.get(0).getTaskDefName()); } @Test public void testRetries() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("test_task"); workflowTask.setType("USER_TASK"); workflowTask.setTaskReferenceName("t0"); workflowTask.getInputParameters().put("taskId", "${CPEWF_TASK_ID}"); workflowTask.getInputParameters().put("requestId", "${workflow.input.requestId}"); workflowTask.setTaskDefinition(new TaskDef("test_task")); def.getTasks().add(workflowTask); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.getInput().put("requestId", 123); workflow.setCreateTime(System.currentTimeMillis()); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals( workflowTask.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); String task1Id = outcome.tasksToBeScheduled.get(0).getTaskId(); assertEquals(task1Id, outcome.tasksToBeScheduled.get(0).getInputData().get("taskId")); assertEquals(123, outcome.tasksToBeScheduled.get(0).getInputData().get("requestId")); outcome.tasksToBeScheduled.get(0).setStatus(TaskModel.Status.FAILED); workflow.getTasks().addAll(outcome.tasksToBeScheduled); outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(1, outcome.tasksToBeUpdated.size()); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals(task1Id, outcome.tasksToBeUpdated.get(0).getTaskId()); assertNotSame(task1Id, outcome.tasksToBeScheduled.get(0).getTaskId()); assertEquals( outcome.tasksToBeScheduled.get(0).getTaskId(), outcome.tasksToBeScheduled.get(0).getInputData().get("taskId")); assertEquals(task1Id, outcome.tasksToBeScheduled.get(0).getRetriedTaskId()); assertEquals(123, outcome.tasksToBeScheduled.get(0).getInputData().get("requestId")); WorkflowTask fork = new WorkflowTask(); fork.setName("fork0"); fork.setWorkflowTaskType(TaskType.FORK_JOIN_DYNAMIC); fork.setTaskReferenceName("fork0"); fork.setDynamicForkTasksInputParamName("forkedInputs"); fork.setDynamicForkTasksParam("forks"); fork.getInputParameters().put("forks", "${workflow.input.forks}"); fork.getInputParameters().put("forkedInputs", "${workflow.input.forkedInputs}"); WorkflowTask join = new WorkflowTask(); join.setName("join0"); join.setType("JOIN"); join.setTaskReferenceName("join0"); def.getTasks().clear(); def.getTasks().add(fork); def.getTasks().add(join); List forks = new LinkedList<>(); Map> forkedInputs = new HashMap<>(); for (int i = 0; i < 1; i++) { WorkflowTask wft = new WorkflowTask(); wft.setName("f" + i); wft.setTaskReferenceName("f" + i); wft.setWorkflowTaskType(TaskType.SIMPLE); wft.getInputParameters().put("requestId", "${workflow.input.requestId}"); wft.getInputParameters().put("taskId", "${CPEWF_TASK_ID}"); wft.setTaskDefinition(new TaskDef("f" + i)); forks.add(wft); Map input = new HashMap<>(); input.put("k", "v"); input.put("k1", 1); forkedInputs.put(wft.getTaskReferenceName(), input); } workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.getInput().put("requestId", 123); workflow.setCreateTime(System.currentTimeMillis()); workflow.getInput().put("forks", forks); workflow.getInput().put("forkedInputs", forkedInputs); outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(3, outcome.tasksToBeScheduled.size()); assertEquals(0, outcome.tasksToBeUpdated.size()); assertEquals("v", outcome.tasksToBeScheduled.get(1).getInputData().get("k")); assertEquals(1, outcome.tasksToBeScheduled.get(1).getInputData().get("k1")); assertEquals( outcome.tasksToBeScheduled.get(1).getTaskId(), outcome.tasksToBeScheduled.get(1).getInputData().get("taskId")); task1Id = outcome.tasksToBeScheduled.get(1).getTaskId(); outcome.tasksToBeScheduled.get(1).setStatus(TaskModel.Status.FAILED); for (TaskModel taskToBeScheduled : outcome.tasksToBeScheduled) { taskToBeScheduled.setUpdateTime(System.currentTimeMillis()); } workflow.getTasks().addAll(outcome.tasksToBeScheduled); outcome = deciderService.decide(workflow); assertTrue( outcome.tasksToBeScheduled.stream() .anyMatch(task1 -> task1.getReferenceTaskName().equals("f0"))); Optional optionalTask = outcome.tasksToBeScheduled.stream() .filter(t -> t.getReferenceTaskName().equals("f0")) .findFirst(); assertTrue(optionalTask.isPresent()); TaskModel task = optionalTask.get(); assertEquals("v", task.getInputData().get("k")); assertEquals(1, task.getInputData().get("k1")); assertEquals(task.getTaskId(), task.getInputData().get("taskId")); assertNotSame(task1Id, task.getTaskId()); assertEquals(task1Id, task.getRetriedTaskId()); } @Test public void testOptional() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowTask task1 = new WorkflowTask(); task1.setName("task0"); task1.setType("SIMPLE"); task1.setTaskReferenceName("t0"); task1.getInputParameters().put("taskId", "${CPEWF_TASK_ID}"); task1.setOptional(true); task1.setTaskDefinition(new TaskDef("task0")); WorkflowTask task2 = new WorkflowTask(); task2.setName("task1"); task2.setType("SIMPLE"); task2.setTaskReferenceName("t1"); task2.setTaskDefinition(new TaskDef("task1")); def.getTasks().add(task1); def.getTasks().add(task2); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCreateTime(System.currentTimeMillis()); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals( task1.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); for (int i = 0; i < 3; i++) { String task1Id = outcome.tasksToBeScheduled.get(0).getTaskId(); assertEquals(task1Id, outcome.tasksToBeScheduled.get(0).getInputData().get("taskId")); workflow.getTasks().clear(); workflow.getTasks().addAll(outcome.tasksToBeScheduled); workflow.getTasks().get(0).setStatus(TaskModel.Status.FAILED); outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(1, outcome.tasksToBeUpdated.size()); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals(TaskModel.Status.FAILED, workflow.getTasks().get(0).getStatus()); assertEquals(task1Id, outcome.tasksToBeUpdated.get(0).getTaskId()); assertEquals( task1.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertEquals(i + 1, outcome.tasksToBeScheduled.get(0).getRetryCount()); } String task1Id = outcome.tasksToBeScheduled.get(0).getTaskId(); workflow.getTasks().clear(); workflow.getTasks().addAll(outcome.tasksToBeScheduled); workflow.getTasks().get(0).setStatus(TaskModel.Status.FAILED); outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(1, outcome.tasksToBeUpdated.size()); assertEquals(1, outcome.tasksToBeScheduled.size()); assertEquals( TaskModel.Status.COMPLETED_WITH_ERRORS, workflow.getTasks().get(0).getStatus()); assertEquals(task1Id, outcome.tasksToBeUpdated.get(0).getTaskId()); assertEquals( task2.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); } @Test public void testOptionalWithDynamicFork() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowTask task1 = new WorkflowTask(); task1.setName("fork0"); task1.setWorkflowTaskType(TaskType.FORK_JOIN_DYNAMIC); task1.setTaskReferenceName("fork0"); task1.setDynamicForkTasksInputParamName("forkedInputs"); task1.setDynamicForkTasksParam("forks"); task1.getInputParameters().put("forks", "${workflow.input.forks}"); task1.getInputParameters().put("forkedInputs", "${workflow.input.forkedInputs}"); WorkflowTask task2 = new WorkflowTask(); task2.setName("join0"); task2.setType("JOIN"); task2.setTaskReferenceName("join0"); def.getTasks().add(task1); def.getTasks().add(task2); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); List forks = new LinkedList<>(); Map> forkedInputs = new HashMap<>(); for (int i = 0; i < 3; i++) { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("f" + i); workflowTask.setTaskReferenceName("f" + i); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setOptional(true); workflowTask.setTaskDefinition(new TaskDef("f" + i)); forks.add(workflowTask); forkedInputs.put(workflowTask.getTaskReferenceName(), new HashMap<>()); } workflow.getInput().put("forks", forks); workflow.getInput().put("forkedInputs", forkedInputs); workflow.setCreateTime(System.currentTimeMillis()); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(5, outcome.tasksToBeScheduled.size()); assertEquals(0, outcome.tasksToBeUpdated.size()); assertEquals(TASK_TYPE_FORK, outcome.tasksToBeScheduled.get(0).getTaskType()); assertEquals(TaskModel.Status.COMPLETED, outcome.tasksToBeScheduled.get(0).getStatus()); for (int retryCount = 0; retryCount < 4; retryCount++) { for (TaskModel taskToBeScheduled : outcome.tasksToBeScheduled) { if (taskToBeScheduled.getTaskDefName().equals("join0")) { assertEquals(TaskModel.Status.IN_PROGRESS, taskToBeScheduled.getStatus()); } else if (taskToBeScheduled.getTaskType().matches("(f0|f1|f2)")) { assertEquals(TaskModel.Status.SCHEDULED, taskToBeScheduled.getStatus()); taskToBeScheduled.setStatus(TaskModel.Status.FAILED); } taskToBeScheduled.setUpdateTime(System.currentTimeMillis()); } workflow.getTasks().addAll(outcome.tasksToBeScheduled); outcome = deciderService.decide(workflow); assertNotNull(outcome); } assertEquals(TASK_TYPE_JOIN, outcome.tasksToBeScheduled.get(0).getTaskType()); for (int i = 0; i < 3; i++) { assertEquals( TaskModel.Status.COMPLETED_WITH_ERRORS, outcome.tasksToBeUpdated.get(i).getStatus()); assertEquals("f" + (i), outcome.tasksToBeUpdated.get(i).getTaskDefName()); } assertEquals(TaskModel.Status.IN_PROGRESS, outcome.tasksToBeScheduled.get(0).getStatus()); new Join().execute(workflow, outcome.tasksToBeScheduled.get(0), null); assertEquals( TaskModel.Status.COMPLETED_WITH_ERRORS, outcome.tasksToBeScheduled.get(0).getStatus()); } @Test public void testDecisionCases() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowTask even = new WorkflowTask(); even.setName("even"); even.setType("SIMPLE"); even.setTaskReferenceName("even"); even.setTaskDefinition(new TaskDef("even")); WorkflowTask odd = new WorkflowTask(); odd.setName("odd"); odd.setType("SIMPLE"); odd.setTaskReferenceName("odd"); odd.setTaskDefinition(new TaskDef("odd")); WorkflowTask defaultt = new WorkflowTask(); defaultt.setName("defaultt"); defaultt.setType("SIMPLE"); defaultt.setTaskReferenceName("defaultt"); defaultt.setTaskDefinition(new TaskDef("defaultt")); WorkflowTask decide = new WorkflowTask(); decide.setName("decide"); decide.setWorkflowTaskType(TaskType.DECISION); decide.setTaskReferenceName("d0"); decide.getInputParameters().put("Id", "${workflow.input.Id}"); decide.getInputParameters().put("location", "${workflow.input.location}"); decide.setCaseExpression( "if ($.Id == null) 'bad input'; else if ( ($.Id != null && $.Id % 2 == 0) || $.location == 'usa') 'even'; else 'odd'; "); decide.getDecisionCases().put("even", Collections.singletonList(even)); decide.getDecisionCases().put("odd", Collections.singletonList(odd)); decide.setDefaultCase(Collections.singletonList(defaultt)); def.getTasks().add(decide); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCreateTime(System.currentTimeMillis()); DeciderOutcome outcome = deciderService.decide(workflow); assertNotNull(outcome); assertEquals(2, outcome.tasksToBeScheduled.size()); assertEquals( decide.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertEquals( defaultt.getTaskReferenceName(), outcome.tasksToBeScheduled.get(1).getReferenceTaskName()); // default assertEquals( Collections.singletonList("bad input"), outcome.tasksToBeScheduled.get(0).getOutputData().get("caseOutput")); workflow.getInput().put("Id", 9); workflow.getInput().put("location", "usa"); outcome = deciderService.decide(workflow); assertEquals(2, outcome.tasksToBeScheduled.size()); assertEquals( decide.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertEquals( even.getTaskReferenceName(), outcome.tasksToBeScheduled .get(1) .getReferenceTaskName()); // even because of location == usa assertEquals( Collections.singletonList("even"), outcome.tasksToBeScheduled.get(0).getOutputData().get("caseOutput")); workflow.getInput().put("Id", 9); workflow.getInput().put("location", "canada"); outcome = deciderService.decide(workflow); assertEquals(2, outcome.tasksToBeScheduled.size()); assertEquals( decide.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertEquals( odd.getTaskReferenceName(), outcome.tasksToBeScheduled.get(1).getReferenceTaskName()); // odd assertEquals( Collections.singletonList("odd"), outcome.tasksToBeScheduled.get(0).getOutputData().get("caseOutput")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/TestDeciderService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.io.IOException; import java.io.InputStream; import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskDef.TimeoutPolicy; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.DeciderService.DeciderOutcome; import com.netflix.conductor.core.execution.mapper.TaskMapper; import com.netflix.conductor.core.execution.tasks.SubWorkflow; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Spectator; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.*; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration( classes = {TestObjectMapperConfiguration.class, TestDeciderService.TestConfiguration.class}) @RunWith(SpringRunner.class) public class TestDeciderService { @Configuration @ComponentScan(basePackageClasses = TaskMapper.class) // loads all TaskMapper beans public static class TestConfiguration { @Bean(TASK_TYPE_SUB_WORKFLOW) public SubWorkflow subWorkflow(ObjectMapper objectMapper) { return new SubWorkflow(objectMapper, mock(StartWorkflowOperation.class)); } @Bean("asyncCompleteSystemTask") public WorkflowSystemTaskStub asyncCompleteSystemTask() { return new WorkflowSystemTaskStub("asyncCompleteSystemTask") { @Override public boolean isAsyncComplete(TaskModel task) { return true; } }; } @Bean public SystemTaskRegistry systemTaskRegistry(Set tasks) { return new SystemTaskRegistry(tasks); } @Bean public MetadataDAO mockMetadataDAO() { return mock(MetadataDAO.class); } @Bean public Map taskMapperMap(Collection taskMappers) { return taskMappers.stream() .collect(Collectors.toMap(TaskMapper::getTaskType, Function.identity())); } @Bean public ParametersUtils parametersUtils(ObjectMapper mapper) { return new ParametersUtils(mapper); } @Bean public IDGenerator idGenerator() { return new IDGenerator(); } } private DeciderService deciderService; private ExternalPayloadStorageUtils externalPayloadStorageUtils; private static Registry registry; @Autowired private ObjectMapper objectMapper; @Autowired private SystemTaskRegistry systemTaskRegistry; @Autowired @Qualifier("taskMapperMap") private Map taskMappers; @Autowired private ParametersUtils parametersUtils; @Autowired private MetadataDAO metadataDAO; @Rule public ExpectedException exception = ExpectedException.none(); @BeforeClass public static void init() { registry = new DefaultRegistry(); Spectator.globalRegistry().add(registry); } @Before public void setup() { externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("TestDeciderService"); workflowDef.setVersion(1); TaskDef taskDef = new TaskDef(); when(metadataDAO.getTaskDef(any())).thenReturn(taskDef); when(metadataDAO.getLatestWorkflowDef(any())).thenReturn(Optional.of(workflowDef)); deciderService = new DeciderService( new IDGenerator(), parametersUtils, metadataDAO, externalPayloadStorageUtils, systemTaskRegistry, taskMappers, Duration.ofMinutes(60)); } @Test public void testGetTaskInputV2() { WorkflowModel workflow = createDefaultWorkflow(); workflow.getWorkflowDefinition().setSchemaVersion(2); Map inputParams = new HashMap<>(); inputParams.put("workflowInputParam", "${workflow.input.requestId}"); inputParams.put("taskOutputParam", "${task2.output.location}"); inputParams.put("taskOutputParam2", "${task2.output.locationBad}"); inputParams.put("taskOutputParam3", "${task3.output.location}"); inputParams.put("constParam", "Some String value"); inputParams.put("nullValue", null); inputParams.put("task2Status", "${task2.status}"); inputParams.put("channelMap", "${workflow.input.channelMapping}"); Map taskInput = parametersUtils.getTaskInput(inputParams, workflow, null, null); assertNotNull(taskInput); assertTrue(taskInput.containsKey("workflowInputParam")); assertTrue(taskInput.containsKey("taskOutputParam")); assertTrue(taskInput.containsKey("taskOutputParam2")); assertTrue(taskInput.containsKey("taskOutputParam3")); assertNull(taskInput.get("taskOutputParam2")); assertNotNull(taskInput.get("channelMap")); assertEquals(5, taskInput.get("channelMap")); assertEquals("request id 001", taskInput.get("workflowInputParam")); assertEquals("http://location", taskInput.get("taskOutputParam")); assertNull(taskInput.get("taskOutputParam3")); assertNull(taskInput.get("nullValue")); assertEquals( workflow.getTasks().get(0).getStatus().name(), taskInput.get("task2Status")); // task2 and task3 are the tasks respectively } @Test public void testGetTaskInputV2Partial() { WorkflowModel workflow = createDefaultWorkflow(); System.setProperty("EC2_INSTANCE", "i-123abcdef990"); workflow.getWorkflowDefinition().setSchemaVersion(2); Map inputParams = new HashMap<>(); inputParams.put("workflowInputParam", "${workflow.input.requestId}"); inputParams.put("workfowOutputParam", "${workflow.output.name}"); inputParams.put("taskOutputParam", "${task2.output.location}"); inputParams.put("taskOutputParam2", "${task2.output.locationBad}"); inputParams.put("taskOutputParam3", "${task3.output.location}"); inputParams.put("constParam", "Some String value &"); inputParams.put("partial", "${task2.output.location}/something?host=${EC2_INSTANCE}"); inputParams.put("jsonPathExtracted", "${workflow.output.names[*].year}"); inputParams.put("secondName", "${workflow.output.names[1].name}"); inputParams.put( "concatenatedName", "The Band is: ${workflow.output.names[1].name}-\t${EC2_INSTANCE}"); TaskDef taskDef = new TaskDef(); taskDef.getInputTemplate().put("opname", "${workflow.output.name}"); List listParams = new LinkedList<>(); List listParams2 = new LinkedList<>(); listParams2.add("${workflow.input.requestId}-10-${EC2_INSTANCE}"); listParams.add(listParams2); Map map = new HashMap<>(); map.put("name", "${workflow.output.names[0].name}"); map.put("hasAwards", "${workflow.input.hasAwards}"); listParams.add(map); taskDef.getInputTemplate().put("listValues", listParams); Map taskInput = parametersUtils.getTaskInput(inputParams, workflow, taskDef, null); assertNotNull(taskInput); assertTrue(taskInput.containsKey("workflowInputParam")); assertTrue(taskInput.containsKey("taskOutputParam")); assertTrue(taskInput.containsKey("taskOutputParam2")); assertTrue(taskInput.containsKey("taskOutputParam3")); assertNull(taskInput.get("taskOutputParam2")); assertNotNull(taskInput.get("jsonPathExtracted")); assertTrue(taskInput.get("jsonPathExtracted") instanceof List); assertNotNull(taskInput.get("secondName")); assertTrue(taskInput.get("secondName") instanceof String); assertEquals("The Doors", taskInput.get("secondName")); assertEquals("The Band is: The Doors-\ti-123abcdef990", taskInput.get("concatenatedName")); assertEquals("request id 001", taskInput.get("workflowInputParam")); assertEquals("http://location", taskInput.get("taskOutputParam")); assertNull(taskInput.get("taskOutputParam3")); assertNotNull(taskInput.get("partial")); assertEquals("http://location/something?host=i-123abcdef990", taskInput.get("partial")); } @SuppressWarnings("unchecked") @Test public void testGetTaskInput() { Map ip = new HashMap<>(); ip.put("workflowInputParam", "${workflow.input.requestId}"); ip.put("taskOutputParam", "${task2.output.location}"); List> json = new LinkedList<>(); Map m1 = new HashMap<>(); m1.put("name", "person name"); m1.put("city", "New York"); m1.put("phone", 2120001234); m1.put("status", "${task2.output.isPersonActive}"); Map m2 = new HashMap<>(); m2.put("employer", "City Of New York"); m2.put("color", "purple"); m2.put("requestId", "${workflow.input.requestId}"); json.add(m1); json.add(m2); ip.put("complexJson", json); WorkflowDef def = new WorkflowDef(); def.setName("testGetTaskInput"); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.getInput().put("requestId", "request id 001"); TaskModel task = new TaskModel(); task.setReferenceTaskName("task2"); task.addOutput("location", "http://location"); task.addOutput("isPersonActive", true); workflow.getTasks().add(task); Map taskInput = parametersUtils.getTaskInput(ip, workflow, null, null); assertNotNull(taskInput); assertTrue(taskInput.containsKey("workflowInputParam")); assertTrue(taskInput.containsKey("taskOutputParam")); assertEquals("request id 001", taskInput.get("workflowInputParam")); assertEquals("http://location", taskInput.get("taskOutputParam")); assertNotNull(taskInput.get("complexJson")); assertTrue(taskInput.get("complexJson") instanceof List); List> resolvedInput = (List>) taskInput.get("complexJson"); assertEquals(2, resolvedInput.size()); } @Test public void testGetTaskInputV1() { Map ip = new HashMap<>(); ip.put("workflowInputParam", "workflow.input.requestId"); ip.put("taskOutputParam", "task2.output.location"); WorkflowDef def = new WorkflowDef(); def.setSchemaVersion(1); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.getInput().put("requestId", "request id 001"); TaskModel task = new TaskModel(); task.setReferenceTaskName("task2"); task.addOutput("location", "http://location"); task.addOutput("isPersonActive", true); workflow.getTasks().add(task); Map taskInput = parametersUtils.getTaskInput(ip, workflow, null, null); assertNotNull(taskInput); assertTrue(taskInput.containsKey("workflowInputParam")); assertTrue(taskInput.containsKey("taskOutputParam")); assertEquals("request id 001", taskInput.get("workflowInputParam")); assertEquals("http://location", taskInput.get("taskOutputParam")); } @Test public void testGetTaskInputV2WithInputTemplate() { TaskDef def = new TaskDef(); Map inputTemplate = new HashMap<>(); inputTemplate.put("url", "https://some_url:7004"); inputTemplate.put("default_url", "https://default_url:7004"); inputTemplate.put("someKey", "someValue"); def.getInputTemplate().putAll(inputTemplate); Map workflowInput = new HashMap<>(); workflowInput.put("some_new_url", "https://some_new_url:7004"); workflowInput.put("workflow_input_url", "https://workflow_input_url:7004"); workflowInput.put("some_other_key", "some_other_value"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testGetTaskInputV2WithInputTemplate"); workflowDef.setVersion(1); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setInput(workflowInput); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.getInputParameters().put("url", "${workflow.input.some_new_url}"); workflowTask .getInputParameters() .put("workflow_input_url", "${workflow.input.workflow_input_url}"); workflowTask.getInputParameters().put("someKey", "${workflow.input.someKey}"); workflowTask.getInputParameters().put("someOtherKey", "${workflow.input.some_other_key}"); workflowTask .getInputParameters() .put("someNowhereToBeFoundKey", "${workflow.input.some_ne_key}"); Map taskInput = parametersUtils.getTaskInputV2( workflowTask.getInputParameters(), workflow, null, def); assertTrue(taskInput.containsKey("url")); assertTrue(taskInput.containsKey("default_url")); assertEquals(taskInput.get("url"), "https://some_new_url:7004"); assertEquals(taskInput.get("default_url"), "https://default_url:7004"); assertEquals(taskInput.get("workflow_input_url"), "https://workflow_input_url:7004"); assertEquals("some_other_value", taskInput.get("someOtherKey")); assertEquals("someValue", taskInput.get("someKey")); assertNull(taskInput.get("someNowhereToBeFoundKey")); } @Test public void testGetNextTask() { WorkflowDef def = createNestedWorkflow(); WorkflowTask firstTask = def.getTasks().get(0); assertNotNull(firstTask); assertEquals("fork1", firstTask.getTaskReferenceName()); WorkflowTask nextAfterFirst = def.getNextTask(firstTask.getTaskReferenceName()); assertNotNull(nextAfterFirst); assertEquals("join1", nextAfterFirst.getTaskReferenceName()); WorkflowTask fork2 = def.getTaskByRefName("fork2"); assertNotNull(fork2); assertEquals("fork2", fork2.getTaskReferenceName()); WorkflowTask taskAfterFork2 = def.getNextTask("fork2"); assertNotNull(taskAfterFork2); assertEquals("join2", taskAfterFork2.getTaskReferenceName()); WorkflowTask t2 = def.getTaskByRefName("t2"); assertNotNull(t2); assertEquals("t2", t2.getTaskReferenceName()); WorkflowTask taskAfterT2 = def.getNextTask("t2"); assertNotNull(taskAfterT2); assertEquals("t4", taskAfterT2.getTaskReferenceName()); WorkflowTask taskAfterT3 = def.getNextTask("t3"); assertNotNull(taskAfterT3); assertEquals(DECISION.name(), taskAfterT3.getType()); assertEquals("d1", taskAfterT3.getTaskReferenceName()); WorkflowTask taskAfterT4 = def.getNextTask("t4"); assertNotNull(taskAfterT4); assertEquals("join2", taskAfterT4.getTaskReferenceName()); WorkflowTask taskAfterT6 = def.getNextTask("t6"); assertNotNull(taskAfterT6); assertEquals("t9", taskAfterT6.getTaskReferenceName()); WorkflowTask taskAfterJoin2 = def.getNextTask("join2"); assertNotNull(taskAfterJoin2); assertEquals("join1", taskAfterJoin2.getTaskReferenceName()); WorkflowTask taskAfterJoin1 = def.getNextTask("join1"); assertNotNull(taskAfterJoin1); assertEquals("t5", taskAfterJoin1.getTaskReferenceName()); WorkflowTask taskAfterSubWF = def.getNextTask("sw1"); assertNotNull(taskAfterSubWF); assertEquals("join1", taskAfterSubWF.getTaskReferenceName()); WorkflowTask taskAfterT9 = def.getNextTask("t9"); assertNotNull(taskAfterT9); assertEquals("join2", taskAfterT9.getTaskReferenceName()); } @Test public void testCaseStatement() { WorkflowDef def = createConditionalWF(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCreateTime(0L); workflow.setWorkflowId("a"); workflow.setCorrelationId("b"); workflow.setStatus(WorkflowModel.Status.RUNNING); DeciderOutcome outcome = deciderService.decide(workflow); List scheduledTasks = outcome.tasksToBeScheduled; assertNotNull(scheduledTasks); assertEquals(2, scheduledTasks.size()); assertEquals(TaskModel.Status.IN_PROGRESS, scheduledTasks.get(0).getStatus()); assertEquals(TaskModel.Status.SCHEDULED, scheduledTasks.get(1).getStatus()); } @Test public void testGetTaskByRef() { WorkflowModel workflow = new WorkflowModel(); TaskModel t1 = new TaskModel(); t1.setReferenceTaskName("ref"); t1.setSeq(0); t1.setStatus(TaskModel.Status.TIMED_OUT); TaskModel t2 = new TaskModel(); t2.setReferenceTaskName("ref"); t2.setSeq(1); t2.setStatus(TaskModel.Status.FAILED); TaskModel t3 = new TaskModel(); t3.setReferenceTaskName("ref"); t3.setSeq(2); t3.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().add(t1); workflow.getTasks().add(t2); workflow.getTasks().add(t3); TaskModel task = workflow.getTaskByRefName("ref"); assertNotNull(task); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(t3.getSeq(), task.getSeq()); } @Test public void testTaskTimeout() { Counter counter = registry.counter("task_timeout", "class", "WorkflowMonitor", "taskType", "test"); long counterCount = counter.count(); TaskDef taskType = new TaskDef(); taskType.setName("test"); taskType.setTimeoutPolicy(TimeoutPolicy.RETRY); taskType.setTimeoutSeconds(1); TaskModel task = new TaskModel(); task.setTaskType(taskType.getName()); task.setStartTime(System.currentTimeMillis() - 2_000); // 2 seconds ago! task.setStatus(TaskModel.Status.IN_PROGRESS); deciderService.checkTaskTimeout(taskType, task); // Task should be marked as timed out assertEquals(TaskModel.Status.TIMED_OUT, task.getStatus()); assertNotNull(task.getReasonForIncompletion()); assertEquals(++counterCount, counter.count()); taskType.setTimeoutPolicy(TimeoutPolicy.ALERT_ONLY); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setReasonForIncompletion(null); deciderService.checkTaskTimeout(taskType, task); // Nothing will happen assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals(++counterCount, counter.count()); boolean exception = false; taskType.setTimeoutPolicy(TimeoutPolicy.TIME_OUT_WF); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setReasonForIncompletion(null); try { deciderService.checkTaskTimeout(taskType, task); } catch (TerminateWorkflowException tw) { exception = true; } assertTrue(exception); assertEquals(TaskModel.Status.TIMED_OUT, task.getStatus()); assertNotNull(task.getReasonForIncompletion()); assertEquals(++counterCount, counter.count()); taskType.setTimeoutPolicy(TimeoutPolicy.TIME_OUT_WF); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setReasonForIncompletion(null); deciderService.checkTaskTimeout(null, task); // this will be a no-op assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals(counterCount, counter.count()); } @Test public void testCheckTaskPollTimeout() { Counter counter = registry.counter("task_timeout", "class", "WorkflowMonitor", "taskType", "test"); long counterCount = counter.count(); TaskDef taskType = new TaskDef(); taskType.setName("test"); taskType.setTimeoutPolicy(TimeoutPolicy.RETRY); taskType.setPollTimeoutSeconds(1); TaskModel task = new TaskModel(); task.setTaskType(taskType.getName()); task.setScheduledTime(System.currentTimeMillis() - 2_000); task.setStatus(TaskModel.Status.SCHEDULED); deciderService.checkTaskPollTimeout(taskType, task); assertEquals(++counterCount, counter.count()); assertEquals(TaskModel.Status.TIMED_OUT, task.getStatus()); assertNotNull(task.getReasonForIncompletion()); task.setScheduledTime(System.currentTimeMillis()); task.setReasonForIncompletion(null); task.setStatus(TaskModel.Status.SCHEDULED); deciderService.checkTaskPollTimeout(taskType, task); assertEquals(counterCount, counter.count()); assertEquals(TaskModel.Status.SCHEDULED, task.getStatus()); assertNull(task.getReasonForIncompletion()); } @SuppressWarnings("unchecked") @Test public void testConcurrentTaskInputCalc() throws InterruptedException { TaskDef def = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("path", "${workflow.input.inputLocation}"); inputMap.put("type", "${workflow.input.sourceType}"); inputMap.put("channelMapping", "${workflow.input.channelMapping}"); List> input = new LinkedList<>(); input.add(inputMap); Map body = new HashMap<>(); body.put("input", input); def.getInputTemplate().putAll(body); ExecutorService executorService = Executors.newFixedThreadPool(10); final int[] result = new int[10]; CountDownLatch latch = new CountDownLatch(10); for (int i = 0; i < 10; i++) { final int x = i; executorService.submit( () -> { try { Map workflowInput = new HashMap<>(); workflowInput.put("outputLocation", "baggins://outputlocation/" + x); workflowInput.put("inputLocation", "baggins://inputlocation/" + x); workflowInput.put("sourceType", "MuxedSource"); workflowInput.put("channelMapping", x); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testConcurrentTaskInputCalc"); workflowDef.setVersion(1); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setInput(workflowInput); Map taskInput = parametersUtils.getTaskInputV2( new HashMap<>(), workflow, null, def); Object reqInputObj = taskInput.get("input"); assertNotNull(reqInputObj); assertTrue(reqInputObj instanceof List); List> reqInput = (List>) reqInputObj; Object cmObj = reqInput.get(0).get("channelMapping"); assertNotNull(cmObj); if (!(cmObj instanceof Number)) { result[x] = -1; } else { Number channelMapping = (Number) cmObj; result[x] = channelMapping.intValue(); } latch.countDown(); } catch (Exception e) { e.printStackTrace(); } }); } latch.await(1, TimeUnit.MINUTES); if (latch.getCount() > 0) { fail( "Executions did not complete in a minute. Something wrong with the build server?"); } executorService.shutdownNow(); for (int i = 0; i < result.length; i++) { assertEquals(i, result[i]); } } @SuppressWarnings("unchecked") @Test public void testTaskRetry() { WorkflowModel workflow = createDefaultWorkflow(); workflow.getWorkflowDefinition().setSchemaVersion(2); Map inputParams = new HashMap<>(); inputParams.put("workflowInputParam", "${workflow.input.requestId}"); inputParams.put("taskOutputParam", "${task2.output.location}"); inputParams.put("constParam", "Some String value"); inputParams.put("nullValue", null); inputParams.put("task2Status", "${task2.status}"); inputParams.put("null", null); inputParams.put("task_id", "${CPEWF_TASK_ID}"); Map env = new HashMap<>(); env.put("env_task_id", "${CPEWF_TASK_ID}"); inputParams.put("env", env); Map taskInput = parametersUtils.getTaskInput(inputParams, workflow, null, "t1"); TaskModel task = new TaskModel(); task.getInputData().putAll(taskInput); task.setStatus(TaskModel.Status.FAILED); task.setTaskId("t1"); TaskDef taskDef = new TaskDef(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.getInputParameters().put("task_id", "${CPEWF_TASK_ID}"); workflowTask.getInputParameters().put("env", env); Optional task2 = deciderService.retry(taskDef, workflowTask, task, workflow); assertEquals("t1", task.getInputData().get("task_id")); assertEquals( "t1", ((Map) task.getInputData().get("env")).get("env_task_id")); assertNotSame(task.getTaskId(), task2.get().getTaskId()); assertEquals(task2.get().getTaskId(), task2.get().getInputData().get("task_id")); assertEquals( task2.get().getTaskId(), ((Map) task2.get().getInputData().get("env")).get("env_task_id")); TaskModel task3 = new TaskModel(); task3.getInputData().putAll(taskInput); task3.setStatus(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); task3.setTaskId("t1"); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); exception.expect(TerminateWorkflowException.class); deciderService.retry(taskDef, workflowTask, task3, workflow); } @SuppressWarnings("unchecked") @Test public void testWorkflowTaskRetry() { WorkflowModel workflow = createDefaultWorkflow(); workflow.getWorkflowDefinition().setSchemaVersion(2); Map inputParams = new HashMap<>(); inputParams.put("workflowInputParam", "${workflow.input.requestId}"); inputParams.put("taskOutputParam", "${task2.output.location}"); inputParams.put("constParam", "Some String value"); inputParams.put("nullValue", null); inputParams.put("task2Status", "${task2.status}"); inputParams.put("null", null); inputParams.put("task_id", "${CPEWF_TASK_ID}"); Map env = new HashMap<>(); env.put("env_task_id", "${CPEWF_TASK_ID}"); inputParams.put("env", env); Map taskInput = parametersUtils.getTaskInput(inputParams, workflow, null, "t1"); // Create a first failed task TaskModel task = new TaskModel(); task.getInputData().putAll(taskInput); task.setStatus(TaskModel.Status.FAILED); task.setTaskId("t1"); TaskDef taskDef = new TaskDef(); assertEquals(3, taskDef.getRetryCount()); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.getInputParameters().put("task_id", "${CPEWF_TASK_ID}"); workflowTask.getInputParameters().put("env", env); workflowTask.setRetryCount(1); // Retry the failed task and assert that a new one has been created Optional task2 = deciderService.retry(taskDef, workflowTask, task, workflow); assertEquals("t1", task.getInputData().get("task_id")); assertEquals( "t1", ((Map) task.getInputData().get("env")).get("env_task_id")); assertNotSame(task.getTaskId(), task2.get().getTaskId()); assertEquals(task2.get().getTaskId(), task2.get().getInputData().get("task_id")); assertEquals( task2.get().getTaskId(), ((Map) task2.get().getInputData().get("env")).get("env_task_id")); // Set the retried task to FAILED, retry it again and assert that the workflow failed task2.get().setStatus(TaskModel.Status.FAILED); exception.expect(TerminateWorkflowException.class); final Optional task3 = deciderService.retry(taskDef, workflowTask, task2.get(), workflow); assertFalse(task3.isPresent()); assertEquals(WorkflowModel.Status.FAILED, workflow.getStatus()); } @Test public void testLinearBackoff() { WorkflowModel workflow = createDefaultWorkflow(); TaskModel task = new TaskModel(); task.setStatus(TaskModel.Status.FAILED); task.setTaskId("t1"); TaskDef taskDef = new TaskDef(); taskDef.setRetryDelaySeconds(60); taskDef.setRetryLogic(TaskDef.RetryLogic.LINEAR_BACKOFF); taskDef.setBackoffScaleFactor(2); WorkflowTask workflowTask = new WorkflowTask(); Optional task2 = deciderService.retry(taskDef, workflowTask, task, workflow); assertEquals(120, task2.get().getCallbackAfterSeconds()); // 60*2*1 Optional task3 = deciderService.retry(taskDef, workflowTask, task2.get(), workflow); assertEquals(240, task3.get().getCallbackAfterSeconds()); // 60*2*2 Optional task4 = deciderService.retry(taskDef, workflowTask, task3.get(), workflow); // // 60*2*3 assertEquals(360, task4.get().getCallbackAfterSeconds()); // 60*2*3 taskDef.setRetryCount(Integer.MAX_VALUE); task4.get().setRetryCount(Integer.MAX_VALUE - 100); Optional task5 = deciderService.retry(taskDef, workflowTask, task4.get(), workflow); assertEquals(Integer.MAX_VALUE, task5.get().getCallbackAfterSeconds()); } @Test public void testExponentialBackoff() { WorkflowModel workflow = createDefaultWorkflow(); TaskModel task = new TaskModel(); task.setStatus(TaskModel.Status.FAILED); task.setTaskId("t1"); TaskDef taskDef = new TaskDef(); taskDef.setRetryDelaySeconds(60); taskDef.setRetryLogic(TaskDef.RetryLogic.EXPONENTIAL_BACKOFF); WorkflowTask workflowTask = new WorkflowTask(); Optional task2 = deciderService.retry(taskDef, workflowTask, task, workflow); assertEquals(60, task2.get().getCallbackAfterSeconds()); Optional task3 = deciderService.retry(taskDef, workflowTask, task2.get(), workflow); assertEquals(120, task3.get().getCallbackAfterSeconds()); Optional task4 = deciderService.retry(taskDef, workflowTask, task3.get(), workflow); assertEquals(240, task4.get().getCallbackAfterSeconds()); taskDef.setRetryCount(Integer.MAX_VALUE); task4.get().setRetryCount(Integer.MAX_VALUE - 100); Optional task5 = deciderService.retry(taskDef, workflowTask, task4.get(), workflow); assertEquals(Integer.MAX_VALUE, task5.get().getCallbackAfterSeconds()); } @Test public void testFork() throws IOException { InputStream stream = TestDeciderService.class.getResourceAsStream("/test.json"); WorkflowModel workflow = objectMapper.readValue(stream, WorkflowModel.class); DeciderOutcome outcome = deciderService.decide(workflow); assertFalse(outcome.isComplete); assertEquals(5, outcome.tasksToBeScheduled.size()); assertEquals(1, outcome.tasksToBeUpdated.size()); } @Test public void testDecideSuccessfulWorkflow() { WorkflowDef workflowDef = createLinearWorkflow(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setStatus(WorkflowModel.Status.RUNNING); TaskModel task1 = new TaskModel(); task1.setTaskType("junit_task_l1"); task1.setReferenceTaskName("s1"); task1.setSeq(1); task1.setRetried(false); task1.setExecuted(false); task1.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().add(task1); DeciderOutcome deciderOutcome = deciderService.decide(workflow); assertNotNull(deciderOutcome); assertFalse(workflow.getTaskByRefName("s1").isRetried()); assertEquals(1, deciderOutcome.tasksToBeUpdated.size()); assertEquals("s1", deciderOutcome.tasksToBeUpdated.get(0).getReferenceTaskName()); assertEquals(1, deciderOutcome.tasksToBeScheduled.size()); assertEquals("s2", deciderOutcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertFalse(deciderOutcome.isComplete); TaskModel task2 = new TaskModel(); task2.setTaskType("junit_task_l2"); task2.setReferenceTaskName("s2"); task2.setSeq(2); task2.setRetried(false); task2.setExecuted(false); task2.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().add(task2); deciderOutcome = deciderService.decide(workflow); assertNotNull(deciderOutcome); assertTrue(workflow.getTaskByRefName("s2").isExecuted()); assertFalse(workflow.getTaskByRefName("s2").isRetried()); assertEquals(1, deciderOutcome.tasksToBeUpdated.size()); assertEquals("s2", deciderOutcome.tasksToBeUpdated.get(0).getReferenceTaskName()); assertEquals(0, deciderOutcome.tasksToBeScheduled.size()); assertTrue(deciderOutcome.isComplete); } @Test public void testDecideWithLoopTask() { WorkflowDef workflowDef = createLinearWorkflow(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setStatus(WorkflowModel.Status.RUNNING); TaskModel task1 = new TaskModel(); task1.setTaskType("junit_task_l1"); task1.setReferenceTaskName("s1"); task1.setSeq(1); task1.setIteration(1); task1.setRetried(false); task1.setExecuted(false); task1.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().add(task1); DeciderOutcome deciderOutcome = deciderService.decide(workflow); assertNotNull(deciderOutcome); assertFalse(workflow.getTaskByRefName("s1").isRetried()); assertEquals(1, deciderOutcome.tasksToBeUpdated.size()); assertEquals("s1", deciderOutcome.tasksToBeUpdated.get(0).getReferenceTaskName()); assertEquals(1, deciderOutcome.tasksToBeScheduled.size()); assertEquals("s2__1", deciderOutcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertFalse(deciderOutcome.isComplete); } @Test public void testDecideFailedTask() { WorkflowDef workflowDef = createLinearWorkflow(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setStatus(WorkflowModel.Status.RUNNING); TaskModel task = new TaskModel(); task.setTaskType("junit_task_l1"); task.setReferenceTaskName("s1"); task.setSeq(1); task.setRetried(false); task.setExecuted(false); task.setStatus(TaskModel.Status.FAILED); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("s1"); workflowTask.setName("junit_task_l1"); workflowTask.setTaskDefinition(new TaskDef("junit_task_l1")); task.setWorkflowTask(workflowTask); workflow.getTasks().add(task); DeciderOutcome deciderOutcome = deciderService.decide(workflow); assertNotNull(deciderOutcome); assertFalse(workflow.getTaskByRefName("s1").isExecuted()); assertTrue(workflow.getTaskByRefName("s1").isRetried()); assertEquals(1, deciderOutcome.tasksToBeUpdated.size()); assertEquals("s1", deciderOutcome.tasksToBeUpdated.get(0).getReferenceTaskName()); assertEquals(1, deciderOutcome.tasksToBeScheduled.size()); assertEquals("s1", deciderOutcome.tasksToBeScheduled.get(0).getReferenceTaskName()); assertFalse(deciderOutcome.isComplete); } @Test public void testGetTasksToBeScheduled() { WorkflowDef workflowDef = createLinearWorkflow(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setStatus(WorkflowModel.Status.RUNNING); WorkflowTask workflowTask1 = new WorkflowTask(); workflowTask1.setName("s1"); workflowTask1.setTaskReferenceName("s1"); workflowTask1.setType(SIMPLE.name()); workflowTask1.setTaskDefinition(new TaskDef("s1")); List tasksToBeScheduled = deciderService.getTasksToBeScheduled(workflow, workflowTask1, 0, null); assertNotNull(tasksToBeScheduled); assertEquals(1, tasksToBeScheduled.size()); assertEquals("s1", tasksToBeScheduled.get(0).getReferenceTaskName()); WorkflowTask workflowTask2 = new WorkflowTask(); workflowTask2.setName("s2"); workflowTask2.setTaskReferenceName("s2"); workflowTask2.setType(SIMPLE.name()); workflowTask2.setTaskDefinition(new TaskDef("s2")); tasksToBeScheduled = deciderService.getTasksToBeScheduled(workflow, workflowTask2, 0, null); assertNotNull(tasksToBeScheduled); assertEquals(1, tasksToBeScheduled.size()); assertEquals("s2", tasksToBeScheduled.get(0).getReferenceTaskName()); } @Test public void testIsResponseTimedOut() { TaskDef taskDef = new TaskDef(); taskDef.setName("test_rt"); taskDef.setResponseTimeoutSeconds(10); TaskModel task = new TaskModel(); task.setTaskDefName("test_rt"); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setTaskId("aa"); task.setTaskType(TaskType.TASK_TYPE_SIMPLE); task.setUpdateTime(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(11)); assertTrue(deciderService.isResponseTimedOut(taskDef, task)); // verify that sub workflow tasks are not response timed out task.setTaskType(TaskType.TASK_TYPE_SUB_WORKFLOW); assertFalse(deciderService.isResponseTimedOut(taskDef, task)); task.setTaskType("asyncCompleteSystemTask"); assertFalse(deciderService.isResponseTimedOut(taskDef, task)); } @Test public void testFilterNextLoopOverTasks() { WorkflowModel workflow = new WorkflowModel(); TaskModel task1 = new TaskModel(); task1.setReferenceTaskName("task1"); task1.setStatus(TaskModel.Status.COMPLETED); task1.setTaskId("task1"); task1.setIteration(1); TaskModel task2 = new TaskModel(); task2.setReferenceTaskName("task2"); task2.setStatus(TaskModel.Status.SCHEDULED); task2.setTaskId("task2"); TaskModel task3 = new TaskModel(); task3.setReferenceTaskName("task3__1"); task3.setStatus(TaskModel.Status.IN_PROGRESS); task3.setTaskId("task3__1"); TaskModel task4 = new TaskModel(); task4.setReferenceTaskName("task4"); task4.setStatus(TaskModel.Status.SCHEDULED); task4.setTaskId("task4"); TaskModel task5 = new TaskModel(); task5.setReferenceTaskName("task5"); task5.setStatus(TaskModel.Status.COMPLETED); task5.setTaskId("task5"); workflow.getTasks().addAll(Arrays.asList(task1, task2, task3, task4, task5)); List tasks = deciderService.filterNextLoopOverTasks( Arrays.asList(task2, task3, task4), task1, workflow); assertEquals(2, tasks.size()); tasks.forEach( task -> { assertTrue( task.getReferenceTaskName() .endsWith(TaskUtils.getLoopOverTaskRefNameSuffix(1))); assertEquals(1, task.getIteration()); }); } @Test public void testUpdateWorkflowOutput() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(new WorkflowDef()); deciderService.updateWorkflowOutput(workflow, null); assertNotNull(workflow.getOutput()); assertTrue(workflow.getOutput().isEmpty()); TaskModel task = new TaskModel(); Map taskOutput = new HashMap<>(); taskOutput.put("taskKey", "taskValue"); task.setOutputData(taskOutput); workflow.getTasks().add(task); WorkflowDef workflowDef = new WorkflowDef(); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(workflowDef)); deciderService.updateWorkflowOutput(workflow, null); assertNotNull(workflow.getOutput()); assertEquals("taskValue", workflow.getOutput().get("taskKey")); } // when workflow definition has outputParameters defined @SuppressWarnings({"unchecked", "rawtypes"}) @Test public void testUpdateWorkflowOutput_WhenDefinitionHasOutputParameters() { WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setOutputParameters( new HashMap() { { put("workflowKey", "workflowValue"); } }); workflow.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setReferenceTaskName("test_task"); task.setOutputData( new HashMap() { { put("taskKey", "taskValue"); } }); workflow.getTasks().add(task); deciderService.updateWorkflowOutput(workflow, null); assertNotNull(workflow.getOutput()); assertEquals("workflowValue", workflow.getOutput().get("workflowKey")); } @Test public void testUpdateWorkflowOutput_WhenWorkflowHasTerminateTask() { WorkflowModel workflow = new WorkflowModel(); TaskModel task = new TaskModel(); task.setTaskType(TASK_TYPE_TERMINATE); task.setStatus(TaskModel.Status.COMPLETED); task.setOutputData( new HashMap() { { put("taskKey", "taskValue"); } }); workflow.getTasks().add(task); deciderService.updateWorkflowOutput(workflow, null); assertNotNull(workflow.getOutput()); assertEquals("taskValue", workflow.getOutput().get("taskKey")); verify(externalPayloadStorageUtils, never()).downloadPayload(anyString()); // when terminate task has output in external payload storage String externalOutputPayloadStoragePath = "/task/output/terminate.json"; workflow.getTasks().get(0).setOutputData(null); workflow.getTasks() .get(0) .setExternalOutputPayloadStoragePath(externalOutputPayloadStoragePath); when(externalPayloadStorageUtils.downloadPayload(externalOutputPayloadStoragePath)) .thenReturn( new HashMap() { { put("taskKey", "taskValue"); } }); deciderService.updateWorkflowOutput(workflow, null); assertNotNull(workflow.getOutput()); assertEquals("taskValue", workflow.getOutput().get("taskKey")); verify(externalPayloadStorageUtils, times(1)).downloadPayload(anyString()); } @Test public void testCheckWorkflowTimeout() { Counter counter = registry.counter( "workflow_failure", "class", "WorkflowMonitor", "workflowName", "test", "status", "TIMED_OUT", "ownerApp", "junit"); long counterCount = counter.count(); assertEquals(0, counter.count()); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test"); WorkflowModel workflow = new WorkflowModel(); workflow.setOwnerApp("junit"); workflow.setCreateTime(System.currentTimeMillis() - 10_000); workflow.setWorkflowId("workflow_id"); // no-op workflow.setWorkflowDefinition(null); deciderService.checkWorkflowTimeout(workflow); // no-op workflow.setWorkflowDefinition(workflowDef); deciderService.checkWorkflowTimeout(workflow); // alert workflowDef.setTimeoutPolicy(WorkflowDef.TimeoutPolicy.ALERT_ONLY); workflowDef.setTimeoutSeconds(2); workflow.setWorkflowDefinition(workflowDef); deciderService.checkWorkflowTimeout(workflow); assertEquals(++counterCount, counter.count()); // time out workflowDef.setTimeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF); workflow.setWorkflowDefinition(workflowDef); try { deciderService.checkWorkflowTimeout(workflow); } catch (TerminateWorkflowException twe) { assertTrue(twe.getMessage().contains("Workflow timed out")); } // for a retried workflow workflow.setLastRetriedTime(System.currentTimeMillis() - 5_000); try { deciderService.checkWorkflowTimeout(workflow); } catch (TerminateWorkflowException twe) { assertTrue(twe.getMessage().contains("Workflow timed out")); } } @Test public void testCheckForWorkflowCompletion() { WorkflowDef conditionalWorkflowDef = createConditionalWF(); WorkflowTask terminateWT = new WorkflowTask(); terminateWT.setType(TaskType.TERMINATE.name()); terminateWT.setTaskReferenceName("terminate"); terminateWT.setName("terminate"); terminateWT.getInputParameters().put("terminationStatus", "COMPLETED"); conditionalWorkflowDef.getTasks().add(terminateWT); // when workflow has no tasks WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(conditionalWorkflowDef); // then workflow completion check returns false assertFalse(deciderService.checkForWorkflowCompletion(workflow)); // when only part of the tasks are completed TaskModel decTask = new TaskModel(); decTask.setTaskType(DECISION.name()); decTask.setReferenceTaskName("conditional2"); decTask.setStatus(TaskModel.Status.COMPLETED); TaskModel task1 = new TaskModel(); decTask.setTaskType(SIMPLE.name()); task1.setReferenceTaskName("t1"); task1.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(decTask, task1)); // then workflow completion check returns false assertFalse(deciderService.checkForWorkflowCompletion(workflow)); // when the terminate task is COMPLETED TaskModel task2 = new TaskModel(); decTask.setTaskType(SIMPLE.name()); task2.setReferenceTaskName("t2"); task2.setStatus(TaskModel.Status.SCHEDULED); TaskModel terminateTask = new TaskModel(); decTask.setTaskType(TaskType.TERMINATE.name()); terminateTask.setReferenceTaskName("terminate"); terminateTask.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task2, terminateTask)); // then the workflow completion check returns true assertTrue(deciderService.checkForWorkflowCompletion(workflow)); } private WorkflowDef createConditionalWF() { WorkflowTask workflowTask1 = new WorkflowTask(); workflowTask1.setName("junit_task_1"); Map inputParams1 = new HashMap<>(); inputParams1.put("p1", "workflow.input.param1"); inputParams1.put("p2", "workflow.input.param2"); workflowTask1.setInputParameters(inputParams1); workflowTask1.setTaskReferenceName("t1"); workflowTask1.setTaskDefinition(new TaskDef("junit_task_1")); WorkflowTask workflowTask2 = new WorkflowTask(); workflowTask2.setName("junit_task_2"); Map inputParams2 = new HashMap<>(); inputParams2.put("tp1", "workflow.input.param1"); workflowTask2.setInputParameters(inputParams2); workflowTask2.setTaskReferenceName("t2"); workflowTask2.setTaskDefinition(new TaskDef("junit_task_2")); WorkflowTask workflowTask3 = new WorkflowTask(); workflowTask3.setName("junit_task_3"); Map inputParams3 = new HashMap<>(); inputParams2.put("tp3", "workflow.input.param2"); workflowTask3.setInputParameters(inputParams3); workflowTask3.setTaskReferenceName("t3"); workflowTask3.setTaskDefinition(new TaskDef("junit_task_3")); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("Conditional Workflow"); workflowDef.setDescription("Conditional Workflow"); workflowDef.setInputParameters(Arrays.asList("param1", "param2")); WorkflowTask decisionTask2 = new WorkflowTask(); decisionTask2.setType(DECISION.name()); decisionTask2.setCaseValueParam("case"); decisionTask2.setName("conditional2"); decisionTask2.setTaskReferenceName("conditional2"); Map> dc = new HashMap<>(); dc.put("one", Arrays.asList(workflowTask1, workflowTask3)); dc.put("two", Collections.singletonList(workflowTask2)); decisionTask2.setDecisionCases(dc); decisionTask2.getInputParameters().put("case", "workflow.input.param2"); WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(DECISION.name()); decisionTask.setCaseValueParam("case"); decisionTask.setName("conditional"); decisionTask.setTaskReferenceName("conditional"); Map> decisionCases = new HashMap<>(); decisionCases.put("nested", Collections.singletonList(decisionTask2)); decisionCases.put("three", Collections.singletonList(workflowTask3)); decisionTask.setDecisionCases(decisionCases); decisionTask.getInputParameters().put("case", "workflow.input.param1"); decisionTask.getDefaultCase().add(workflowTask2); workflowDef.getTasks().add(decisionTask); WorkflowTask notifyTask = new WorkflowTask(); notifyTask.setName("junit_task_4"); notifyTask.setTaskReferenceName("junit_task_4"); notifyTask.setTaskDefinition(new TaskDef("junit_task_4")); WorkflowTask finalDecisionTask = new WorkflowTask(); finalDecisionTask.setName("finalcondition"); finalDecisionTask.setTaskReferenceName("tf"); finalDecisionTask.setType(DECISION.name()); finalDecisionTask.setCaseValueParam("finalCase"); Map fi = new HashMap<>(); fi.put("finalCase", "workflow.input.finalCase"); finalDecisionTask.setInputParameters(fi); finalDecisionTask.getDecisionCases().put("notify", Collections.singletonList(notifyTask)); workflowDef.getTasks().add(finalDecisionTask); return workflowDef; } private WorkflowDef createLinearWorkflow() { Map inputParams = new HashMap<>(); inputParams.put("p1", "workflow.input.param1"); inputParams.put("p2", "workflow.input.param2"); WorkflowTask workflowTask1 = new WorkflowTask(); workflowTask1.setName("junit_task_l1"); workflowTask1.setInputParameters(inputParams); workflowTask1.setTaskReferenceName("s1"); workflowTask1.setTaskDefinition(new TaskDef("junit_task_l1")); WorkflowTask workflowTask2 = new WorkflowTask(); workflowTask2.setName("junit_task_l2"); workflowTask2.setInputParameters(inputParams); workflowTask2.setTaskReferenceName("s2"); workflowTask2.setTaskDefinition(new TaskDef("junit_task_l2")); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); workflowDef.setInputParameters(Arrays.asList("param1", "param2")); workflowDef.setName("Linear Workflow"); workflowDef.getTasks().addAll(Arrays.asList(workflowTask1, workflowTask2)); return workflowDef; } private WorkflowModel createDefaultWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("TestDeciderService"); workflowDef.setVersion(1); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.getInput().put("requestId", "request id 001"); workflow.getInput().put("hasAwards", true); workflow.getInput().put("channelMapping", 5); Map name = new HashMap<>(); name.put("name", "The Who"); name.put("year", 1970); Map name2 = new HashMap<>(); name2.put("name", "The Doors"); name2.put("year", 1975); List names = new LinkedList<>(); names.add(name); names.add(name2); workflow.addOutput("name", name); workflow.addOutput("names", names); workflow.addOutput("awards", 200); TaskModel task = new TaskModel(); task.setReferenceTaskName("task2"); task.addOutput("location", "http://location"); task.setStatus(TaskModel.Status.COMPLETED); TaskModel task2 = new TaskModel(); task2.setReferenceTaskName("task3"); task2.addOutput("refId", "abcddef_1234_7890_aaffcc"); task2.setStatus(TaskModel.Status.SCHEDULED); workflow.getTasks().add(task); workflow.getTasks().add(task2); return workflow; } private WorkflowDef createNestedWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("Nested Workflow"); workflowDef.setDescription(workflowDef.getName()); workflowDef.setVersion(1); workflowDef.setInputParameters(Arrays.asList("param1", "param2")); Map inputParams = new HashMap<>(); inputParams.put("p1", "workflow.input.param1"); inputParams.put("p2", "workflow.input.param2"); List tasks = new ArrayList<>(10); for (int i = 0; i < 10; i++) { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("junit_task_" + i); workflowTask.setInputParameters(inputParams); workflowTask.setTaskReferenceName("t" + i); workflowTask.setTaskDefinition(new TaskDef("junit_task_" + i)); tasks.add(workflowTask); } WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(DECISION.name()); decisionTask.setName("Decision"); decisionTask.setTaskReferenceName("d1"); decisionTask.setDefaultCase(Collections.singletonList(tasks.get(8))); decisionTask.setCaseValueParam("case"); Map> decisionCases = new HashMap<>(); decisionCases.put("a", Arrays.asList(tasks.get(6), tasks.get(9))); decisionCases.put("b", Collections.singletonList(tasks.get(7))); decisionTask.setDecisionCases(decisionCases); WorkflowDef subWorkflowDef = createLinearWorkflow(); WorkflowTask subWorkflow = new WorkflowTask(); subWorkflow.setType(SUB_WORKFLOW.name()); subWorkflow.setName("sw1"); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName(subWorkflowDef.getName()); subWorkflow.setSubWorkflowParam(subWorkflowParams); subWorkflow.setTaskReferenceName("sw1"); WorkflowTask forkTask2 = new WorkflowTask(); forkTask2.setType(FORK_JOIN.name()); forkTask2.setName("second fork"); forkTask2.setTaskReferenceName("fork2"); forkTask2.getForkTasks().add(Arrays.asList(tasks.get(2), tasks.get(4))); forkTask2.getForkTasks().add(Arrays.asList(tasks.get(3), decisionTask)); WorkflowTask joinTask2 = new WorkflowTask(); joinTask2.setName("join2"); joinTask2.setType(JOIN.name()); joinTask2.setTaskReferenceName("join2"); joinTask2.setJoinOn(Arrays.asList("t4", "d1")); WorkflowTask forkTask1 = new WorkflowTask(); forkTask1.setType(FORK_JOIN.name()); forkTask1.setName("fork1"); forkTask1.setTaskReferenceName("fork1"); forkTask1.getForkTasks().add(Collections.singletonList(tasks.get(1))); forkTask1.getForkTasks().add(Arrays.asList(forkTask2, joinTask2)); forkTask1.getForkTasks().add(Collections.singletonList(subWorkflow)); WorkflowTask joinTask1 = new WorkflowTask(); joinTask1.setName("join1"); joinTask1.setType(JOIN.name()); joinTask1.setTaskReferenceName("join1"); joinTask1.setJoinOn(Arrays.asList("t1", "fork2")); workflowDef.getTasks().add(forkTask1); workflowDef.getTasks().add(joinTask1); workflowDef.getTasks().add(tasks.get(5)); return workflowDef; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowDef.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class TestWorkflowDef { @Test public void testContainsType() { WorkflowDef def = new WorkflowDef(); def.setName("test_workflow"); def.setVersion(1); def.setSchemaVersion(2); def.getTasks().add(createWorkflowTask("simple_task_1")); def.getTasks().add(createWorkflowTask("simple_task_2")); WorkflowTask task3 = createWorkflowTask("decision_task_1"); def.getTasks().add(task3); task3.setType(TaskType.DECISION.name()); task3.getDecisionCases() .put( "Case1", Arrays.asList( createWorkflowTask("case_1_task_1"), createWorkflowTask("case_1_task_2"))); task3.getDecisionCases() .put( "Case2", Arrays.asList( createWorkflowTask("case_2_task_1"), createWorkflowTask("case_2_task_2"))); task3.getDecisionCases() .put( "Case3", Collections.singletonList( deciderTask( "decision_task_2", toMap("Case31", "case31_task_1", "case_31_task_2"), Collections.singletonList("case3_def_task")))); def.getTasks().add(createWorkflowTask("simple_task_3")); assertTrue(def.containsType(TaskType.SIMPLE.name())); assertTrue(def.containsType(TaskType.DECISION.name())); assertFalse(def.containsType(TaskType.DO_WHILE.name())); } @Test public void testGetNextTask_Decision() { WorkflowDef def = new WorkflowDef(); def.setName("test_workflow"); def.setVersion(1); def.setSchemaVersion(2); def.getTasks().add(createWorkflowTask("simple_task_1")); def.getTasks().add(createWorkflowTask("simple_task_2")); WorkflowTask task3 = createWorkflowTask("decision_task_1"); def.getTasks().add(task3); task3.setType(TaskType.DECISION.name()); task3.getDecisionCases() .put( "Case1", Arrays.asList( createWorkflowTask("case_1_task_1"), createWorkflowTask("case_1_task_2"))); task3.getDecisionCases() .put( "Case2", Arrays.asList( createWorkflowTask("case_2_task_1"), createWorkflowTask("case_2_task_2"))); task3.getDecisionCases() .put( "Case3", Collections.singletonList( deciderTask( "decision_task_2", toMap("Case31", "case31_task_1", "case_31_task_2"), Collections.singletonList("case3_def_task")))); def.getTasks().add(createWorkflowTask("simple_task_3")); // Assertions WorkflowTask next = def.getNextTask("simple_task_1"); assertNotNull(next); assertEquals("simple_task_2", next.getTaskReferenceName()); next = def.getNextTask("simple_task_2"); assertNotNull(next); assertEquals(task3.getTaskReferenceName(), next.getTaskReferenceName()); next = def.getNextTask("decision_task_1"); assertNotNull(next); assertEquals("simple_task_3", next.getTaskReferenceName()); next = def.getNextTask("case_1_task_1"); assertNotNull(next); assertEquals("case_1_task_2", next.getTaskReferenceName()); next = def.getNextTask("case_1_task_2"); assertNotNull(next); assertEquals("simple_task_3", next.getTaskReferenceName()); next = def.getNextTask("case3_def_task"); assertNotNull(next); assertEquals("simple_task_3", next.getTaskReferenceName()); next = def.getNextTask("case31_task_1"); assertNotNull(next); assertEquals("case_31_task_2", next.getTaskReferenceName()); } @Test public void testGetNextTask_Conditional() { String COND_TASK_WF = "COND_TASK_WF"; List workflowTasks = new ArrayList<>(10); for (int i = 0; i < 10; i++) { workflowTasks.add(createWorkflowTask("junit_task_" + i)); } WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName(COND_TASK_WF); workflowDef.setDescription(COND_TASK_WF); WorkflowTask subCaseTask = new WorkflowTask(); subCaseTask.setType(TaskType.DECISION.name()); subCaseTask.setCaseValueParam("case2"); subCaseTask.setName("case2"); subCaseTask.setTaskReferenceName("case2"); Map> dcx = new HashMap<>(); dcx.put("sc1", workflowTasks.subList(4, 5)); dcx.put("sc2", workflowTasks.subList(5, 7)); subCaseTask.setDecisionCases(dcx); WorkflowTask caseTask = new WorkflowTask(); caseTask.setType(TaskType.DECISION.name()); caseTask.setCaseValueParam("case"); caseTask.setName("case"); caseTask.setTaskReferenceName("case"); Map> dc = new HashMap<>(); dc.put("c1", Arrays.asList(workflowTasks.get(0), subCaseTask, workflowTasks.get(1))); dc.put("c2", Collections.singletonList(workflowTasks.get(3))); caseTask.setDecisionCases(dc); workflowDef.getTasks().add(caseTask); workflowDef.getTasks().addAll(workflowTasks.subList(8, 9)); WorkflowTask nextTask = workflowDef.getNextTask("case"); assertEquals("junit_task_8", nextTask.getTaskReferenceName()); nextTask = workflowDef.getNextTask("junit_task_8"); assertNull(nextTask); nextTask = workflowDef.getNextTask("junit_task_0"); assertNotNull(nextTask); assertEquals("case2", nextTask.getTaskReferenceName()); nextTask = workflowDef.getNextTask("case2"); assertNotNull(nextTask); assertEquals("junit_task_1", nextTask.getTaskReferenceName()); } private WorkflowTask createWorkflowTask(String name) { WorkflowTask task = new WorkflowTask(); task.setName(name); task.setTaskReferenceName(name); return task; } private WorkflowTask deciderTask( String name, Map> decisions, List defaultTasks) { WorkflowTask task = createWorkflowTask(name); task.setType(TaskType.DECISION.name()); decisions.forEach( (key, value) -> { List tasks = new LinkedList<>(); value.forEach(taskName -> tasks.add(createWorkflowTask(taskName))); task.getDecisionCases().put(key, tasks); }); List tasks = new LinkedList<>(); defaultTasks.forEach(defaultTask -> tasks.add(createWorkflowTask(defaultTask))); task.setDefaultCase(tasks); return task; } private Map> toMap(String key, String... values) { Map> map = new HashMap<>(); List vals = Arrays.asList(values); map.put(key, vals); return map; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import java.time.Duration; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.event.WorkflowCreationEvent; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.execution.mapper.*; import com.netflix.conductor.core.execution.tasks.*; import com.netflix.conductor.core.listener.TaskStatusListener; import com.netflix.conductor.core.listener.WorkflowStatusListener; import com.netflix.conductor.core.metadata.MetadataMapperService; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.*; import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.maxBy; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, TestWorkflowExecutor.TestConfiguration.class }) @RunWith(SpringRunner.class) public class TestWorkflowExecutor { private WorkflowExecutor workflowExecutor; private ExecutionDAOFacade executionDAOFacade; private MetadataDAO metadataDAO; private QueueDAO queueDAO; private WorkflowStatusListener workflowStatusListener; private TaskStatusListener taskStatusListener; private ExecutionLockService executionLockService; private ExternalPayloadStorageUtils externalPayloadStorageUtils; @Configuration @ComponentScan(basePackageClasses = {Evaluator.class}) // load all Evaluator beans. public static class TestConfiguration { @Bean(TASK_TYPE_SUB_WORKFLOW) public SubWorkflow subWorkflow(ObjectMapper objectMapper) { return new SubWorkflow(objectMapper, mock(StartWorkflowOperation.class)); } @Bean(TASK_TYPE_LAMBDA) public Lambda lambda() { return new Lambda(); } @Bean(TASK_TYPE_WAIT) public Wait waitBean() { return new Wait(); } @Bean("HTTP") public WorkflowSystemTask http() { return new WorkflowSystemTaskStub("HTTP") { @Override public boolean isAsync() { return true; } }; } @Bean("HTTP2") public WorkflowSystemTask http2() { return new WorkflowSystemTaskStub("HTTP2"); } @Bean(TASK_TYPE_JSON_JQ_TRANSFORM) public WorkflowSystemTask jsonBean() { return new WorkflowSystemTaskStub("JSON_JQ_TRANSFORM") { @Override public boolean isAsync() { return false; } @Override public void start( WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { task.setStatus(TaskModel.Status.COMPLETED); } }; } @Bean public SystemTaskRegistry systemTaskRegistry(Set tasks) { return new SystemTaskRegistry(tasks); } } @Autowired private ObjectMapper objectMapper; @Autowired private SystemTaskRegistry systemTaskRegistry; @Autowired private DefaultListableBeanFactory beanFactory; @Autowired private Map evaluators; private ApplicationEventPublisher eventPublisher; @Before public void init() { executionDAOFacade = mock(ExecutionDAOFacade.class); metadataDAO = mock(MetadataDAO.class); queueDAO = mock(QueueDAO.class); workflowStatusListener = mock(WorkflowStatusListener.class); taskStatusListener = mock(TaskStatusListener.class); externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); executionLockService = mock(ExecutionLockService.class); eventPublisher = mock(ApplicationEventPublisher.class); ParametersUtils parametersUtils = new ParametersUtils(objectMapper); IDGenerator idGenerator = new IDGenerator(); Map taskMappers = new HashMap<>(); taskMappers.put(DECISION.name(), new DecisionTaskMapper()); taskMappers.put(SWITCH.name(), new SwitchTaskMapper(evaluators)); taskMappers.put(DYNAMIC.name(), new DynamicTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(FORK_JOIN.name(), new ForkJoinTaskMapper()); taskMappers.put(JOIN.name(), new JoinTaskMapper()); taskMappers.put( FORK_JOIN_DYNAMIC.name(), new ForkJoinDynamicTaskMapper( idGenerator, parametersUtils, objectMapper, metadataDAO)); taskMappers.put( USER_DEFINED.name(), new UserDefinedTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(SIMPLE.name(), new SimpleTaskMapper(parametersUtils)); taskMappers.put( SUB_WORKFLOW.name(), new SubWorkflowTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(EVENT.name(), new EventTaskMapper(parametersUtils)); taskMappers.put(WAIT.name(), new WaitTaskMapper(parametersUtils)); taskMappers.put(HTTP.name(), new HTTPTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(LAMBDA.name(), new LambdaTaskMapper(parametersUtils, metadataDAO)); taskMappers.put(INLINE.name(), new InlineTaskMapper(parametersUtils, metadataDAO)); DeciderService deciderService = new DeciderService( idGenerator, parametersUtils, metadataDAO, externalPayloadStorageUtils, systemTaskRegistry, taskMappers, Duration.ofMinutes(60)); MetadataMapperService metadataMapperService = new MetadataMapperService(metadataDAO); ConductorProperties properties = mock(ConductorProperties.class); when(properties.getActiveWorkerLastPollTimeout()).thenReturn(Duration.ofSeconds(100)); when(properties.getTaskExecutionPostponeDuration()).thenReturn(Duration.ofSeconds(60)); when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(30)); workflowExecutor = new WorkflowExecutor( deciderService, metadataDAO, queueDAO, metadataMapperService, workflowStatusListener, taskStatusListener, executionDAOFacade, properties, executionLockService, systemTaskRegistry, parametersUtils, idGenerator, eventPublisher); } @Test public void testScheduleTask() { IDGenerator idGenerator = new IDGenerator(); WorkflowSystemTaskStub httpTask = beanFactory.getBean("HTTP", WorkflowSystemTaskStub.class); WorkflowSystemTaskStub http2Task = beanFactory.getBean("HTTP2", WorkflowSystemTaskStub.class); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("1"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("1"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); List tasks = new LinkedList<>(); WorkflowTask taskToSchedule = new WorkflowTask(); taskToSchedule.setWorkflowTaskType(TaskType.USER_DEFINED); taskToSchedule.setType("HTTP"); WorkflowTask taskToSchedule2 = new WorkflowTask(); taskToSchedule2.setWorkflowTaskType(TaskType.USER_DEFINED); taskToSchedule2.setType("HTTP2"); WorkflowTask wait = new WorkflowTask(); wait.setWorkflowTaskType(TaskType.WAIT); wait.setType("WAIT"); wait.setTaskReferenceName("wait"); TaskModel task1 = new TaskModel(); task1.setTaskType(taskToSchedule.getType()); task1.setTaskDefName(taskToSchedule.getName()); task1.setReferenceTaskName(taskToSchedule.getTaskReferenceName()); task1.setWorkflowInstanceId(workflow.getWorkflowId()); task1.setCorrelationId(workflow.getCorrelationId()); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setInputData(new HashMap<>()); task1.setStatus(TaskModel.Status.SCHEDULED); task1.setRetryCount(0); task1.setCallbackAfterSeconds(taskToSchedule.getStartDelay()); task1.setWorkflowTask(taskToSchedule); TaskModel task2 = new TaskModel(); task2.setTaskType(TASK_TYPE_WAIT); task2.setTaskDefName(taskToSchedule.getName()); task2.setReferenceTaskName(taskToSchedule.getTaskReferenceName()); task2.setWorkflowInstanceId(workflow.getWorkflowId()); task2.setCorrelationId(workflow.getCorrelationId()); task2.setScheduledTime(System.currentTimeMillis()); task2.setInputData(new HashMap<>()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.IN_PROGRESS); task2.setWorkflowTask(taskToSchedule); TaskModel task3 = new TaskModel(); task3.setTaskType(taskToSchedule2.getType()); task3.setTaskDefName(taskToSchedule.getName()); task3.setReferenceTaskName(taskToSchedule.getTaskReferenceName()); task3.setWorkflowInstanceId(workflow.getWorkflowId()); task3.setCorrelationId(workflow.getCorrelationId()); task3.setScheduledTime(System.currentTimeMillis()); task3.setTaskId(idGenerator.generate()); task3.setInputData(new HashMap<>()); task3.setStatus(TaskModel.Status.SCHEDULED); task3.setRetryCount(0); task3.setCallbackAfterSeconds(taskToSchedule.getStartDelay()); task3.setWorkflowTask(taskToSchedule); tasks.add(task1); tasks.add(task2); tasks.add(task3); when(executionDAOFacade.createTasks(tasks)).thenReturn(tasks); AtomicInteger startedTaskCount = new AtomicInteger(0); doAnswer( invocation -> { startedTaskCount.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTask(any()); AtomicInteger queuedTaskCount = new AtomicInteger(0); final Answer answer = invocation -> { String queueName = invocation.getArgument(0, String.class); queuedTaskCount.incrementAndGet(); return null; }; doAnswer(answer).when(queueDAO).push(any(), any(), anyLong()); doAnswer(answer).when(queueDAO).push(any(), any(), anyInt(), anyLong()); boolean stateChanged = workflowExecutor.scheduleTask(workflow, tasks); // Wait task is no async to it will be queued. assertEquals(1, startedTaskCount.get()); assertEquals(2, queuedTaskCount.get()); assertTrue(stateChanged); assertFalse(httpTask.isStarted()); assertTrue(http2Task.isStarted()); } @Test(expected = TerminateWorkflowException.class) public void testScheduleTaskFailure() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("wid_01"); List tasks = new LinkedList<>(); TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.TASK_TYPE_SIMPLE); task1.setTaskDefName("task_1"); task1.setReferenceTaskName("task_1"); task1.setWorkflowInstanceId(workflow.getWorkflowId()); task1.setTaskId("tid_01"); task1.setStatus(TaskModel.Status.SCHEDULED); task1.setRetryCount(0); tasks.add(task1); when(executionDAOFacade.createTasks(tasks)).thenThrow(new RuntimeException()); workflowExecutor.scheduleTask(workflow, tasks); } /** Simulate Queue push failures and assert that scheduleTask doesn't throw an exception. */ @Test public void testQueueFailuresDuringScheduleTask() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("wid_01"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("wid"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); List tasks = new LinkedList<>(); TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.TASK_TYPE_SIMPLE); task1.setTaskDefName("task_1"); task1.setReferenceTaskName("task_1"); task1.setWorkflowInstanceId(workflow.getWorkflowId()); task1.setTaskId("tid_01"); task1.setStatus(TaskModel.Status.SCHEDULED); task1.setRetryCount(0); tasks.add(task1); when(executionDAOFacade.createTasks(tasks)).thenReturn(tasks); doThrow(new RuntimeException()) .when(queueDAO) .push(anyString(), anyString(), anyInt(), anyLong()); assertFalse(workflowExecutor.scheduleTask(workflow, tasks)); } @Test @SuppressWarnings("unchecked") public void testCompleteWorkflow() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setWorkflowId("1"); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOwnerApp("junit_test"); workflow.setCreateTime(10L); workflow.setEndTime(100L); workflow.setOutput(Collections.EMPTY_MAP); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); AtomicInteger updateTasksCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTasksCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTasks(any()); AtomicInteger removeQueueEntryCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { removeQueueEntryCalledCounter.incrementAndGet(); return null; }) .when(queueDAO) .remove(anyString(), anyString()); workflowExecutor.completeWorkflow(workflow); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getStatus()); assertEquals(1, updateWorkflowCalledCounter.get()); assertEquals(0, updateTasksCalledCounter.get()); assertEquals(0, removeQueueEntryCalledCounter.get()); verify(workflowStatusListener, times(1)) .onWorkflowCompletedIfEnabled(any(WorkflowModel.class)); verify(workflowStatusListener, times(0)) .onWorkflowFinalizedIfEnabled(any(WorkflowModel.class)); def.setWorkflowStatusListenerEnabled(true); workflow.setStatus(WorkflowModel.Status.RUNNING); workflowExecutor.completeWorkflow(workflow); verify(workflowStatusListener, times(2)) .onWorkflowCompletedIfEnabled(any(WorkflowModel.class)); verify(workflowStatusListener, times(0)) .onWorkflowFinalizedIfEnabled(any(WorkflowModel.class)); } @Test @SuppressWarnings("unchecked") public void testTerminateWorkflow() { WorkflowDef def = new WorkflowDef(); def.setName("test"); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setWorkflowId("1"); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOwnerApp("junit_test"); workflow.setCreateTime(10L); workflow.setEndTime(100L); workflow.setOutput(Collections.EMPTY_MAP); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); AtomicInteger updateTasksCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTasksCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTasks(any()); AtomicInteger removeQueueEntryCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { removeQueueEntryCalledCounter.incrementAndGet(); return null; }) .when(queueDAO) .remove(anyString(), anyString()); workflowExecutor.terminateWorkflow("workflowId", "reason"); assertEquals(WorkflowModel.Status.TERMINATED, workflow.getStatus()); assertEquals(1, updateWorkflowCalledCounter.get()); assertEquals(1, removeQueueEntryCalledCounter.get()); verify(workflowStatusListener, times(1)) .onWorkflowTerminatedIfEnabled(any(WorkflowModel.class)); verify(workflowStatusListener, times(1)) .onWorkflowFinalizedIfEnabled(any(WorkflowModel.class)); def.setWorkflowStatusListenerEnabled(true); workflow.setStatus(WorkflowModel.Status.RUNNING); workflowExecutor.completeWorkflow(workflow); verify(workflowStatusListener, times(1)) .onWorkflowCompletedIfEnabled(any(WorkflowModel.class)); verify(workflowStatusListener, times(1)) .onWorkflowFinalizedIfEnabled(any(WorkflowModel.class)); } @Test public void testUploadOutputFailuresDuringTerminateWorkflow() { WorkflowDef def = new WorkflowDef(); def.setName("test"); def.setWorkflowStatusListenerEnabled(true); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setWorkflowId("1"); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOwnerApp("junit_test"); workflow.setCreateTime(10L); workflow.setEndTime(100L); workflow.setOutput(Collections.EMPTY_MAP); List tasks = new LinkedList<>(); TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(UUID.randomUUID().toString()); task.setReferenceTaskName("t1"); task.setWorkflowInstanceId(workflow.getWorkflowId()); task.setTaskDefName("task1"); task.setStatus(TaskModel.Status.IN_PROGRESS); tasks.add(task); workflow.setTasks(tasks); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); doThrow(new RuntimeException("any exception")) .when(externalPayloadStorageUtils) .verifyAndUpload(workflow, ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT); workflowExecutor.terminateWorkflow(workflow.getWorkflowId(), "reason"); assertEquals(WorkflowModel.Status.TERMINATED, workflow.getStatus()); assertEquals(1, updateWorkflowCalledCounter.get()); verify(workflowStatusListener, times(1)) .onWorkflowTerminatedIfEnabled(any(WorkflowModel.class)); } @Test @SuppressWarnings("unchecked") public void testQueueExceptionsIgnoredDuringTerminateWorkflow() { WorkflowDef def = new WorkflowDef(); def.setName("test"); def.setWorkflowStatusListenerEnabled(true); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setWorkflowId("1"); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOwnerApp("junit_test"); workflow.setCreateTime(10L); workflow.setEndTime(100L); workflow.setOutput(Collections.EMPTY_MAP); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); AtomicInteger updateTasksCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTasksCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTasks(any()); doThrow(new RuntimeException()).when(queueDAO).remove(anyString(), anyString()); workflowExecutor.terminateWorkflow("workflowId", "reason"); assertEquals(WorkflowModel.Status.TERMINATED, workflow.getStatus()); assertEquals(1, updateWorkflowCalledCounter.get()); verify(workflowStatusListener, times(1)) .onWorkflowTerminatedIfEnabled(any(WorkflowModel.class)); } @Test public void testRestartWorkflow() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("test_task"); workflowTask.setTaskReferenceName("task_ref"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testDef"); workflowDef.setVersion(1); workflowDef.setRestartable(true); workflowDef.getTasks().add(workflowTask); TaskModel task_1 = new TaskModel(); task_1.setTaskId(UUID.randomUUID().toString()); task_1.setSeq(1); task_1.setStatus(TaskModel.Status.FAILED); task_1.setTaskDefName(workflowTask.getName()); task_1.setReferenceTaskName(workflowTask.getTaskReferenceName()); TaskModel task_2 = new TaskModel(); task_2.setTaskId(UUID.randomUUID().toString()); task_2.setSeq(2); task_2.setStatus(TaskModel.Status.FAILED); task_2.setTaskDefName(workflowTask.getName()); task_2.setReferenceTaskName(workflowTask.getTaskReferenceName()); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setWorkflowId("test-workflow-id"); workflow.getTasks().addAll(Arrays.asList(task_1, task_2)); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setEndTime(500); workflow.setLastRetriedTime(100); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); doNothing().when(executionDAOFacade).removeTask(any()); when(metadataDAO.getWorkflowDef(workflow.getWorkflowName(), workflow.getWorkflowVersion())) .thenReturn(Optional.of(workflowDef)); when(metadataDAO.getTaskDef(workflowTask.getName())).thenReturn(new TaskDef()); when(executionDAOFacade.updateWorkflow(any())).thenReturn(""); workflowExecutor.restart(workflow.getWorkflowId(), false); assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertEquals(0, workflow.getEndTime()); assertEquals(0, workflow.getLastRetriedTime()); verify(metadataDAO, never()).getLatestWorkflowDef(any()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(WorkflowModel.class); verify(executionDAOFacade, times(1)).createWorkflow(argumentCaptor.capture()); assertEquals( workflow.getWorkflowId(), argumentCaptor.getAllValues().get(0).getWorkflowId()); assertEquals( workflow.getWorkflowDefinition(), argumentCaptor.getAllValues().get(0).getWorkflowDefinition()); // add a new version of the workflow definition and restart with latest workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.setEndTime(500); workflow.setLastRetriedTime(100); workflowDef = new WorkflowDef(); workflowDef.setName("testDef"); workflowDef.setVersion(2); workflowDef.setRestartable(true); workflowDef.getTasks().addAll(Collections.singletonList(workflowTask)); when(metadataDAO.getLatestWorkflowDef(workflow.getWorkflowName())) .thenReturn(Optional.of(workflowDef)); workflowExecutor.restart(workflow.getWorkflowId(), true); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertEquals(0, workflow.getEndTime()); assertEquals(0, workflow.getLastRetriedTime()); verify(metadataDAO, times(1)).getLatestWorkflowDef(anyString()); argumentCaptor = ArgumentCaptor.forClass(WorkflowModel.class); verify(executionDAOFacade, times(2)).createWorkflow(argumentCaptor.capture()); assertEquals( workflow.getWorkflowId(), argumentCaptor.getAllValues().get(1).getWorkflowId()); assertEquals(workflowDef, argumentCaptor.getAllValues().get(1).getWorkflowDefinition()); } @Test(expected = NotFoundException.class) public void testRetryNonTerminalWorkflow() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryNonTerminalWorkflow"); workflow.setStatus(WorkflowModel.Status.RUNNING); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); workflowExecutor.retry(workflow.getWorkflowId(), false); } @Test(expected = ConflictException.class) public void testRetryWorkflowNoTasks() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("ApplicationException"); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setTasks(Collections.emptyList()); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); workflowExecutor.retry(workflow.getWorkflowId(), false); } @Test(expected = ConflictException.class) public void testRetryWorkflowNoFailedTasks() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); // add 2 failed task in 2 forks and 1 cancelled in the 3rd fork TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(1); task_1_1.setRetryCount(0); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setTaskDefName("task1"); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_1_2 = new TaskModel(); task_1_2.setTaskId(UUID.randomUUID().toString()); task_1_2.setSeq(2); task_1_2.setRetryCount(1); task_1_2.setTaskType(TaskType.SIMPLE.toString()); task_1_2.setStatus(TaskModel.Status.COMPLETED); task_1_2.setTaskDefName("task1"); task_1_2.setReferenceTaskName("task1_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_1_2)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); } @Test public void testRetryWorkflow() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); AtomicInteger updateTasksCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTasksCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTasks(any()); AtomicInteger updateTaskCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTaskCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTask(any()); // add 2 failed task in 2 forks and 1 cancelled in the 3rd fork TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(20); task_1_1.setRetryCount(1); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.CANCELED); task_1_1.setRetried(true); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_1_2 = new TaskModel(); task_1_2.setTaskId(UUID.randomUUID().toString()); task_1_2.setSeq(21); task_1_2.setRetryCount(1); task_1_2.setTaskType(TaskType.SIMPLE.toString()); task_1_2.setStatus(TaskModel.Status.FAILED); task_1_2.setTaskDefName("task1"); task_1_2.setWorkflowTask(new WorkflowTask()); task_1_2.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(22); task_2_1.setRetryCount(1); task_2_1.setStatus(TaskModel.Status.FAILED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); TaskModel task_3_1 = new TaskModel(); task_3_1.setTaskId(UUID.randomUUID().toString()); task_3_1.setSeq(23); task_3_1.setRetryCount(1); task_3_1.setStatus(TaskModel.Status.CANCELED); task_3_1.setTaskType(TaskType.SIMPLE.toString()); task_3_1.setTaskDefName("task3"); task_3_1.setWorkflowTask(new WorkflowTask()); task_3_1.setReferenceTaskName("task3_ref1"); TaskModel task_4_1 = new TaskModel(); task_4_1.setTaskId(UUID.randomUUID().toString()); task_4_1.setSeq(122); task_4_1.setRetryCount(1); task_4_1.setStatus(TaskModel.Status.FAILED); task_4_1.setTaskType(TaskType.SIMPLE.toString()); task_4_1.setTaskDefName("task1"); task_4_1.setWorkflowTask(new WorkflowTask()); task_4_1.setReferenceTaskName("task4_refABC"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_1_2, task_2_1, task_3_1, task_4_1)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); // then: assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertEquals(1, updateWorkflowCalledCounter.get()); assertEquals(1, updateTasksCalledCounter.get()); assertEquals(0, updateTaskCalledCounter.get()); } @Test public void testRetryWorkflowReturnsNoDuplicates() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(10); task_1_1.setRetryCount(0); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_1_2 = new TaskModel(); task_1_2.setTaskId(UUID.randomUUID().toString()); task_1_2.setSeq(11); task_1_2.setRetryCount(1); task_1_2.setTaskType(TaskType.SIMPLE.toString()); task_1_2.setStatus(TaskModel.Status.COMPLETED); task_1_2.setTaskDefName("task1"); task_1_2.setWorkflowTask(new WorkflowTask()); task_1_2.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(21); task_2_1.setRetryCount(0); task_2_1.setStatus(TaskModel.Status.CANCELED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); TaskModel task_3_1 = new TaskModel(); task_3_1.setTaskId(UUID.randomUUID().toString()); task_3_1.setSeq(31); task_3_1.setRetryCount(1); task_3_1.setStatus(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); task_3_1.setTaskType(TaskType.SIMPLE.toString()); task_3_1.setTaskDefName("task1"); task_3_1.setWorkflowTask(new WorkflowTask()); task_3_1.setReferenceTaskName("task3_ref1"); TaskModel task_4_1 = new TaskModel(); task_4_1.setTaskId(UUID.randomUUID().toString()); task_4_1.setSeq(41); task_4_1.setRetryCount(0); task_4_1.setStatus(TaskModel.Status.TIMED_OUT); task_4_1.setTaskType(TaskType.SIMPLE.toString()); task_4_1.setTaskDefName("task1"); task_4_1.setWorkflowTask(new WorkflowTask()); task_4_1.setReferenceTaskName("task4_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_1_2, task_2_1, task_3_1, task_4_1)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); assertEquals(8, workflow.getTasks().size()); } @Test public void testRetryWorkflowMultipleRetries() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(10); task_1_1.setRetryCount(0); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(20); task_2_1.setRetryCount(0); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setStatus(TaskModel.Status.CANCELED); task_2_1.setTaskDefName("task1"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_2_1)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); assertEquals(4, workflow.getTasks().size()); // Reset Last Workflow Task to FAILED. TaskModel lastTask = workflow.getTasks().stream() .filter(t -> t.getReferenceTaskName().equals("task1_ref1")) .collect( groupingBy( TaskModel::getReferenceTaskName, maxBy(comparingInt(TaskModel::getSeq)))) .values() .stream() .map(Optional::get) .collect(Collectors.toList()) .get(0); lastTask.setStatus(TaskModel.Status.FAILED); workflow.setStatus(WorkflowModel.Status.FAILED); workflowExecutor.retry(workflow.getWorkflowId(), false); assertEquals(5, workflow.getTasks().size()); // Reset Last Workflow Task to FAILED. // Reset Last Workflow Task to FAILED. TaskModel lastTask2 = workflow.getTasks().stream() .filter(t -> t.getReferenceTaskName().equals("task1_ref1")) .collect( groupingBy( TaskModel::getReferenceTaskName, maxBy(comparingInt(TaskModel::getSeq)))) .values() .stream() .map(Optional::get) .collect(Collectors.toList()) .get(0); lastTask2.setStatus(TaskModel.Status.FAILED); workflow.setStatus(WorkflowModel.Status.FAILED); workflowExecutor.retry(workflow.getWorkflowId(), false); assertEquals(6, workflow.getTasks().size()); } @Test public void testRetryWorkflowWithJoinTask() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); TaskModel forkTask = new TaskModel(); forkTask.setTaskType(TaskType.FORK_JOIN.toString()); forkTask.setTaskId(UUID.randomUUID().toString()); forkTask.setSeq(1); forkTask.setRetryCount(1); forkTask.setStatus(TaskModel.Status.COMPLETED); forkTask.setReferenceTaskName("task_fork"); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(20); task_1_1.setRetryCount(1); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(22); task_2_1.setRetryCount(1); task_2_1.setStatus(TaskModel.Status.CANCELED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); TaskModel joinTask = new TaskModel(); joinTask.setTaskType(TaskType.JOIN.toString()); joinTask.setTaskId(UUID.randomUUID().toString()); joinTask.setSeq(25); joinTask.setRetryCount(1); joinTask.setStatus(TaskModel.Status.CANCELED); joinTask.setReferenceTaskName("task_join"); joinTask.getInputData() .put( "joinOn", Arrays.asList( task_1_1.getReferenceTaskName(), task_2_1.getReferenceTaskName())); workflow.getTasks().addAll(Arrays.asList(forkTask, task_1_1, task_2_1, joinTask)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); assertEquals(6, workflow.getTasks().size()); assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); } @Test public void testRetryFromLastFailedSubWorkflowTaskThenStartWithLastFailedTask() { IDGenerator idGenerator = new IDGenerator(); // given String id = idGenerator.generate(); String workflowInstanceId = idGenerator.generate(); TaskModel task = new TaskModel(); task.setTaskType(TaskType.SIMPLE.name()); task.setTaskDefName("task"); task.setReferenceTaskName("task_ref"); task.setWorkflowInstanceId(workflowInstanceId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED); task.setRetryCount(0); task.setWorkflowTask(new WorkflowTask()); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(id); task.setSeq(1); TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(workflowInstanceId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.FAILED); task1.setRetryCount(0); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); task1.setSubWorkflowId(id); task1.setSeq(2); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setWorkflowId(id); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("subworkflow"); workflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(workflowDef); subWorkflow.setStatus(WorkflowModel.Status.FAILED); subWorkflow.getTasks().addAll(Arrays.asList(task, task1)); subWorkflow.setParentWorkflowId("testRunWorkflowId"); TaskModel task2 = new TaskModel(); task2.setWorkflowInstanceId(subWorkflow.getWorkflowId()); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.FAILED); task2.setRetryCount(0); task2.setOutputData(new HashMap<>()); task2.setSubWorkflowId(id); task2.setTaskType(TaskType.SUB_WORKFLOW.name()); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRunWorkflowId"); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setTasks(Collections.singletonList(task2)); workflowDef = new WorkflowDef(); workflowDef.setName("first_workflow"); workflow.setWorkflowDefinition(workflowDef); // when when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(workflowDef)); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task1); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); workflowExecutor.retry(workflow.getWorkflowId(), true); // then assertEquals(task.getStatus(), TaskModel.Status.COMPLETED); assertEquals(task1.getStatus(), TaskModel.Status.IN_PROGRESS); assertEquals(workflow.getPreviousStatus(), WorkflowModel.Status.FAILED); assertEquals(workflow.getStatus(), WorkflowModel.Status.RUNNING); assertEquals(subWorkflow.getPreviousStatus(), WorkflowModel.Status.FAILED); assertEquals(subWorkflow.getStatus(), WorkflowModel.Status.RUNNING); } @Test public void testRetryTimedOutWorkflowWithoutFailedTasks() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.TIMED_OUT); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(20); task_1_1.setRetryCount(1); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.COMPLETED); task_1_1.setRetried(true); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(22); task_2_1.setRetryCount(1); task_2_1.setStatus(TaskModel.Status.COMPLETED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_2_1)); AtomicInteger updateWorkflowCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateWorkflowCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateWorkflow(any()); AtomicInteger updateTasksCalledCounter = new AtomicInteger(0); doAnswer( invocation -> { updateTasksCalledCounter.incrementAndGet(); return null; }) .when(executionDAOFacade) .updateTasks(any()); // end of setup // when when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); workflowExecutor.retry(workflow.getWorkflowId(), false); // then assertEquals(WorkflowModel.Status.TIMED_OUT, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertTrue(workflow.getLastRetriedTime() > 0); assertEquals(1, updateWorkflowCalledCounter.get()); assertEquals(1, updateTasksCalledCounter.get()); } @Test(expected = ConflictException.class) public void testRerunNonTerminalWorkflow() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryNonTerminalWorkflow"); workflow.setStatus(WorkflowModel.Status.RUNNING); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(workflow.getWorkflowId()); workflowExecutor.rerun(rerunWorkflowRequest); } @Test public void testRerunWorkflow() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRerunWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRerunWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setReasonForIncompletion("task1 failed"); workflow.setFailedReferenceTaskNames( new HashSet<>() { { add("task1_ref1"); } }); workflow.setFailedTaskNames( new HashSet<>() { { add("task1"); } }); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(20); task_1_1.setRetryCount(1); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setRetried(true); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(22); task_2_1.setRetryCount(1); task_2_1.setStatus(TaskModel.Status.CANCELED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_2_1)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(workflow.getWorkflowId()); workflowExecutor.rerun(rerunWorkflowRequest); // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertNull(workflow.getReasonForIncompletion()); assertEquals(new HashSet<>(), workflow.getFailedReferenceTaskNames()); assertEquals(new HashSet<>(), workflow.getFailedTaskNames()); } @Test public void testRerunSubWorkflow() { IDGenerator idGenerator = new IDGenerator(); // setup String parentWorkflowId = idGenerator.generate(); String subWorkflowId = idGenerator.generate(); // sub workflow setup TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(subWorkflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.SIMPLE.name()); task2.setTaskDefName("task2"); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(subWorkflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.COMPLETED); task2.setWorkflowTask(new WorkflowTask()); task2.setOutputData(new HashMap<>()); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setParentWorkflowId(parentWorkflowId); subWorkflow.setWorkflowId(subWorkflowId); WorkflowDef subworkflowDef = new WorkflowDef(); subworkflowDef.setName("subworkflow"); subworkflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(subworkflowDef); subWorkflow.setOwnerApp("junit_testRerunWorkflowId"); subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); subWorkflow.getTasks().addAll(Arrays.asList(task1, task2)); // parent workflow setup TaskModel task = new TaskModel(); task.setWorkflowInstanceId(parentWorkflowId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(subWorkflowId); task.setTaskType(TaskType.SUB_WORKFLOW.name()); task.setWorkflowTask(new WorkflowTask()); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(parentWorkflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("parentworkflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(subWorkflow.getWorkflowId()); workflowExecutor.rerun(rerunWorkflowRequest); // then: assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, subWorkflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, subWorkflow.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); } @Test public void testRerunWorkflowWithTaskId() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRerunWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setReasonForIncompletion("task1 failed"); workflow.setFailedReferenceTaskNames( new HashSet<>() { { add("task1_ref1"); } }); workflow.setFailedTaskNames( new HashSet<>() { { add("task1"); } }); TaskModel task_1_1 = new TaskModel(); task_1_1.setTaskId(UUID.randomUUID().toString()); task_1_1.setSeq(20); task_1_1.setRetryCount(1); task_1_1.setTaskType(TaskType.SIMPLE.toString()); task_1_1.setStatus(TaskModel.Status.FAILED); task_1_1.setRetried(true); task_1_1.setTaskDefName("task1"); task_1_1.setWorkflowTask(new WorkflowTask()); task_1_1.setReferenceTaskName("task1_ref1"); TaskModel task_2_1 = new TaskModel(); task_2_1.setTaskId(UUID.randomUUID().toString()); task_2_1.setSeq(22); task_2_1.setRetryCount(1); task_2_1.setStatus(TaskModel.Status.CANCELED); task_2_1.setTaskType(TaskType.SIMPLE.toString()); task_2_1.setTaskDefName("task2"); task_2_1.setWorkflowTask(new WorkflowTask()); task_2_1.setReferenceTaskName("task2_ref1"); workflow.getTasks().addAll(Arrays.asList(task_1_1, task_2_1)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); when(metadataDAO.getWorkflowDef(anyString(), anyInt())) .thenReturn(Optional.of(new WorkflowDef())); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(workflow.getWorkflowId()); rerunWorkflowRequest.setReRunFromTaskId(task_1_1.getTaskId()); workflowExecutor.rerun(rerunWorkflowRequest); // when: when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertNull(workflow.getReasonForIncompletion()); assertEquals(new HashSet<>(), workflow.getFailedReferenceTaskNames()); assertEquals(new HashSet<>(), workflow.getFailedTaskNames()); } @Test public void testRerunWorkflowWithSyncSystemTaskId() { IDGenerator idGenerator = new IDGenerator(); // setup String workflowId = idGenerator.generate(); TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(workflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.JSON_JQ_TRANSFORM.name()); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(workflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId("system-task-id"); task2.setStatus(TaskModel.Status.FAILED); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("workflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setReasonForIncompletion("task2 failed"); workflow.setFailedReferenceTaskNames( new HashSet<>() { { add("task2_ref"); } }); workflow.setFailedTaskNames( new HashSet<>() { { add("task2"); } }); workflow.getTasks().addAll(Arrays.asList(task1, task2)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(workflow.getWorkflowId()); rerunWorkflowRequest.setReRunFromTaskId(task2.getTaskId()); workflowExecutor.rerun(rerunWorkflowRequest); // then: assertEquals(TaskModel.Status.COMPLETED, task2.getStatus()); assertEquals(WorkflowModel.Status.FAILED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertNull(workflow.getReasonForIncompletion()); assertEquals(new HashSet<>(), workflow.getFailedReferenceTaskNames()); assertEquals(new HashSet<>(), workflow.getFailedTaskNames()); } @Test public void testRerunSubWorkflowWithTaskId() { IDGenerator idGenerator = new IDGenerator(); // setup String parentWorkflowId = idGenerator.generate(); String subWorkflowId = idGenerator.generate(); // sub workflow setup TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(subWorkflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.SIMPLE.name()); task2.setTaskDefName("task2"); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(subWorkflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.COMPLETED); task2.setWorkflowTask(new WorkflowTask()); task2.setOutputData(new HashMap<>()); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setParentWorkflowId(parentWorkflowId); subWorkflow.setWorkflowId(subWorkflowId); WorkflowDef subworkflowDef = new WorkflowDef(); subworkflowDef.setName("subworkflow"); subworkflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(subworkflowDef); subWorkflow.setOwnerApp("junit_testRerunWorkflowId"); subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); subWorkflow.getTasks().addAll(Arrays.asList(task1, task2)); // parent workflow setup TaskModel task = new TaskModel(); task.setWorkflowInstanceId(parentWorkflowId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(subWorkflowId); task.setTaskType(TaskType.SUB_WORKFLOW.name()); task.setWorkflowTask(new WorkflowTask()); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(parentWorkflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("parentworkflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(subWorkflow.getWorkflowId()); rerunWorkflowRequest.setReRunFromTaskId(task2.getTaskId()); workflowExecutor.rerun(rerunWorkflowRequest); // then: assertEquals(TaskModel.Status.SCHEDULED, task2.getStatus()); assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, subWorkflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, subWorkflow.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); } @Test public void testGetActiveDomain() throws Exception { String taskType = "test-task"; String[] domains = new String[] {"domain1", "domain2"}; PollData pollData1 = new PollData( "queue1", domains[0], "worker1", System.currentTimeMillis() - 99 * 1000); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[0])) .thenReturn(pollData1); String activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[0], activeDomain); Thread.sleep(2000L); PollData pollData2 = new PollData( "queue2", domains[1], "worker2", System.currentTimeMillis() - 99 * 1000); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[1])) .thenReturn(pollData2); activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[1], activeDomain); Thread.sleep(2000L); activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[1], activeDomain); domains = new String[] {""}; when(executionDAOFacade.getTaskPollDataByDomain(any(), any())).thenReturn(new PollData()); activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertNotNull(activeDomain); assertEquals("", activeDomain); domains = new String[] {}; activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertNull(activeDomain); activeDomain = workflowExecutor.getActiveDomain(taskType, null); assertNull(activeDomain); domains = new String[] {"test-domain"}; when(executionDAOFacade.getTaskPollDataByDomain(anyString(), anyString())).thenReturn(null); activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertNotNull(activeDomain); assertEquals("test-domain", activeDomain); } @Test public void testInactiveDomains() { String taskType = "test-task"; String[] domains = new String[] {"domain1", "domain2"}; PollData pollData1 = new PollData( "queue1", domains[0], "worker1", System.currentTimeMillis() - 99 * 10000); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[0])) .thenReturn(pollData1); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[1])).thenReturn(null); String activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals("domain2", activeDomain); } @Test public void testDefaultDomain() { String taskType = "test-task"; String[] domains = new String[] {"domain1", "domain2", "NO_DOMAIN"}; PollData pollData1 = new PollData( "queue1", domains[0], "worker1", System.currentTimeMillis() - 99 * 10000); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[0])) .thenReturn(pollData1); when(executionDAOFacade.getTaskPollDataByDomain(taskType, domains[1])).thenReturn(null); String activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertNull(activeDomain); } @Test public void testTaskToDomain() { WorkflowModel workflow = generateSampleWorkflow(); List tasks = generateSampleTasks(3); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "mydomain"); workflow.setTaskToDomain(taskToDomain); PollData pollData1 = new PollData( "queue1", "mydomain", "worker1", System.currentTimeMillis() - 99 * 100); when(executionDAOFacade.getTaskPollDataByDomain(anyString(), anyString())) .thenReturn(pollData1); workflowExecutor.setTaskDomains(tasks, workflow); assertNotNull(tasks); tasks.forEach(task -> assertEquals("mydomain", task.getDomain())); } @Test public void testTaskToDomainsPerTask() { WorkflowModel workflow = generateSampleWorkflow(); List tasks = generateSampleTasks(2); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "mydomain, NO_DOMAIN"); workflow.setTaskToDomain(taskToDomain); PollData pollData1 = new PollData( "queue1", "mydomain", "worker1", System.currentTimeMillis() - 99 * 100); when(executionDAOFacade.getTaskPollDataByDomain(eq("task1"), anyString())) .thenReturn(pollData1); when(executionDAOFacade.getTaskPollDataByDomain(eq("task2"), anyString())).thenReturn(null); workflowExecutor.setTaskDomains(tasks, workflow); assertEquals("mydomain", tasks.get(0).getDomain()); assertNull(tasks.get(1).getDomain()); } @Test public void testTaskToDomainOverrides() { WorkflowModel workflow = generateSampleWorkflow(); List tasks = generateSampleTasks(4); Map taskToDomain = new HashMap<>(); taskToDomain.put("*", "mydomain"); taskToDomain.put("task2", "someInactiveDomain, NO_DOMAIN"); taskToDomain.put("task3", "someActiveDomain, NO_DOMAIN"); taskToDomain.put("task4", "someInactiveDomain, someInactiveDomain2"); workflow.setTaskToDomain(taskToDomain); PollData pollData1 = new PollData( "queue1", "mydomain", "worker1", System.currentTimeMillis() - 99 * 100); PollData pollData2 = new PollData( "queue2", "someActiveDomain", "worker2", System.currentTimeMillis() - 99 * 100); when(executionDAOFacade.getTaskPollDataByDomain(anyString(), eq("mydomain"))) .thenReturn(pollData1); when(executionDAOFacade.getTaskPollDataByDomain(anyString(), eq("someInactiveDomain"))) .thenReturn(null); when(executionDAOFacade.getTaskPollDataByDomain(anyString(), eq("someActiveDomain"))) .thenReturn(pollData2); when(executionDAOFacade.getTaskPollDataByDomain(anyString(), eq("someInactiveDomain"))) .thenReturn(null); workflowExecutor.setTaskDomains(tasks, workflow); assertEquals("mydomain", tasks.get(0).getDomain()); assertNull(tasks.get(1).getDomain()); assertEquals("someActiveDomain", tasks.get(2).getDomain()); assertEquals("someInactiveDomain2", tasks.get(3).getDomain()); } @Test public void testDedupAndAddTasks() { WorkflowModel workflow = new WorkflowModel(); TaskModel task1 = new TaskModel(); task1.setReferenceTaskName("task1"); task1.setRetryCount(1); TaskModel task2 = new TaskModel(); task2.setReferenceTaskName("task2"); task2.setRetryCount(2); List tasks = new ArrayList<>(Arrays.asList(task1, task2)); List taskList = workflowExecutor.dedupAndAddTasks(workflow, tasks); assertEquals(2, taskList.size()); assertEquals(tasks, taskList); assertEquals(workflow.getTasks(), taskList); // Adding the same tasks again taskList = workflowExecutor.dedupAndAddTasks(workflow, tasks); assertEquals(0, taskList.size()); assertEquals(workflow.getTasks(), tasks); // Adding 2 new tasks TaskModel newTask = new TaskModel(); newTask.setReferenceTaskName("newTask"); newTask.setRetryCount(0); taskList = workflowExecutor.dedupAndAddTasks(workflow, Collections.singletonList(newTask)); assertEquals(1, taskList.size()); assertEquals(newTask, taskList.get(0)); assertEquals(3, workflow.getTasks().size()); } @Test(expected = ConflictException.class) public void testTerminateCompletedWorkflow() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testTerminateTerminalWorkflow"); workflow.setStatus(WorkflowModel.Status.COMPLETED); when(executionDAOFacade.getWorkflowModel(anyString(), anyBoolean())).thenReturn(workflow); workflowExecutor.terminateWorkflow( workflow.getWorkflowId(), "test terminating terminal workflow"); } @Test public void testResetCallbacksForWorkflowTasks() { String workflowId = "test-workflow-id"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); workflow.setStatus(WorkflowModel.Status.RUNNING); TaskModel completedTask = new TaskModel(); completedTask.setTaskType(TaskType.SIMPLE.name()); completedTask.setReferenceTaskName("completedTask"); completedTask.setWorkflowInstanceId(workflowId); completedTask.setScheduledTime(System.currentTimeMillis()); completedTask.setCallbackAfterSeconds(300); completedTask.setTaskId("simple-task-id"); completedTask.setStatus(TaskModel.Status.COMPLETED); TaskModel systemTask = new TaskModel(); systemTask.setTaskType(TaskType.WAIT.name()); systemTask.setReferenceTaskName("waitTask"); systemTask.setWorkflowInstanceId(workflowId); systemTask.setScheduledTime(System.currentTimeMillis()); systemTask.setTaskId("system-task-id"); systemTask.setStatus(TaskModel.Status.SCHEDULED); TaskModel simpleTask = new TaskModel(); simpleTask.setTaskType(TaskType.SIMPLE.name()); simpleTask.setReferenceTaskName("simpleTask"); simpleTask.setWorkflowInstanceId(workflowId); simpleTask.setScheduledTime(System.currentTimeMillis()); simpleTask.setCallbackAfterSeconds(300); simpleTask.setTaskId("simple-task-id"); simpleTask.setStatus(TaskModel.Status.SCHEDULED); TaskModel noCallbackTask = new TaskModel(); noCallbackTask.setTaskType(TaskType.SIMPLE.name()); noCallbackTask.setReferenceTaskName("noCallbackTask"); noCallbackTask.setWorkflowInstanceId(workflowId); noCallbackTask.setScheduledTime(System.currentTimeMillis()); noCallbackTask.setCallbackAfterSeconds(0); noCallbackTask.setTaskId("no-callback-task-id"); noCallbackTask.setStatus(TaskModel.Status.SCHEDULED); workflow.getTasks() .addAll(Arrays.asList(completedTask, systemTask, simpleTask, noCallbackTask)); when(executionDAOFacade.getWorkflowModel(workflowId, true)).thenReturn(workflow); workflowExecutor.resetCallbacksForWorkflow(workflowId); verify(queueDAO, times(1)).resetOffsetTime(anyString(), anyString()); } @Test public void testUpdateParentWorkflowTask() { String parentWorkflowTaskId = "parent_workflow_task_id"; String workflowId = "workflow_id"; WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setWorkflowId(workflowId); subWorkflow.setParentWorkflowTaskId(parentWorkflowTaskId); subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); TaskModel subWorkflowTask = new TaskModel(); subWorkflowTask.setSubWorkflowId(workflowId); subWorkflowTask.setStatus(TaskModel.Status.IN_PROGRESS); subWorkflowTask.setExternalOutputPayloadStoragePath(null); when(executionDAOFacade.getTaskModel(parentWorkflowTaskId)).thenReturn(subWorkflowTask); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(subWorkflow); workflowExecutor.updateParentWorkflowTask(subWorkflow); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskModel.class); verify(executionDAOFacade, times(1)).updateTask(argumentCaptor.capture()); assertEquals(TaskModel.Status.COMPLETED, argumentCaptor.getAllValues().get(0).getStatus()); assertEquals(workflowId, argumentCaptor.getAllValues().get(0).getSubWorkflowId()); } @Test public void testScheduleNextIteration() { WorkflowModel workflow = generateSampleWorkflow(); workflow.setTaskToDomain( new HashMap<>() { { put("TEST", "domain1"); } }); TaskModel loopTask = mock(TaskModel.class); WorkflowTask loopWfTask = mock(WorkflowTask.class); when(loopTask.getWorkflowTask()).thenReturn(loopWfTask); List loopOver = new ArrayList<>() { { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_SIMPLE); workflowTask.setName("TEST"); workflowTask.setTaskDefinition(new TaskDef()); add(workflowTask); } }; when(loopWfTask.getLoopOver()).thenReturn(loopOver); workflowExecutor.scheduleNextIteration(loopTask, workflow); verify(executionDAOFacade).getTaskPollDataByDomain("TEST", "domain1"); } @Test public void testCancelNonTerminalTasks() { WorkflowDef def = new WorkflowDef(); def.setWorkflowStatusListenerEnabled(true); WorkflowModel workflow = generateSampleWorkflow(); workflow.setWorkflowDefinition(def); TaskModel subWorkflowTask = new TaskModel(); subWorkflowTask.setTaskId(UUID.randomUUID().toString()); subWorkflowTask.setTaskType(TaskType.SUB_WORKFLOW.name()); subWorkflowTask.setStatus(TaskModel.Status.IN_PROGRESS); TaskModel lambdaTask = new TaskModel(); lambdaTask.setTaskId(UUID.randomUUID().toString()); lambdaTask.setTaskType(TaskType.LAMBDA.name()); lambdaTask.setStatus(TaskModel.Status.SCHEDULED); TaskModel simpleTask = new TaskModel(); simpleTask.setTaskId(UUID.randomUUID().toString()); simpleTask.setTaskType(TaskType.SIMPLE.name()); simpleTask.setStatus(TaskModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(subWorkflowTask, lambdaTask, simpleTask)); List erroredTasks = workflowExecutor.cancelNonTerminalTasks(workflow); assertTrue(erroredTasks.isEmpty()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskModel.class); verify(executionDAOFacade, times(2)).updateTask(argumentCaptor.capture()); assertEquals(2, argumentCaptor.getAllValues().size()); assertEquals( TaskType.SUB_WORKFLOW.name(), argumentCaptor.getAllValues().get(0).getTaskType()); assertEquals(TaskModel.Status.CANCELED, argumentCaptor.getAllValues().get(0).getStatus()); assertEquals(TaskType.LAMBDA.name(), argumentCaptor.getAllValues().get(1).getTaskType()); assertEquals(TaskModel.Status.CANCELED, argumentCaptor.getAllValues().get(1).getStatus()); verify(workflowStatusListener, times(1)) .onWorkflowFinalizedIfEnabled(any(WorkflowModel.class)); } @Test public void testPauseWorkflow() { when(executionLockService.acquireLock(anyString(), anyLong())).thenReturn(true); doNothing().when(executionLockService).releaseLock(anyString()); String workflowId = "testPauseWorkflowId"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); // if workflow is in terminal state workflow.setStatus(WorkflowModel.Status.COMPLETED); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); try { workflowExecutor.pauseWorkflow(workflowId); fail("Expected " + ConflictException.class); } catch (ConflictException e) { verify(executionDAOFacade, never()).updateWorkflow(any(WorkflowModel.class)); verify(queueDAO, never()).remove(anyString(), anyString()); } // if workflow is already PAUSED workflow.setStatus(WorkflowModel.Status.PAUSED); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); workflowExecutor.pauseWorkflow(workflowId); assertEquals(WorkflowModel.Status.PAUSED, workflow.getStatus()); verify(executionDAOFacade, never()).updateWorkflow(any(WorkflowModel.class)); verify(queueDAO, never()).remove(anyString(), anyString()); // if workflow is RUNNING workflow.setStatus(WorkflowModel.Status.RUNNING); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); workflowExecutor.pauseWorkflow(workflowId); assertEquals(WorkflowModel.Status.PAUSED, workflow.getStatus()); verify(executionDAOFacade, times(1)).updateWorkflow(any(WorkflowModel.class)); verify(queueDAO, times(1)).remove(anyString(), anyString()); } @Test public void testResumeWorkflow() { String workflowId = "testResumeWorkflowId"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); // if workflow is not in PAUSED state workflow.setStatus(WorkflowModel.Status.COMPLETED); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); try { workflowExecutor.resumeWorkflow(workflowId); } catch (Exception e) { assertTrue(e instanceof IllegalStateException); verify(executionDAOFacade, never()).updateWorkflow(any(WorkflowModel.class)); verify(queueDAO, never()).push(anyString(), anyString(), anyInt(), anyLong()); } // if workflow is in PAUSED state workflow.setStatus(WorkflowModel.Status.PAUSED); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); workflowExecutor.resumeWorkflow(workflowId); assertEquals(WorkflowModel.Status.RUNNING, workflow.getStatus()); assertTrue(workflow.getLastRetriedTime() > 0); verify(executionDAOFacade, times(1)).updateWorkflow(any(WorkflowModel.class)); verify(queueDAO, times(1)).push(anyString(), anyString(), anyInt(), anyLong()); } @Test @SuppressWarnings("unchecked") public void testTerminateWorkflowWithFailureWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("workflow"); workflowDef.setFailureWorkflow("failure_workflow"); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("1"); workflow.setCorrelationId("testid"); workflow.setWorkflowDefinition(new WorkflowDef()); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setOwnerApp("junit_test"); workflow.setEndTime(100L); workflow.setOutput(Collections.EMPTY_MAP); workflow.setWorkflowDefinition(workflowDef); TaskModel successTask = new TaskModel(); successTask.setTaskId("taskid1"); successTask.setReferenceTaskName("success"); successTask.setStatus(TaskModel.Status.COMPLETED); TaskModel failedTask = new TaskModel(); failedTask.setTaskId("taskid2"); failedTask.setReferenceTaskName("failed"); failedTask.setStatus(TaskModel.Status.FAILED); workflow.getTasks().addAll(Arrays.asList(successTask, failedTask)); WorkflowDef failureWorkflowDef = new WorkflowDef(); failureWorkflowDef.setName("failure_workflow"); when(metadataDAO.getLatestWorkflowDef(failureWorkflowDef.getName())) .thenReturn(Optional.of(failureWorkflowDef)); when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionLockService.acquireLock(anyString())).thenReturn(true); workflowExecutor.decide(workflow.getWorkflowId()); assertEquals(WorkflowModel.Status.FAILED, workflow.getStatus()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(WorkflowCreationEvent.class); verify(eventPublisher, times(1)).publishEvent(argumentCaptor.capture()); StartWorkflowInput startWorkflowInput = argumentCaptor.getValue().getStartWorkflowInput(); assertEquals(workflow.getCorrelationId(), startWorkflowInput.getCorrelationId()); assertEquals( workflow.getWorkflowId(), startWorkflowInput.getWorkflowInput().get("workflowId")); assertEquals( failedTask.getTaskId(), startWorkflowInput.getWorkflowInput().get("failureTaskId")); assertNotNull( failedTask.getTaskId(), startWorkflowInput.getWorkflowInput().get("failedWorkflow")); } @Test public void testRerunOptionalSubWorkflow() { IDGenerator idGenerator = new IDGenerator(); // setup String parentWorkflowId = idGenerator.generate(); String subWorkflowId = idGenerator.generate(); // sub workflow setup TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(subWorkflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.SIMPLE.name()); task2.setTaskDefName("task2"); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(subWorkflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.FAILED); task2.setWorkflowTask(new WorkflowTask()); task2.setOutputData(new HashMap<>()); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setParentWorkflowId(parentWorkflowId); subWorkflow.setWorkflowId(subWorkflowId); WorkflowDef subworkflowDef = new WorkflowDef(); subworkflowDef.setName("subworkflow"); subworkflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(subworkflowDef); subWorkflow.setOwnerApp("junit_testRerunWorkflowId"); subWorkflow.setStatus(WorkflowModel.Status.FAILED); subWorkflow.getTasks().addAll(Arrays.asList(task1, task2)); // parent workflow setup TaskModel task = new TaskModel(); task.setWorkflowInstanceId(parentWorkflowId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED_WITH_ERRORS); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(subWorkflowId); task.setTaskType(TaskType.SUB_WORKFLOW.name()); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setOptional(true); task.setWorkflowTask(workflowTask); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(parentWorkflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("parentworkflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest(); rerunWorkflowRequest.setReRunFromWorkflowId(subWorkflow.getWorkflowId()); workflowExecutor.rerun(rerunWorkflowRequest); // then: parent workflow remains the same assertEquals(WorkflowModel.Status.FAILED, subWorkflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, subWorkflow.getStatus()); assertEquals(TaskModel.Status.COMPLETED_WITH_ERRORS, task.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getStatus()); } @Test public void testRestartOptionalSubWorkflow() { IDGenerator idGenerator = new IDGenerator(); // setup String parentWorkflowId = idGenerator.generate(); String subWorkflowId = idGenerator.generate(); // sub workflow setup TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(subWorkflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.SIMPLE.name()); task2.setTaskDefName("task2"); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(subWorkflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.FAILED); task2.setWorkflowTask(new WorkflowTask()); task2.setOutputData(new HashMap<>()); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setParentWorkflowId(parentWorkflowId); subWorkflow.setWorkflowId(subWorkflowId); WorkflowDef subworkflowDef = new WorkflowDef(); subworkflowDef.setName("subworkflow"); subworkflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(subworkflowDef); subWorkflow.setOwnerApp("junit_testRerunWorkflowId"); subWorkflow.setStatus(WorkflowModel.Status.FAILED); subWorkflow.getTasks().addAll(Arrays.asList(task1, task2)); // parent workflow setup TaskModel task = new TaskModel(); task.setWorkflowInstanceId(parentWorkflowId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED_WITH_ERRORS); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(subWorkflowId); task.setTaskType(TaskType.SUB_WORKFLOW.name()); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setOptional(true); task.setWorkflowTask(workflowTask); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(parentWorkflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("parentworkflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); workflowExecutor.restart(subWorkflowId, false); // then: parent workflow remains the same assertEquals(WorkflowModel.Status.FAILED, subWorkflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, subWorkflow.getStatus()); assertEquals(TaskModel.Status.COMPLETED_WITH_ERRORS, task.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getStatus()); } @Test public void testRetryOptionalSubWorkflow() { IDGenerator idGenerator = new IDGenerator(); // setup String parentWorkflowId = idGenerator.generate(); String subWorkflowId = idGenerator.generate(); // sub workflow setup TaskModel task1 = new TaskModel(); task1.setTaskType(TaskType.SIMPLE.name()); task1.setTaskDefName("task1"); task1.setReferenceTaskName("task1_ref"); task1.setWorkflowInstanceId(subWorkflowId); task1.setScheduledTime(System.currentTimeMillis()); task1.setTaskId(idGenerator.generate()); task1.setStatus(TaskModel.Status.COMPLETED); task1.setWorkflowTask(new WorkflowTask()); task1.setOutputData(new HashMap<>()); TaskModel task2 = new TaskModel(); task2.setTaskType(TaskType.SIMPLE.name()); task2.setTaskDefName("task2"); task2.setReferenceTaskName("task2_ref"); task2.setWorkflowInstanceId(subWorkflowId); task2.setScheduledTime(System.currentTimeMillis()); task2.setTaskId(idGenerator.generate()); task2.setStatus(TaskModel.Status.FAILED); task2.setWorkflowTask(new WorkflowTask()); task2.setOutputData(new HashMap<>()); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setParentWorkflowId(parentWorkflowId); subWorkflow.setWorkflowId(subWorkflowId); WorkflowDef subworkflowDef = new WorkflowDef(); subworkflowDef.setName("subworkflow"); subworkflowDef.setVersion(1); subWorkflow.setWorkflowDefinition(subworkflowDef); subWorkflow.setOwnerApp("junit_testRerunWorkflowId"); subWorkflow.setStatus(WorkflowModel.Status.FAILED); subWorkflow.getTasks().addAll(Arrays.asList(task1, task2)); // parent workflow setup TaskModel task = new TaskModel(); task.setWorkflowInstanceId(parentWorkflowId); task.setScheduledTime(System.currentTimeMillis()); task.setTaskId(idGenerator.generate()); task.setStatus(TaskModel.Status.COMPLETED_WITH_ERRORS); task.setOutputData(new HashMap<>()); task.setSubWorkflowId(subWorkflowId); task.setTaskType(TaskType.SUB_WORKFLOW.name()); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setOptional(true); task.setWorkflowTask(workflowTask); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(parentWorkflowId); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("parentworkflow"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setStatus(WorkflowModel.Status.COMPLETED); workflow.getTasks().addAll(Arrays.asList(task)); // end of setup // when: when(executionDAOFacade.getWorkflowModel(workflow.getWorkflowId(), true)) .thenReturn(workflow); when(executionDAOFacade.getWorkflowModel(task.getSubWorkflowId(), true)) .thenReturn(subWorkflow); when(executionDAOFacade.getTaskModel(subWorkflow.getParentWorkflowTaskId())) .thenReturn(task); when(executionDAOFacade.getWorkflowModel(subWorkflow.getParentWorkflowId(), false)) .thenReturn(workflow); workflowExecutor.retry(subWorkflowId, true); // then: parent workflow remains the same assertEquals(WorkflowModel.Status.FAILED, subWorkflow.getPreviousStatus()); assertEquals(WorkflowModel.Status.RUNNING, subWorkflow.getStatus()); assertEquals(TaskModel.Status.COMPLETED_WITH_ERRORS, task.getStatus()); assertEquals(WorkflowModel.Status.COMPLETED, workflow.getStatus()); } @Test public void testUpdateTaskWithCallbackAfterSeconds() { String workflowId = "test-workflow-id"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setWorkflowDefinition(new WorkflowDef()); TaskModel simpleTask = new TaskModel(); simpleTask.setTaskType(TaskType.SIMPLE.name()); simpleTask.setReferenceTaskName("simpleTask"); simpleTask.setWorkflowInstanceId(workflowId); simpleTask.setScheduledTime(System.currentTimeMillis()); simpleTask.setCallbackAfterSeconds(0); simpleTask.setTaskId("simple-task-id"); simpleTask.setStatus(TaskModel.Status.IN_PROGRESS); workflow.getTasks().add(simpleTask); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); when(executionDAOFacade.getTaskModel(simpleTask.getTaskId())).thenReturn(simpleTask); TaskResult taskResult = new TaskResult(); taskResult.setWorkflowInstanceId(workflowId); taskResult.setTaskId(simpleTask.getTaskId()); taskResult.setWorkerId("test-worker-id"); taskResult.log("not ready yet"); taskResult.setCallbackAfterSeconds(300); taskResult.setStatus(TaskResult.Status.IN_PROGRESS); workflowExecutor.updateTask(taskResult); verify(queueDAO, times(1)).postpone(anyString(), anyString(), anyInt(), anyLong()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskModel.class); verify(executionDAOFacade, times(1)).updateTask(argumentCaptor.capture()); assertEquals(TaskModel.Status.SCHEDULED, argumentCaptor.getAllValues().get(0).getStatus()); assertEquals( taskResult.getCallbackAfterSeconds(), argumentCaptor.getAllValues().get(0).getCallbackAfterSeconds()); assertEquals(taskResult.getWorkerId(), argumentCaptor.getAllValues().get(0).getWorkerId()); } @Test public void testUpdateTaskWithOutCallbackAfterSeconds() { String workflowId = "test-workflow-id"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); workflow.setStatus(WorkflowModel.Status.RUNNING); workflow.setWorkflowDefinition(new WorkflowDef()); TaskModel simpleTask = new TaskModel(); simpleTask.setTaskType(TaskType.SIMPLE.name()); simpleTask.setReferenceTaskName("simpleTask"); simpleTask.setWorkflowInstanceId(workflowId); simpleTask.setScheduledTime(System.currentTimeMillis()); simpleTask.setCallbackAfterSeconds(0); simpleTask.setTaskId("simple-task-id"); simpleTask.setStatus(TaskModel.Status.IN_PROGRESS); workflow.getTasks().add(simpleTask); when(executionDAOFacade.getWorkflowModel(workflowId, false)).thenReturn(workflow); when(executionDAOFacade.getTaskModel(simpleTask.getTaskId())).thenReturn(simpleTask); TaskResult taskResult = new TaskResult(); taskResult.setWorkflowInstanceId(workflowId); taskResult.setTaskId(simpleTask.getTaskId()); taskResult.setWorkerId("test-worker-id"); taskResult.log("not ready yet"); taskResult.setStatus(TaskResult.Status.IN_PROGRESS); workflowExecutor.updateTask(taskResult); verify(queueDAO, times(1)).postpone(anyString(), anyString(), anyInt(), anyLong()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskModel.class); verify(executionDAOFacade, times(1)).updateTask(argumentCaptor.capture()); assertEquals(TaskModel.Status.SCHEDULED, argumentCaptor.getAllValues().get(0).getStatus()); assertEquals(0, argumentCaptor.getAllValues().get(0).getCallbackAfterSeconds()); assertEquals(taskResult.getWorkerId(), argumentCaptor.getAllValues().get(0).getWorkerId()); } @Test public void testIsLazyEvaluateWorkflow() { // setup WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("lazyEvaluate"); workflowDef.setVersion(1); WorkflowTask simpleTask = new WorkflowTask(); simpleTask.setType(SIMPLE.name()); simpleTask.setName("simple"); simpleTask.setTaskReferenceName("simple"); WorkflowTask forkTask = new WorkflowTask(); forkTask.setType(FORK_JOIN.name()); forkTask.setName("fork"); forkTask.setTaskReferenceName("fork"); WorkflowTask branchTask1 = new WorkflowTask(); branchTask1.setType(SIMPLE.name()); branchTask1.setName("branchTask1"); branchTask1.setTaskReferenceName("branchTask1"); WorkflowTask branchTask2 = new WorkflowTask(); branchTask2.setType(SIMPLE.name()); branchTask2.setName("branchTask2"); branchTask2.setTaskReferenceName("branchTask2"); forkTask.getForkTasks().add(Arrays.asList(branchTask1, branchTask2)); WorkflowTask joinTask = new WorkflowTask(); joinTask.setType(JOIN.name()); joinTask.setName("join"); joinTask.setTaskReferenceName("join"); joinTask.setJoinOn(List.of("branchTask2")); WorkflowTask doWhile = new WorkflowTask(); doWhile.setType(DO_WHILE.name()); doWhile.setName("doWhile"); doWhile.setTaskReferenceName("doWhile"); WorkflowTask loopTask = new WorkflowTask(); loopTask.setType(SIMPLE.name()); loopTask.setName("loopTask"); loopTask.setTaskReferenceName("loopTask"); doWhile.setLoopOver(List.of(loopTask)); workflowDef.getTasks().addAll(List.of(simpleTask, forkTask, joinTask, doWhile)); TaskModel task = new TaskModel(); task.setStatus(TaskModel.Status.COMPLETED); // when: task.setReferenceTaskName("dynamic"); assertTrue(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); task.setReferenceTaskName("branchTask1"); assertFalse(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); task.setReferenceTaskName("branchTask2"); assertTrue(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); task.setReferenceTaskName("simple"); assertFalse(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); task.setReferenceTaskName("loopTask__1"); task.setIteration(1); assertFalse(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); task.setReferenceTaskName("branchTask1"); task.setStatus(TaskModel.Status.FAILED); assertFalse(workflowExecutor.isLazyEvaluateWorkflow(workflowDef, task)); } @Test public void testTaskExtendLease() { TaskModel simpleTask = new TaskModel(); simpleTask.setTaskType(TaskType.SIMPLE.name()); simpleTask.setReferenceTaskName("simpleTask"); simpleTask.setWorkflowInstanceId("test-workflow-id"); simpleTask.setScheduledTime(System.currentTimeMillis()); simpleTask.setCallbackAfterSeconds(0); simpleTask.setTaskId("simple-task-id"); simpleTask.setStatus(TaskModel.Status.IN_PROGRESS); when(executionDAOFacade.getTaskModel(simpleTask.getTaskId())).thenReturn(simpleTask); TaskResult taskResult = new TaskResult(); taskResult.setWorkflowInstanceId(simpleTask.getWorkflowInstanceId()); taskResult.setTaskId(simpleTask.getTaskId()); taskResult.log("extend lease"); taskResult.setExtendLease(true); workflowExecutor.updateTask(taskResult); verify(executionDAOFacade, times(1)).extendLease(simpleTask); verify(queueDAO, times(0)).postpone(anyString(), anyString(), anyInt(), anyLong()); verify(executionDAOFacade, times(0)).updateTask(any()); } private WorkflowModel generateSampleWorkflow() { // setup WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("testRetryWorkflowId"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testRetryWorkflowId"); workflowDef.setVersion(1); workflow.setWorkflowDefinition(workflowDef); workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); //noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); return workflow; } private List generateSampleTasks(int count) { if (count == 0) { return null; } List tasks = new ArrayList<>(); for (int i = 0; i < count; i++) { TaskModel task = new TaskModel(); task.setTaskId(UUID.randomUUID().toString()); task.setSeq(i); task.setRetryCount(1); task.setTaskType("task" + (i + 1)); task.setStatus(TaskModel.Status.COMPLETED); task.setTaskDefName("taskX"); task.setReferenceTaskName("task_ref" + (i + 1)); tasks.add(task); } return tasks; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/WorkflowSystemTaskStub.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; public class WorkflowSystemTaskStub extends WorkflowSystemTask { private boolean started = false; public WorkflowSystemTaskStub(String taskType) { super(taskType); } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { started = true; task.setStatus(TaskModel.Status.COMPLETED); super.start(workflow, task, executor); } public boolean isStarted() { return started; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class DecisionTaskMapperTest { private IDGenerator idGenerator; private ParametersUtils parametersUtils; private DeciderService deciderService; // Subject private DecisionTaskMapper decisionTaskMapper; @Autowired private ObjectMapper objectMapper; @Rule public ExpectedException expectedException = ExpectedException.none(); Map ip1; WorkflowTask task1; WorkflowTask task2; WorkflowTask task3; @Before public void setUp() { parametersUtils = new ParametersUtils(objectMapper); idGenerator = new IDGenerator(); ip1 = new HashMap<>(); ip1.put("p1", "${workflow.input.param1}"); ip1.put("p2", "${workflow.input.param2}"); ip1.put("case", "${workflow.input.case}"); task1 = new WorkflowTask(); task1.setName("Test1"); task1.setInputParameters(ip1); task1.setTaskReferenceName("t1"); task2 = new WorkflowTask(); task2.setName("Test2"); task2.setInputParameters(ip1); task2.setTaskReferenceName("t2"); task3 = new WorkflowTask(); task3.setName("Test3"); task3.setInputParameters(ip1); task3.setTaskReferenceName("t3"); deciderService = mock(DeciderService.class); decisionTaskMapper = new DecisionTaskMapper(); } @Test public void getMappedTasks() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("Id", "${workflow.input.Id}"); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Decision task instance WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(TaskType.DECISION.name()); decisionTask.setName("Decision"); decisionTask.setTaskReferenceName("decisionTask"); decisionTask.setDefaultCase(Collections.singletonList(task1)); decisionTask.setCaseValueParam("case"); decisionTask.getInputParameters().put("Id", "${workflow.input.Id}"); decisionTask.setCaseExpression( "if ($.Id == null) 'bad input'; else if ( ($.Id != null && $.Id % 2 == 0)) 'even'; else 'odd'; "); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); decisionCases.put("odd", Collections.singletonList(task3)); decisionTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); Map workflowInput = new HashMap<>(); workflowInput.put("Id", "22"); workflowModel.setInput(workflowInput); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map input = parametersUtils.getTaskInput( decisionTask.getInputParameters(), workflowModel, null, null); TaskModel theTask = new TaskModel(); theTask.setReferenceTaskName("Foo"); theTask.setTaskId(idGenerator.generate()); when(deciderService.getTasksToBeScheduled(workflowModel, task2, 0, null)) .thenReturn(Collections.singletonList(theTask)); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(decisionTask) .withTaskInput(input) .withRetryCount(0) .withTaskId(idGenerator.generate()) .withDeciderService(deciderService) .build(); // When List mappedTasks = decisionTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(2, mappedTasks.size()); assertEquals("decisionTask", mappedTasks.get(0).getReferenceTaskName()); assertEquals("Foo", mappedTasks.get(1).getReferenceTaskName()); } @Test public void getEvaluatedCaseValue() { WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(TaskType.DECISION.name()); decisionTask.setName("Decision"); decisionTask.setTaskReferenceName("decisionTask"); decisionTask.setInputParameters(ip1); decisionTask.setDefaultCase(Collections.singletonList(task1)); decisionTask.setCaseValueParam("case"); Map> decisionCases = new HashMap<>(); decisionCases.put("0", Collections.singletonList(task2)); decisionCases.put("1", Collections.singletonList(task3)); decisionTask.setDecisionCases(decisionCases); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(new WorkflowDef()); Map workflowInput = new HashMap<>(); workflowInput.put("param1", "test1"); workflowInput.put("param2", "test2"); workflowInput.put("case", "0"); workflowModel.setInput(workflowInput); Map input = parametersUtils.getTaskInput( decisionTask.getInputParameters(), workflowModel, null, null); assertEquals("0", decisionTaskMapper.getEvaluatedCaseValue(decisionTask, input)); } @Test public void getEvaluatedCaseValueUsingExpression() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("Id", "${workflow.input.Id}"); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Decision task instance WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(TaskType.DECISION.name()); decisionTask.setName("Decision"); decisionTask.setTaskReferenceName("decisionTask"); decisionTask.setDefaultCase(Collections.singletonList(task1)); decisionTask.setCaseValueParam("case"); decisionTask.getInputParameters().put("Id", "${workflow.input.Id}"); decisionTask.setCaseExpression( "if ($.Id == null) 'bad input'; else if ( ($.Id != null && $.Id % 2 == 0)) 'even'; else 'odd'; "); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); decisionCases.put("odd", Collections.singletonList(task3)); decisionTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef def = new WorkflowDef(); def.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(def); Map workflowInput = new HashMap<>(); workflowInput.put("Id", "22"); workflowModel.setInput(workflowInput); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map evaluatorInput = parametersUtils.getTaskInput( decisionTask.getInputParameters(), workflowModel, taskDef, null); assertEquals( "even", decisionTaskMapper.getEvaluatedCaseValue(decisionTask, evaluatorInput)); } @Test public void getEvaluatedCaseValueException() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("Id", "${workflow.input.Id}"); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Decision task instance WorkflowTask decisionTask = new WorkflowTask(); decisionTask.setType(TaskType.DECISION.name()); decisionTask.setName("Decision"); decisionTask.setTaskReferenceName("decisionTask"); decisionTask.setDefaultCase(Collections.singletonList(task1)); decisionTask.setCaseValueParam("case"); decisionTask.getInputParameters().put("Id", "${workflow.input.Id}"); decisionTask.setCaseExpression( "if ($Id == null) 'bad input'; else if ( ($Id != null && $Id % 2 == 0)) 'even'; else 'odd'; "); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); decisionCases.put("odd", Collections.singletonList(task3)); decisionTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef def = new WorkflowDef(); def.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(def); Map workflowInput = new HashMap<>(); workflowInput.put(".Id", "22"); workflowModel.setInput(workflowInput); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map evaluatorInput = parametersUtils.getTaskInput( decisionTask.getInputParameters(), workflowModel, taskDef, null); expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( "Error while evaluating script: " + decisionTask.getCaseExpression()); decisionTaskMapper.getEvaluatedCaseValue(decisionTask, evaluatorInput); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/DoWhileTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DO_WHILE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class DoWhileTaskMapperTest { private TaskModel task1; private DeciderService deciderService; private WorkflowModel workflow; private WorkflowTask workflowTask1; private TaskMapperContext taskMapperContext; private MetadataDAO metadataDAO; private ParametersUtils parametersUtils; @Before public void setup() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.DO_WHILE.name()); workflowTask.setTaskReferenceName("Test"); workflowTask.setInputParameters(Map.of("value", "${workflow.input.foo}")); task1 = new TaskModel(); task1.setReferenceTaskName("task1"); TaskModel task2 = new TaskModel(); task2.setReferenceTaskName("task2"); workflowTask1 = new WorkflowTask(); workflowTask1.setTaskReferenceName("task1"); WorkflowTask workflowTask2 = new WorkflowTask(); workflowTask2.setTaskReferenceName("task2"); task1.setWorkflowTask(workflowTask1); task2.setWorkflowTask(workflowTask2); workflowTask.setLoopOver(Arrays.asList(task1.getWorkflowTask(), task2.getWorkflowTask())); workflowTask.setLoopCondition( "if ($.second_task + $.first_task > 10) { false; } else { true; }"); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); workflow.setInput(Map.of("foo", "bar")); deciderService = Mockito.mock(DeciderService.class); metadataDAO = Mockito.mock(MetadataDAO.class); taskMapperContext = TaskMapperContext.newBuilder() .withDeciderService(deciderService) .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); parametersUtils = new ParametersUtils(new ObjectMapper()); } @Test public void getMappedTasks() { Mockito.doReturn(Collections.singletonList(task1)) .when(deciderService) .getTasksToBeScheduled(workflow, workflowTask1, 0); List mappedTasks = new DoWhileTaskMapper(metadataDAO, parametersUtils) .getMappedTasks(taskMapperContext); assertNotNull(mappedTasks); assertEquals(mappedTasks.size(), 1); assertEquals(TASK_TYPE_DO_WHILE, mappedTasks.get(0).getTaskType()); assertNotNull(mappedTasks.get(0).getInputData()); assertEquals(Map.of("value", "bar"), mappedTasks.get(0).getInputData()); } @Test public void shouldNotScheduleCompletedTask() { task1.setStatus(TaskModel.Status.COMPLETED); List mappedTasks = new DoWhileTaskMapper(metadataDAO, parametersUtils) .getMappedTasks(taskMapperContext); assertNotNull(mappedTasks); assertEquals(mappedTasks.size(), 1); } @Test public void testAppendIteration() { assertEquals("task__1", TaskUtils.appendIteration("task", 1)); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class DynamicTaskMapperTest { @Rule public ExpectedException expectedException = ExpectedException.none(); private ParametersUtils parametersUtils; private MetadataDAO metadataDAO; private DynamicTaskMapper dynamicTaskMapper; @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); metadataDAO = mock(MetadataDAO.class); dynamicTaskMapper = new DynamicTaskMapper(parametersUtils, metadataDAO); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("DynoTask"); workflowTask.setDynamicTaskNameParam("dynamicTaskName"); TaskDef taskDef = new TaskDef(); taskDef.setName("DynoTask"); workflowTask.setTaskDefinition(taskDef); Map taskInput = new HashMap<>(); taskInput.put("dynamicTaskName", "DynoTask"); when(parametersUtils.getTaskInput( anyMap(), any(WorkflowModel.class), any(TaskDef.class), anyString())) .thenReturn(taskInput); String taskId = new IDGenerator().generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(workflowTask.getTaskDefinition()) .withWorkflowTask(workflowTask) .withTaskInput(taskInput) .withRetryCount(0) .withTaskId(taskId) .build(); when(metadataDAO.getTaskDef("DynoTask")).thenReturn(new TaskDef()); List mappedTasks = dynamicTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); TaskModel dynamicTask = mappedTasks.get(0); assertEquals(taskId, dynamicTask.getTaskId()); } @Test public void getDynamicTaskName() { Map taskInput = new HashMap<>(); taskInput.put("dynamicTaskName", "DynoTask"); String dynamicTaskName = dynamicTaskMapper.getDynamicTaskName(taskInput, "dynamicTaskName"); assertEquals("DynoTask", dynamicTaskName); } @Test public void getDynamicTaskNameNotAvailable() { Map taskInput = new HashMap<>(); expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( String.format( "Cannot map a dynamic task based on the parameter and input. " + "Parameter= %s, input= %s", "dynamicTaskName", taskInput)); dynamicTaskMapper.getDynamicTaskName(taskInput, "dynamicTaskName"); } @Test public void getDynamicTaskDefinition() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Foo"); TaskDef taskDef = new TaskDef(); taskDef.setName("Foo"); workflowTask.setTaskDefinition(taskDef); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); // when TaskDef dynamicTaskDefinition = dynamicTaskMapper.getDynamicTaskDefinition(workflowTask); assertEquals(dynamicTaskDefinition, taskDef); } @Test public void getDynamicTaskDefinitionNull() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Foo"); expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( String.format( "Invalid task specified. Cannot find task by name %s in the task definitions", workflowTask.getName())); dynamicTaskMapper.getDynamicTaskDefinition(workflowTask); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/EventTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class EventTaskMapperTest { @Test public void getMappedTasks() { ParametersUtils parametersUtils = Mockito.mock(ParametersUtils.class); EventTaskMapper eventTaskMapper = new EventTaskMapper(parametersUtils); WorkflowTask taskToBeScheduled = new WorkflowTask(); taskToBeScheduled.setSink("SQSSINK"); String taskId = new IDGenerator().generate(); Map eventTaskInput = new HashMap<>(); eventTaskInput.put("sink", "SQSSINK"); when(parametersUtils.getTaskInput( anyMap(), any(WorkflowModel.class), any(TaskDef.class), anyString())) .thenReturn(eventTaskInput); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(taskToBeScheduled) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = eventTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); TaskModel eventTask = mappedTasks.get(0); assertEquals(taskId, eventTask.getTaskId()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.*; import org.apache.commons.lang3.tuple.Pair; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.DynamicForkJoinTaskList; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; @SuppressWarnings("unchecked") public class ForkJoinDynamicTaskMapperTest { private IDGenerator idGenerator; private ParametersUtils parametersUtils; private ObjectMapper objectMapper; private DeciderService deciderService; private ForkJoinDynamicTaskMapper forkJoinDynamicTaskMapper; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { MetadataDAO metadataDAO = Mockito.mock(MetadataDAO.class); idGenerator = new IDGenerator(); parametersUtils = Mockito.mock(ParametersUtils.class); objectMapper = Mockito.mock(ObjectMapper.class); deciderService = Mockito.mock(DeciderService.class); forkJoinDynamicTaskMapper = new ForkJoinDynamicTaskMapper( idGenerator, parametersUtils, objectMapper, metadataDAO); } @Test public void getMappedTasksException() { WorkflowDef def = new WorkflowDef(); def.setName("DYNAMIC_FORK_JOIN_WF"); def.setDescription(def.getName()); def.setVersion(1); def.setInputParameters(Arrays.asList("param1", "param2")); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(def); WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkTasksParam("dynamicTasks"); dynamicForkJoinToSchedule.setDynamicForkTasksInputParamName("dynamicTasksInput"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); WorkflowTask join = new WorkflowTask(); join.setType(TaskType.JOIN.name()); join.setTaskReferenceName("dynamictask_join"); def.getTasks().add(dynamicForkJoinToSchedule); Map input1 = new HashMap<>(); input1.put("k1", "v1"); WorkflowTask wt2 = new WorkflowTask(); wt2.setName("junit_task_2"); wt2.setTaskReferenceName("xdt1"); Map input2 = new HashMap<>(); input2.put("k2", "v2"); WorkflowTask wt3 = new WorkflowTask(); wt3.setName("junit_task_3"); wt3.setTaskReferenceName("xdt2"); HashMap dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("xdt1", input1); dynamicTasksInput.put("xdt2", input2); dynamicTasksInput.put("dynamicTasks", Arrays.asList(wt2, wt3)); dynamicTasksInput.put("dynamicTasksInput", dynamicTasksInput); // when when(parametersUtils.getTaskInput(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(TypeReference.class))) .thenReturn(Arrays.asList(wt2, wt3)); TaskModel simpleTask1 = new TaskModel(); simpleTask1.setReferenceTaskName("xdt1"); TaskModel simpleTask2 = new TaskModel(); simpleTask2.setReferenceTaskName("xdt2"); when(deciderService.getTasksToBeScheduled(workflowModel, wt2, 0)) .thenReturn(Collections.singletonList(simpleTask1)); when(deciderService.getTasksToBeScheduled(workflowModel, wt3, 0)) .thenReturn(Collections.singletonList(simpleTask2)); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(dynamicForkJoinToSchedule) .withRetryCount(0) .withTaskId(taskId) .withDeciderService(deciderService) .build(); // then expectedException.expect(TerminateWorkflowException.class); forkJoinDynamicTaskMapper.getMappedTasks(taskMapperContext); } @Test public void getMappedTasks() { WorkflowDef def = new WorkflowDef(); def.setName("DYNAMIC_FORK_JOIN_WF"); def.setDescription(def.getName()); def.setVersion(1); def.setInputParameters(Arrays.asList("param1", "param2")); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(def); WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkTasksParam("dynamicTasks"); dynamicForkJoinToSchedule.setDynamicForkTasksInputParamName("dynamicTasksInput"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); WorkflowTask join = new WorkflowTask(); join.setType(TaskType.JOIN.name()); join.setTaskReferenceName("dynamictask_join"); def.getTasks().add(dynamicForkJoinToSchedule); def.getTasks().add(join); Map input1 = new HashMap<>(); input1.put("k1", "v1"); WorkflowTask wt2 = new WorkflowTask(); wt2.setName("junit_task_2"); wt2.setTaskReferenceName("xdt1"); Map input2 = new HashMap<>(); input2.put("k2", "v2"); WorkflowTask wt3 = new WorkflowTask(); wt3.setName("junit_task_3"); wt3.setTaskReferenceName("xdt2"); HashMap dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("xdt1", input1); dynamicTasksInput.put("xdt2", input2); dynamicTasksInput.put("dynamicTasks", Arrays.asList(wt2, wt3)); dynamicTasksInput.put("dynamicTasksInput", dynamicTasksInput); // when when(parametersUtils.getTaskInput(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(TypeReference.class))) .thenReturn(Arrays.asList(wt2, wt3)); TaskModel simpleTask1 = new TaskModel(); simpleTask1.setReferenceTaskName("xdt1"); TaskModel simpleTask2 = new TaskModel(); simpleTask2.setReferenceTaskName("xdt2"); when(deciderService.getTasksToBeScheduled(workflowModel, wt2, 0)) .thenReturn(Collections.singletonList(simpleTask1)); when(deciderService.getTasksToBeScheduled(workflowModel, wt3, 0)) .thenReturn(Collections.singletonList(simpleTask2)); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(dynamicForkJoinToSchedule) .withRetryCount(0) .withTaskId(taskId) .withDeciderService(deciderService) .build(); // then List mappedTasks = forkJoinDynamicTaskMapper.getMappedTasks(taskMapperContext); assertEquals(4, mappedTasks.size()); assertEquals(TASK_TYPE_FORK, mappedTasks.get(0).getTaskType()); assertEquals(TASK_TYPE_JOIN, mappedTasks.get(3).getTaskType()); List joinTaskNames = (List) mappedTasks.get(3).getInputData().get("joinOn"); assertEquals("xdt1, xdt2", String.join(", ", joinTaskNames)); } @Test public void getDynamicForkJoinTasksAndInput() { // Given WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkJoinTasksParam("dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); DynamicForkJoinTaskList dtasks = new DynamicForkJoinTaskList(); Map input = new HashMap<>(); input.put("k1", "v1"); dtasks.add("junit_task_2", null, "xdt1", input); HashMap input2 = new HashMap<>(); input2.put("k2", "v2"); dtasks.add("junit_task_3", null, "xdt2", input2); Map dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("dynamicTasks", dtasks); // when when(parametersUtils.getTaskInput( anyMap(), any(WorkflowModel.class), any(TaskDef.class), anyString())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(Class.class))).thenReturn(dtasks); Pair, Map>> dynamicForkJoinTasksAndInput = forkJoinDynamicTaskMapper.getDynamicForkJoinTasksAndInput( dynamicForkJoinToSchedule, new WorkflowModel()); // then assertNotNull(dynamicForkJoinTasksAndInput.getLeft()); assertEquals(2, dynamicForkJoinTasksAndInput.getLeft().size()); assertEquals(2, dynamicForkJoinTasksAndInput.getRight().size()); } @Test public void getDynamicForkJoinTasksAndInputException() { // Given WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkJoinTasksParam("dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); DynamicForkJoinTaskList dtasks = new DynamicForkJoinTaskList(); Map input = new HashMap<>(); input.put("k1", "v1"); dtasks.add("junit_task_2", null, "xdt1", input); HashMap input2 = new HashMap<>(); input2.put("k2", "v2"); dtasks.add("junit_task_3", null, "xdt2", input2); Map dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("dynamicTasks", dtasks); // when when(parametersUtils.getTaskInput( anyMap(), any(WorkflowModel.class), any(TaskDef.class), anyString())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(Class.class))).thenReturn(null); // then expectedException.expect(TerminateWorkflowException.class); forkJoinDynamicTaskMapper.getDynamicForkJoinTasksAndInput( dynamicForkJoinToSchedule, new WorkflowModel()); } @Test public void getDynamicForkTasksAndInput() { // Given WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkTasksParam("dynamicTasks"); dynamicForkJoinToSchedule.setDynamicForkTasksInputParamName("dynamicTasksInput"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); Map input1 = new HashMap<>(); input1.put("k1", "v1"); WorkflowTask wt2 = new WorkflowTask(); wt2.setName("junit_task_2"); wt2.setTaskReferenceName("xdt1"); Map input2 = new HashMap<>(); input2.put("k2", "v2"); WorkflowTask wt3 = new WorkflowTask(); wt3.setName("junit_task_3"); wt3.setTaskReferenceName("xdt2"); HashMap dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("xdt1", input1); dynamicTasksInput.put("xdt2", input2); dynamicTasksInput.put("dynamicTasks", Arrays.asList(wt2, wt3)); dynamicTasksInput.put("dynamicTasksInput", dynamicTasksInput); // when when(parametersUtils.getTaskInput(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(TypeReference.class))) .thenReturn(Arrays.asList(wt2, wt3)); Pair, Map>> dynamicTasks = forkJoinDynamicTaskMapper.getDynamicForkTasksAndInput( dynamicForkJoinToSchedule, new WorkflowModel(), "dynamicTasks"); // then assertNotNull(dynamicTasks.getLeft()); } @Test public void getDynamicForkTasksAndInputException() { // Given WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkTasksParam("dynamicTasks"); dynamicForkJoinToSchedule.setDynamicForkTasksInputParamName("dynamicTasksInput"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); Map input1 = new HashMap<>(); input1.put("k1", "v1"); WorkflowTask wt2 = new WorkflowTask(); wt2.setName("junit_task_2"); wt2.setTaskReferenceName("xdt1"); Map input2 = new HashMap<>(); input2.put("k2", "v2"); WorkflowTask wt3 = new WorkflowTask(); wt3.setName("junit_task_3"); wt3.setTaskReferenceName("xdt2"); HashMap dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("xdt1", input1); dynamicTasksInput.put("xdt2", input2); dynamicTasksInput.put("dynamicTasks", Arrays.asList(wt2, wt3)); dynamicTasksInput.put("dynamicTasksInput", null); when(parametersUtils.getTaskInput(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(TypeReference.class))) .thenReturn(Arrays.asList(wt2, wt3)); // then expectedException.expect(TerminateWorkflowException.class); // when forkJoinDynamicTaskMapper.getDynamicForkTasksAndInput( dynamicForkJoinToSchedule, new WorkflowModel(), "dynamicTasks"); } @Test public void testDynamicTaskDuplicateTaskRefName() { WorkflowDef def = new WorkflowDef(); def.setName("DYNAMIC_FORK_JOIN_WF"); def.setDescription(def.getName()); def.setVersion(1); def.setInputParameters(Arrays.asList("param1", "param2")); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(def); WorkflowTask dynamicForkJoinToSchedule = new WorkflowTask(); dynamicForkJoinToSchedule.setType(TaskType.FORK_JOIN_DYNAMIC.name()); dynamicForkJoinToSchedule.setTaskReferenceName("dynamicfanouttask"); dynamicForkJoinToSchedule.setDynamicForkTasksParam("dynamicTasks"); dynamicForkJoinToSchedule.setDynamicForkTasksInputParamName("dynamicTasksInput"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasks", "dt1.output.dynamicTasks"); dynamicForkJoinToSchedule .getInputParameters() .put("dynamicTasksInput", "dt1.output.dynamicTasksInput"); WorkflowTask join = new WorkflowTask(); join.setType(TaskType.JOIN.name()); join.setTaskReferenceName("dynamictask_join"); def.getTasks().add(dynamicForkJoinToSchedule); def.getTasks().add(join); Map input1 = new HashMap<>(); input1.put("k1", "v1"); WorkflowTask wt2 = new WorkflowTask(); wt2.setName("junit_task_2"); wt2.setTaskReferenceName("xdt1"); Map input2 = new HashMap<>(); input2.put("k2", "v2"); WorkflowTask wt3 = new WorkflowTask(); wt3.setName("junit_task_3"); wt3.setTaskReferenceName("xdt2"); HashMap dynamicTasksInput = new HashMap<>(); dynamicTasksInput.put("xdt1", input1); dynamicTasksInput.put("xdt2", input2); dynamicTasksInput.put("dynamicTasks", Arrays.asList(wt2, wt3)); dynamicTasksInput.put("dynamicTasksInput", dynamicTasksInput); // dynamic when(parametersUtils.getTaskInput(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(dynamicTasksInput); when(objectMapper.convertValue(any(), any(TypeReference.class))) .thenReturn(Arrays.asList(wt2, wt3)); TaskModel simpleTask1 = new TaskModel(); simpleTask1.setReferenceTaskName("xdt1"); // Empty list, this is a bad state, workflow should terminate when(deciderService.getTasksToBeScheduled(workflowModel, wt2, 0)) .thenReturn(new ArrayList<>()); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(dynamicForkJoinToSchedule) .withRetryCount(0) .withTaskId(taskId) .withDeciderService(deciderService) .build(); expectedException.expect(TerminateWorkflowException.class); forkJoinDynamicTaskMapper.getMappedTasks(taskMapperContext); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK; import static org.junit.Assert.assertEquals; public class ForkJoinTaskMapperTest { private DeciderService deciderService; private ForkJoinTaskMapper forkJoinTaskMapper; private IDGenerator idGenerator; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { deciderService = Mockito.mock(DeciderService.class); forkJoinTaskMapper = new ForkJoinTaskMapper(); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { WorkflowDef def = new WorkflowDef(); def.setName("FORK_JOIN_WF"); def.setDescription(def.getName()); def.setVersion(1); def.setInputParameters(Arrays.asList("param1", "param2")); WorkflowTask forkTask = new WorkflowTask(); forkTask.setType(TaskType.FORK_JOIN.name()); forkTask.setTaskReferenceName("forktask"); WorkflowTask wft1 = new WorkflowTask(); wft1.setName("junit_task_1"); Map ip1 = new HashMap<>(); ip1.put("p1", "workflow.input.param1"); ip1.put("p2", "workflow.input.param2"); wft1.setInputParameters(ip1); wft1.setTaskReferenceName("t1"); WorkflowTask wft3 = new WorkflowTask(); wft3.setName("junit_task_3"); wft3.setInputParameters(ip1); wft3.setTaskReferenceName("t3"); WorkflowTask wft2 = new WorkflowTask(); wft2.setName("junit_task_2"); Map ip2 = new HashMap<>(); ip2.put("tp1", "workflow.input.param1"); wft2.setInputParameters(ip2); wft2.setTaskReferenceName("t2"); WorkflowTask wft4 = new WorkflowTask(); wft4.setName("junit_task_4"); wft4.setInputParameters(ip2); wft4.setTaskReferenceName("t4"); forkTask.getForkTasks().add(Arrays.asList(wft1, wft3)); forkTask.getForkTasks().add(Collections.singletonList(wft2)); def.getTasks().add(forkTask); WorkflowTask join = new WorkflowTask(); join.setType(TaskType.JOIN.name()); join.setTaskReferenceName("forktask_join"); join.setJoinOn(Arrays.asList("t3", "t2")); def.getTasks().add(join); def.getTasks().add(wft4); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); TaskModel task1 = new TaskModel(); task1.setReferenceTaskName(wft1.getTaskReferenceName()); TaskModel task3 = new TaskModel(); task3.setReferenceTaskName(wft3.getTaskReferenceName()); Mockito.when(deciderService.getTasksToBeScheduled(workflow, wft1, 0)) .thenReturn(Collections.singletonList(task1)); Mockito.when(deciderService.getTasksToBeScheduled(workflow, wft2, 0)) .thenReturn(Collections.singletonList(task3)); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withWorkflowTask(forkTask) .withRetryCount(0) .withTaskId(taskId) .withDeciderService(deciderService) .build(); List mappedTasks = forkJoinTaskMapper.getMappedTasks(taskMapperContext); assertEquals(3, mappedTasks.size()); assertEquals(TASK_TYPE_FORK, mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasksException() { WorkflowDef def = new WorkflowDef(); def.setName("FORK_JOIN_WF"); def.setDescription(def.getName()); def.setVersion(1); def.setInputParameters(Arrays.asList("param1", "param2")); WorkflowTask forkTask = new WorkflowTask(); forkTask.setType(TaskType.FORK_JOIN.name()); forkTask.setTaskReferenceName("forktask"); WorkflowTask wft1 = new WorkflowTask(); wft1.setName("junit_task_1"); Map ip1 = new HashMap<>(); ip1.put("p1", "workflow.input.param1"); ip1.put("p2", "workflow.input.param2"); wft1.setInputParameters(ip1); wft1.setTaskReferenceName("t1"); WorkflowTask wft3 = new WorkflowTask(); wft3.setName("junit_task_3"); wft3.setInputParameters(ip1); wft3.setTaskReferenceName("t3"); WorkflowTask wft2 = new WorkflowTask(); wft2.setName("junit_task_2"); Map ip2 = new HashMap<>(); ip2.put("tp1", "workflow.input.param1"); wft2.setInputParameters(ip2); wft2.setTaskReferenceName("t2"); WorkflowTask wft4 = new WorkflowTask(); wft4.setName("junit_task_4"); wft4.setInputParameters(ip2); wft4.setTaskReferenceName("t4"); forkTask.getForkTasks().add(Arrays.asList(wft1, wft3)); forkTask.getForkTasks().add(Collections.singletonList(wft2)); def.getTasks().add(forkTask); WorkflowTask join = new WorkflowTask(); join.setType(TaskType.JOIN.name()); join.setTaskReferenceName("forktask_join"); join.setJoinOn(Arrays.asList("t3", "t2")); def.getTasks().add(wft4); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); TaskModel task1 = new TaskModel(); task1.setReferenceTaskName(wft1.getTaskReferenceName()); TaskModel task3 = new TaskModel(); task3.setReferenceTaskName(wft3.getTaskReferenceName()); Mockito.when(deciderService.getTasksToBeScheduled(workflow, wft1, 0)) .thenReturn(Collections.singletonList(task1)); Mockito.when(deciderService.getTasksToBeScheduled(workflow, wft2, 0)) .thenReturn(Collections.singletonList(task3)); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withWorkflowTask(forkTask) .withRetryCount(0) .withTaskId(taskId) .withDeciderService(deciderService) .build(); expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( "Fork task definition is not followed by a join task. Check the blueprint"); forkJoinTaskMapper.getMappedTasks(taskMapperContext); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/HTTPTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class HTTPTaskMapperTest { private HTTPTaskMapper httpTaskMapper; private IDGenerator idGenerator; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { ParametersUtils parametersUtils = mock(ParametersUtils.class); MetadataDAO metadataDAO = mock(MetadataDAO.class); httpTaskMapper = new HTTPTaskMapper(parametersUtils, metadataDAO); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("http_task"); workflowTask.setType(TaskType.HTTP.name()); workflowTask.setTaskDefinition(new TaskDef("http_task")); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // when List mappedTasks = httpTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TaskType.HTTP.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasks_WithoutTaskDef() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("http_task"); workflowTask.setType(TaskType.HTTP.name()); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(null) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // when List mappedTasks = httpTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TaskType.HTTP.name(), mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/HumanTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HUMAN; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class HumanTaskMapperTest { @Test public void getMappedTasks() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("human_task"); workflowTask.setType(TaskType.HUMAN.name()); String taskId = new IDGenerator().generate(); ParametersUtils parametersUtils = mock(ParametersUtils.class); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withTaskId(taskId) .build(); HumanTaskMapper humanTaskMapper = new HumanTaskMapper(parametersUtils); // When List mappedTasks = humanTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TASK_TYPE_HUMAN, mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/InlineTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.evaluators.JavascriptEvaluator; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; public class InlineTaskMapperTest { private ParametersUtils parametersUtils; private MetadataDAO metadataDAO; @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); metadataDAO = mock(MetadataDAO.class); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("inline_task"); workflowTask.setType(TaskType.INLINE.name()); workflowTask.setTaskDefinition(new TaskDef("inline_task")); workflowTask.setEvaluatorType(JavascriptEvaluator.NAME); workflowTask.setExpression( "function scriptFun() {if ($.input.a==1){return {testValue: true}} else{return " + "{testValue: false} }}; scriptFun();"); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new InlineTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.INLINE.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasks_WithoutTaskDef() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.INLINE.name()); workflowTask.setEvaluatorType(JavascriptEvaluator.NAME); workflowTask.setExpression( "function scriptFun() {if ($.input.a==1){return {testValue: true}} else{return " + "{testValue: false} }}; scriptFun();"); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(null) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new InlineTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.INLINE.name(), mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/JoinTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Arrays; import java.util.List; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class JoinTaskMapperTest { @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.JOIN.name()); workflowTask.setJoinOn(Arrays.asList("task1", "task2")); String taskId = new IDGenerator().generate(); WorkflowDef wd = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(wd); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new JoinTaskMapper().getMappedTasks(taskMapperContext); assertNotNull(mappedTasks); assertEquals(TASK_TYPE_JOIN, mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/JsonJQTransformTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; public class JsonJQTransformTaskMapperTest { private IDGenerator idGenerator; private ParametersUtils parametersUtils; private MetadataDAO metadataDAO; @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); metadataDAO = mock(MetadataDAO.class); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("json_jq_transform_task"); workflowTask.setType(TaskType.JSON_JQ_TRANSFORM.name()); workflowTask.setTaskDefinition(new TaskDef("json_jq_transform_task")); Map taskInput = new HashMap<>(); taskInput.put("in1", new String[] {"a", "b"}); taskInput.put("in2", new String[] {"c", "d"}); taskInput.put("queryExpression", "{ out: (.in1 + .in2) }"); workflowTask.setInputParameters(taskInput); String taskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(taskInput) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new JsonJQTransformTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.JSON_JQ_TRANSFORM.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasks_WithoutTaskDef() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("json_jq_transform_task"); workflowTask.setType(TaskType.JSON_JQ_TRANSFORM.name()); Map taskInput = new HashMap<>(); taskInput.put("in1", new String[] {"a", "b"}); taskInput.put("in2", new String[] {"c", "d"}); taskInput.put("queryExpression", "{ out: (.in1 + .in2) }"); workflowTask.setInputParameters(taskInput); String taskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(null) .withWorkflowTask(workflowTask) .withTaskInput(taskInput) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new JsonJQTransformTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.JSON_JQ_TRANSFORM.name(), mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/KafkaPublishTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class KafkaPublishTaskMapperTest { private IDGenerator idGenerator; private KafkaPublishTaskMapper kafkaTaskMapper; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { ParametersUtils parametersUtils = mock(ParametersUtils.class); MetadataDAO metadataDAO = mock(MetadataDAO.class); kafkaTaskMapper = new KafkaPublishTaskMapper(parametersUtils, metadataDAO); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("kafka_task"); workflowTask.setType(TaskType.KAFKA_PUBLISH.name()); workflowTask.setTaskDefinition(new TaskDef("kafka_task")); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // when List mappedTasks = kafkaTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TaskType.KAFKA_PUBLISH.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasks_WithoutTaskDef() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("kafka_task"); workflowTask.setType(TaskType.KAFKA_PUBLISH.name()); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskDef taskdefinition = new TaskDef(); String testExecutionNameSpace = "testExecutionNameSpace"; taskdefinition.setExecutionNameSpace(testExecutionNameSpace); String testIsolationGroupId = "testIsolationGroupId"; taskdefinition.setIsolationGroupId(testIsolationGroupId); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(taskdefinition) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // when List mappedTasks = kafkaTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TaskType.KAFKA_PUBLISH.name(), mappedTasks.get(0).getTaskType()); assertEquals(testExecutionNameSpace, mappedTasks.get(0).getExecutionNameSpace()); assertEquals(testIsolationGroupId, mappedTasks.get(0).getIsolationGroupId()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/LambdaTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; public class LambdaTaskMapperTest { private IDGenerator idGenerator; private ParametersUtils parametersUtils; private MetadataDAO metadataDAO; @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); metadataDAO = mock(MetadataDAO.class); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("lambda_task"); workflowTask.setType(TaskType.LAMBDA.name()); workflowTask.setTaskDefinition(new TaskDef("lambda_task")); workflowTask.setScriptExpression( "if ($.input.a==1){return {testValue: true}} else{return {testValue: false} }"); String taskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new LambdaTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.LAMBDA.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasks_WithoutTaskDef() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.LAMBDA.name()); workflowTask.setScriptExpression( "if ($.input.a==1){return {testValue: true}} else{return {testValue: false} }"); String taskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(null) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new LambdaTaskMapper(parametersUtils, metadataDAO) .getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertNotNull(mappedTasks); assertEquals(TaskType.LAMBDA.name(), mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/NoopTaskMapperTest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.junit.Assert; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; public class NoopTaskMapperTest { @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_NOOP); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new NoopTaskMapper().getMappedTasks(taskMapperContext); Assert.assertNotNull(mappedTasks); Assert.assertEquals(1, mappedTasks.size()); Assert.assertEquals(TaskType.TASK_TYPE_NOOP, mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/SetVariableTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.junit.Assert; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; public class SetVariableTaskMapperTest { @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_SET_VARIABLE); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new SetVariableTaskMapper().getMappedTasks(taskMapperContext); Assert.assertNotNull(mappedTasks); Assert.assertEquals(1, mappedTasks.size()); Assert.assertEquals(TaskType.TASK_TYPE_SET_VARIABLE, mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/SimpleTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; public class SimpleTaskMapperTest { private SimpleTaskMapper simpleTaskMapper; private IDGenerator idGenerator = new IDGenerator(); @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { ParametersUtils parametersUtils = mock(ParametersUtils.class); simpleTaskMapper = new SimpleTaskMapper(parametersUtils); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("simple_task"); workflowTask.setTaskDefinition(new TaskDef("simple_task")); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); List mappedTasks = simpleTaskMapper.getMappedTasks(taskMapperContext); assertNotNull(mappedTasks); assertEquals(1, mappedTasks.size()); } @Test public void getMappedTasksException() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("simple_task"); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // then expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( String.format( "Invalid task. Task %s does not have a definition", workflowTask.getName())); // when simpleTaskMapper.getMappedTasks(taskMapperContext); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SubWorkflowTaskMapperTest { private SubWorkflowTaskMapper subWorkflowTaskMapper; private ParametersUtils parametersUtils; private DeciderService deciderService; private IDGenerator idGenerator; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); MetadataDAO metadataDAO = mock(MetadataDAO.class); subWorkflowTaskMapper = new SubWorkflowTaskMapper(parametersUtils, metadataDAO); deciderService = mock(DeciderService.class); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { // Given WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); WorkflowTask workflowTask = new WorkflowTask(); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("Foo"); subWorkflowParams.setVersion(2); workflowTask.setSubWorkflowParam(subWorkflowParams); workflowTask.setStartDelay(30); Map taskInput = new HashMap<>(); Map taskToDomain = new HashMap<>() { { put("*", "unittest"); } }; Map subWorkflowParamMap = new HashMap<>(); subWorkflowParamMap.put("name", "FooWorkFlow"); subWorkflowParamMap.put("version", 2); subWorkflowParamMap.put("taskToDomain", taskToDomain); when(parametersUtils.getTaskInputV2(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(subWorkflowParamMap); // When TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(workflowTask) .withTaskInput(taskInput) .withRetryCount(0) .withTaskId(idGenerator.generate()) .withDeciderService(deciderService) .build(); List mappedTasks = subWorkflowTaskMapper.getMappedTasks(taskMapperContext); // Then assertFalse(mappedTasks.isEmpty()); assertEquals(1, mappedTasks.size()); TaskModel subWorkFlowTask = mappedTasks.get(0); assertEquals(TaskModel.Status.SCHEDULED, subWorkFlowTask.getStatus()); assertEquals(TASK_TYPE_SUB_WORKFLOW, subWorkFlowTask.getTaskType()); assertEquals(30, subWorkFlowTask.getCallbackAfterSeconds()); assertEquals(taskToDomain, subWorkFlowTask.getInputData().get("subWorkflowTaskToDomain")); } @Test public void testTaskToDomain() { // Given WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); WorkflowTask workflowTask = new WorkflowTask(); Map taskToDomain = new HashMap<>() { { put("*", "unittest"); } }; SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("Foo"); subWorkflowParams.setVersion(2); subWorkflowParams.setTaskToDomain(taskToDomain); workflowTask.setSubWorkflowParam(subWorkflowParams); Map taskInput = new HashMap<>(); Map subWorkflowParamMap = new HashMap<>(); subWorkflowParamMap.put("name", "FooWorkFlow"); subWorkflowParamMap.put("version", 2); when(parametersUtils.getTaskInputV2(anyMap(), any(WorkflowModel.class), any(), any())) .thenReturn(subWorkflowParamMap); // When TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(workflowTask) .withTaskInput(taskInput) .withRetryCount(0) .withTaskId(new IDGenerator().generate()) .withDeciderService(deciderService) .build(); List mappedTasks = subWorkflowTaskMapper.getMappedTasks(taskMapperContext); // Then assertFalse(mappedTasks.isEmpty()); assertEquals(1, mappedTasks.size()); TaskModel subWorkFlowTask = mappedTasks.get(0); assertEquals(TaskModel.Status.SCHEDULED, subWorkFlowTask.getStatus()); assertEquals(TASK_TYPE_SUB_WORKFLOW, subWorkFlowTask.getTaskType()); } @Test public void getSubWorkflowParams() { WorkflowTask workflowTask = new WorkflowTask(); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("Foo"); subWorkflowParams.setVersion(2); workflowTask.setSubWorkflowParam(subWorkflowParams); assertEquals(subWorkflowParams, subWorkflowTaskMapper.getSubWorkflowParams(workflowTask)); } @Test public void getExceptionWhenNoSubWorkflowParamsPassed() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("FooWorkFLow"); expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( String.format( "Task %s is defined as sub-workflow and is missing subWorkflowParams. " + "Please check the workflow definition", workflowTask.getName())); subWorkflowTaskMapper.getSubWorkflowParams(workflowTask); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/SwitchTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.execution.evaluators.JavascriptEvaluator; import com.netflix.conductor.core.execution.evaluators.ValueParamEvaluator; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, SwitchTaskMapperTest.TestConfiguration.class }) @RunWith(SpringRunner.class) public class SwitchTaskMapperTest { private IDGenerator idGenerator; private ParametersUtils parametersUtils; private DeciderService deciderService; // Subject private SwitchTaskMapper switchTaskMapper; @Configuration @ComponentScan(basePackageClasses = {Evaluator.class}) // load all Evaluator beans. public static class TestConfiguration {} @Autowired private ObjectMapper objectMapper; @Autowired private Map evaluators; @Rule public ExpectedException expectedException = ExpectedException.none(); Map ip1; WorkflowTask task1; WorkflowTask task2; WorkflowTask task3; @Before public void setUp() { parametersUtils = new ParametersUtils(objectMapper); idGenerator = new IDGenerator(); ip1 = new HashMap<>(); ip1.put("p1", "${workflow.input.param1}"); ip1.put("p2", "${workflow.input.param2}"); ip1.put("case", "${workflow.input.case}"); task1 = new WorkflowTask(); task1.setName("Test1"); task1.setInputParameters(ip1); task1.setTaskReferenceName("t1"); task2 = new WorkflowTask(); task2.setName("Test2"); task2.setInputParameters(ip1); task2.setTaskReferenceName("t2"); task3 = new WorkflowTask(); task3.setName("Test3"); task3.setInputParameters(ip1); task3.setTaskReferenceName("t3"); deciderService = mock(DeciderService.class); switchTaskMapper = new SwitchTaskMapper(evaluators); } @Test public void getMappedTasks() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("Id", "${workflow.input.Id}"); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Switch task instance WorkflowTask switchTask = new WorkflowTask(); switchTask.setType(TaskType.SWITCH.name()); switchTask.setName("Switch"); switchTask.setTaskReferenceName("switchTask"); switchTask.setDefaultCase(Collections.singletonList(task1)); switchTask.getInputParameters().put("Id", "${workflow.input.Id}"); switchTask.setEvaluatorType(JavascriptEvaluator.NAME); switchTask.setExpression( "if ($.Id == null) 'bad input'; else if ( ($.Id != null && $.Id % 2 == 0)) 'even'; else 'odd'; "); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); decisionCases.put("odd", Collections.singletonList(task3)); switchTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); Map workflowInput = new HashMap<>(); workflowInput.put("Id", "22"); workflowModel.setInput(workflowInput); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map input = parametersUtils.getTaskInput( switchTask.getInputParameters(), workflowModel, null, null); TaskModel theTask = new TaskModel(); theTask.setReferenceTaskName("Foo"); theTask.setTaskId(idGenerator.generate()); when(deciderService.getTasksToBeScheduled(workflowModel, task2, 0, null)) .thenReturn(Collections.singletonList(theTask)); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(switchTask) .withTaskInput(input) .withRetryCount(0) .withTaskId(idGenerator.generate()) .withDeciderService(deciderService) .build(); // When List mappedTasks = switchTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(2, mappedTasks.size()); assertEquals("switchTask", mappedTasks.get(0).getReferenceTaskName()); assertEquals("Foo", mappedTasks.get(1).getReferenceTaskName()); } @Test public void getMappedTasksWithValueParamEvaluator() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); inputMap.put("Id", "${workflow.input.Id}"); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Switch task instance WorkflowTask switchTask = new WorkflowTask(); switchTask.setType(TaskType.SWITCH.name()); switchTask.setName("Switch"); switchTask.setTaskReferenceName("switchTask"); switchTask.setDefaultCase(Collections.singletonList(task1)); switchTask.getInputParameters().put("Id", "${workflow.input.Id}"); switchTask.setEvaluatorType(ValueParamEvaluator.NAME); switchTask.setExpression("Id"); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); decisionCases.put("odd", Collections.singletonList(task3)); switchTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); Map workflowInput = new HashMap<>(); workflowInput.put("Id", "even"); workflowModel.setInput(workflowInput); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map input = parametersUtils.getTaskInput( switchTask.getInputParameters(), workflowModel, null, null); TaskModel theTask = new TaskModel(); theTask.setReferenceTaskName("Foo"); theTask.setTaskId(idGenerator.generate()); when(deciderService.getTasksToBeScheduled(workflowModel, task2, 0, null)) .thenReturn(Collections.singletonList(theTask)); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(switchTask) .withTaskInput(input) .withRetryCount(0) .withTaskId(idGenerator.generate()) .withDeciderService(deciderService) .build(); // When List mappedTasks = switchTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(2, mappedTasks.size()); assertEquals("switchTask", mappedTasks.get(0).getReferenceTaskName()); assertEquals("Foo", mappedTasks.get(1).getReferenceTaskName()); } @Test public void getMappedTasksWhenEvaluatorThrowsException() { // Given // Task Definition TaskDef taskDef = new TaskDef(); Map inputMap = new HashMap<>(); List> taskDefinitionInput = new LinkedList<>(); taskDefinitionInput.add(inputMap); // Switch task instance WorkflowTask switchTask = new WorkflowTask(); switchTask.setType(TaskType.SWITCH.name()); switchTask.setName("Switch"); switchTask.setTaskReferenceName("switchTask"); switchTask.setDefaultCase(Collections.singletonList(task1)); switchTask.setEvaluatorType(JavascriptEvaluator.NAME); switchTask.setExpression("undefinedVariable"); Map> decisionCases = new HashMap<>(); decisionCases.put("even", Collections.singletonList(task2)); switchTask.setDecisionCases(decisionCases); // Workflow instance WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setSchemaVersion(2); WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowDefinition(workflowDef); Map body = new HashMap<>(); body.put("input", taskDefinitionInput); taskDef.getInputTemplate().putAll(body); Map input = parametersUtils.getTaskInput( switchTask.getInputParameters(), workflowModel, null, null); TaskModel theTask = new TaskModel(); theTask.setReferenceTaskName("Foo"); theTask.setTaskId(idGenerator.generate()); when(deciderService.getTasksToBeScheduled(workflowModel, task2, 0, null)) .thenReturn(Collections.singletonList(theTask)); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflowModel) .withWorkflowTask(switchTask) .withTaskInput(input) .withRetryCount(0) .withTaskId(idGenerator.generate()) .withDeciderService(deciderService) .build(); // When List mappedTasks = switchTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals("switchTask", mappedTasks.get(0).getReferenceTaskName()); assertEquals(TaskModel.Status.FAILED, mappedTasks.get(0).getStatus()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/TerminateTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.List; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.mockito.Mockito.mock; public class TerminateTaskMapperTest { private ParametersUtils parametersUtils; @Before public void setUp() { parametersUtils = mock(ParametersUtils.class); } @Test public void getMappedTasks() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_TERMINATE); String taskId = new IDGenerator().generate(); WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withRetryCount(0) .withTaskId(taskId) .build(); List mappedTasks = new TerminateTaskMapper(parametersUtils).getMappedTasks(taskMapperContext); Assert.assertNotNull(mappedTasks); Assert.assertEquals(1, mappedTasks.size()); Assert.assertEquals(TaskType.TASK_TYPE_TERMINATE, mappedTasks.get(0).getTaskType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/UserDefinedTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.util.HashMap; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class UserDefinedTaskMapperTest { private IDGenerator idGenerator; private UserDefinedTaskMapper userDefinedTaskMapper; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setUp() { ParametersUtils parametersUtils = mock(ParametersUtils.class); MetadataDAO metadataDAO = mock(MetadataDAO.class); userDefinedTaskMapper = new UserDefinedTaskMapper(parametersUtils, metadataDAO); idGenerator = new IDGenerator(); } @Test public void getMappedTasks() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("user_task"); workflowTask.setType(TaskType.USER_DEFINED.name()); workflowTask.setTaskDefinition(new TaskDef("user_task")); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // when List mappedTasks = userDefinedTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TaskType.USER_DEFINED.name(), mappedTasks.get(0).getTaskType()); } @Test public void getMappedTasksException() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("user_task"); workflowTask.setType(TaskType.USER_DEFINED.name()); String taskId = idGenerator.generate(); String retriedTaskId = idGenerator.generate(); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withRetryTaskId(retriedTaskId) .withTaskId(taskId) .build(); // then expectedException.expect(TerminateWorkflowException.class); expectedException.expectMessage( String.format( "Invalid task specified. Cannot find task by name %s in the task definitions", workflowTask.getName())); // when userDefinedTaskMapper.getMappedTasks(taskMapperContext); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/mapper/WaitTaskMapperTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.mapper; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.tasks.Wait; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; public class WaitTaskMapperTest { @Test public void getMappedTasks() { // Given WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Wait_task"); workflowTask.setType(TaskType.WAIT.name()); String taskId = new IDGenerator().generate(); ParametersUtils parametersUtils = mock(ParametersUtils.class); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withTaskId(taskId) .build(); WaitTaskMapper waitTaskMapper = new WaitTaskMapper(parametersUtils); // When List mappedTasks = waitTaskMapper.getMappedTasks(taskMapperContext); // Then assertEquals(1, mappedTasks.size()); assertEquals(TASK_TYPE_WAIT, mappedTasks.get(0).getTaskType()); } @Test public void testWaitForever() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Wait_task"); workflowTask.setType(TaskType.WAIT.name()); String taskId = new IDGenerator().generate(); ParametersUtils parametersUtils = mock(ParametersUtils.class); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(new HashMap<>()) .withRetryCount(0) .withTaskId(taskId) .build(); WaitTaskMapper waitTaskMapper = new WaitTaskMapper(parametersUtils); // When List mappedTasks = waitTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertEquals(mappedTasks.get(0).getStatus(), TaskModel.Status.IN_PROGRESS); assertTrue(mappedTasks.get(0).getOutputData().isEmpty()); } @Test public void testWaitUntil() { String dateFormat = "yyyy-MM-dd HH:mm"; DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormat); LocalDateTime now = LocalDateTime.now(); String formatted = formatter.format(now); System.out.println(formatted); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Wait_task"); workflowTask.setType(TaskType.WAIT.name()); String taskId = new IDGenerator().generate(); Map input = Map.of(Wait.UNTIL_INPUT, formatted); workflowTask.setInputParameters(input); ParametersUtils parametersUtils = mock(ParametersUtils.class); doReturn(input).when(parametersUtils).getTaskInputV2(any(), any(), any(), any()); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(Map.of(Wait.UNTIL_INPUT, formatted)) .withRetryCount(0) .withTaskId(taskId) .build(); WaitTaskMapper waitTaskMapper = new WaitTaskMapper(parametersUtils); // When List mappedTasks = waitTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertEquals(mappedTasks.get(0).getStatus(), TaskModel.Status.IN_PROGRESS); assertEquals(mappedTasks.get(0).getCallbackAfterSeconds(), 0L); } @Test public void testWaitDuration() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Wait_task"); workflowTask.setType(TaskType.WAIT.name()); String taskId = new IDGenerator().generate(); Map input = Map.of(Wait.DURATION_INPUT, "1s"); workflowTask.setInputParameters(input); ParametersUtils parametersUtils = mock(ParametersUtils.class); doReturn(input).when(parametersUtils).getTaskInputV2(any(), any(), any(), any()); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput(Map.of(Wait.DURATION_INPUT, "1s")) .withRetryCount(0) .withTaskId(taskId) .build(); WaitTaskMapper waitTaskMapper = new WaitTaskMapper(parametersUtils); // When List mappedTasks = waitTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertEquals(mappedTasks.get(0).getStatus(), TaskModel.Status.IN_PROGRESS); assertTrue(mappedTasks.get(0).getCallbackAfterSeconds() <= 1L); } @Test public void testInvalidWaitConfig() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("Wait_task"); workflowTask.setType(TaskType.WAIT.name()); String taskId = new IDGenerator().generate(); Map input = Map.of(Wait.DURATION_INPUT, "1s", Wait.UNTIL_INPUT, "2022-12-12"); workflowTask.setInputParameters(input); ParametersUtils parametersUtils = mock(ParametersUtils.class); doReturn(input).when(parametersUtils).getTaskInputV2(any(), any(), any(), any()); WorkflowModel workflow = new WorkflowModel(); WorkflowDef workflowDef = new WorkflowDef(); workflow.setWorkflowDefinition(workflowDef); TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder() .withWorkflowModel(workflow) .withTaskDefinition(new TaskDef()) .withWorkflowTask(workflowTask) .withTaskInput( Map.of(Wait.DURATION_INPUT, "1s", Wait.UNTIL_INPUT, "2022-12-12")) .withRetryCount(0) .withTaskId(taskId) .build(); WaitTaskMapper waitTaskMapper = new WaitTaskMapper(parametersUtils); // When List mappedTasks = waitTaskMapper.getMappedTasks(taskMapperContext); assertEquals(1, mappedTasks.size()); assertEquals(mappedTasks.get(0).getStatus(), TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/EventQueueResolutionTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.events.EventQueueProvider; import com.netflix.conductor.core.events.EventQueues; import com.netflix.conductor.core.events.MockQueueProvider; import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; /** * Tests the {@link Event#computeQueueName(WorkflowModel, TaskModel)} and {@link * Event#getQueue(String, String)} methods with a real {@link ParametersUtils} object. */ @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class EventQueueResolutionTest { private WorkflowDef testWorkflowDefinition; private EventQueues eventQueues; private ParametersUtils parametersUtils; @Autowired private ObjectMapper objectMapper; @Before public void setup() { Map providers = new HashMap<>(); providers.put("sqs", new MockQueueProvider("sqs")); providers.put("conductor", new MockQueueProvider("conductor")); parametersUtils = new ParametersUtils(objectMapper); eventQueues = new EventQueues(providers, parametersUtils); testWorkflowDefinition = new WorkflowDef(); testWorkflowDefinition.setName("testWorkflow"); testWorkflowDefinition.setVersion(2); } @Test public void testSinkParam() { String sink = "sqs:queue_name"; WorkflowDef def = new WorkflowDef(); def.setName("wf0"); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); TaskModel task1 = new TaskModel(); task1.setReferenceTaskName("t1"); task1.addOutput("q", "t1_queue"); workflow.getTasks().add(task1); TaskModel task2 = new TaskModel(); task2.setReferenceTaskName("t2"); task2.addOutput("q", "task2_queue"); workflow.getTasks().add(task2); TaskModel task = new TaskModel(); task.setReferenceTaskName("event"); task.getInputData().put("sink", sink); task.setTaskType(TaskType.EVENT.name()); workflow.getTasks().add(task); Event event = new Event(eventQueues, parametersUtils, objectMapper); String queueName = event.computeQueueName(workflow, task); ObservableQueue queue = event.getQueue(queueName, task.getTaskId()); assertNotNull(task.getReasonForIncompletion(), queue); assertEquals("queue_name", queue.getName()); assertEquals("sqs", queue.getType()); sink = "sqs:${t1.output.q}"; task.getInputData().put("sink", sink); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertNotNull(queue); assertEquals("t1_queue", queue.getName()); assertEquals("sqs", queue.getType()); sink = "sqs:${t2.output.q}"; task.getInputData().put("sink", sink); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertNotNull(queue); assertEquals("task2_queue", queue.getName()); assertEquals("sqs", queue.getType()); sink = "conductor"; task.getInputData().put("sink", sink); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertNotNull(queue); assertEquals( workflow.getWorkflowName() + ":" + task.getReferenceTaskName(), queue.getName()); assertEquals("conductor", queue.getType()); sink = "sqs:static_value"; task.getInputData().put("sink", sink); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertNotNull(queue); assertEquals("static_value", queue.getName()); assertEquals("sqs", queue.getType()); } @Test public void testDynamicSinks() { Event event = new Event(eventQueues, parametersUtils, objectMapper); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(testWorkflowDefinition); TaskModel task = new TaskModel(); task.setReferenceTaskName("task0"); task.setTaskId("task_id_0"); task.setStatus(TaskModel.Status.IN_PROGRESS); task.getInputData().put("sink", "conductor:some_arbitary_queue"); String queueName = event.computeQueueName(workflow, task); ObservableQueue queue = event.getQueue(queueName, task.getTaskId()); assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertNotNull(queue); assertEquals("testWorkflow:some_arbitary_queue", queue.getName()); assertEquals("testWorkflow:some_arbitary_queue", queue.getURI()); assertEquals("conductor", queue.getType()); task.getInputData().put("sink", "conductor"); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertEquals( "not in progress: " + task.getReasonForIncompletion(), TaskModel.Status.IN_PROGRESS, task.getStatus()); assertNotNull(queue); assertEquals("testWorkflow:task0", queue.getName()); task.getInputData().put("sink", "sqs:my_sqs_queue_name"); queueName = event.computeQueueName(workflow, task); queue = event.getQueue(queueName, task.getTaskId()); assertEquals( "not in progress: " + task.getReasonForIncompletion(), TaskModel.Status.IN_PROGRESS, task.getStatus()); assertNotNull(queue); assertEquals("my_sqs_queue_name", queue.getName()); assertEquals("sqs", queue.getType()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/InlineTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import org.junit.Test; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.execution.evaluators.JavascriptEvaluator; import com.netflix.conductor.core.execution.evaluators.ValueParamEvaluator; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; public class InlineTest { private final WorkflowModel workflow = new WorkflowModel(); private final WorkflowExecutor executor = mock(WorkflowExecutor.class); @Test public void testInlineTaskValidationFailures() { Inline inline = new Inline(getStringEvaluatorMap()); Map inputObj = new HashMap<>(); inputObj.put("value", 1); inputObj.put("expression", ""); inputObj.put("evaluatorType", "value-param"); TaskModel task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR, task.getStatus()); assertEquals( "Empty 'expression' in Inline task's input parameters. A non-empty String value must be provided.", task.getReasonForIncompletion()); inputObj = new HashMap<>(); inputObj.put("value", 1); inputObj.put("expression", "value"); inputObj.put("evaluatorType", ""); task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR, task.getStatus()); assertEquals( "Empty 'evaluatorType' in INLINE task's input parameters. A non-empty String value must be provided.", task.getReasonForIncompletion()); } @Test public void testInlineValueParamExpression() { Inline inline = new Inline(getStringEvaluatorMap()); Map inputObj = new HashMap<>(); inputObj.put("value", 101); inputObj.put("expression", "value"); inputObj.put("evaluatorType", "value-param"); TaskModel task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals(101, task.getOutputData().get("result")); inputObj = new HashMap<>(); inputObj.put("value", "StringValue"); inputObj.put("expression", "value"); inputObj.put("evaluatorType", "value-param"); task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals("StringValue", task.getOutputData().get("result")); } @SuppressWarnings("unchecked") @Test public void testInlineJavascriptExpression() { Inline inline = new Inline(getStringEvaluatorMap()); Map inputObj = new HashMap<>(); inputObj.put("value", 101); inputObj.put( "expression", "function e() { if ($.value == 101){return {\"evalResult\": true}} else { return {\"evalResult\": false}}} e();"); inputObj.put("evaluatorType", "javascript"); TaskModel task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals( true, ((Map) task.getOutputData().get("result")).get("evalResult")); inputObj = new HashMap<>(); inputObj.put("value", "StringValue"); inputObj.put( "expression", "function e() { if ($.value == 'StringValue'){return {\"evalResult\": true}} else { return {\"evalResult\": false}}} e();"); inputObj.put("evaluatorType", "javascript"); task = new TaskModel(); task.getInputData().putAll(inputObj); inline.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull(task.getReasonForIncompletion()); assertEquals( true, ((Map) task.getOutputData().get("result")).get("evalResult")); } private Map getStringEvaluatorMap() { Map evaluators = new HashMap<>(); evaluators.put(ValueParamEvaluator.NAME, new ValueParamEvaluator()); evaluators.put(JavascriptEvaluator.NAME, new JavascriptEvaluator()); return evaluators; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestLambda.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import org.junit.Test; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; /** * @author x-ultra */ public class TestLambda { private final WorkflowModel workflow = new WorkflowModel(); private final WorkflowExecutor executor = mock(WorkflowExecutor.class); @SuppressWarnings({"rawtypes", "unchecked"}) @Test public void start() { Lambda lambda = new Lambda(); Map inputObj = new HashMap(); inputObj.put("a", 1); // test for scriptExpression == null TaskModel task = new TaskModel(); task.getInputData().put("input", inputObj); lambda.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); // test for normal task = new TaskModel(); task.getInputData().put("input", inputObj); task.getInputData().put("scriptExpression", "if ($.input.a==1){return 1}else{return 0 } "); lambda.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(task.getOutputData().toString(), "{result=1}"); // test for scriptExpression ScriptException task = new TaskModel(); task.getInputData().put("input", inputObj); task.getInputData().put("scriptExpression", "if ($.a.size==1){return 1}else{return 0 } "); lambda.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestNoop.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import org.junit.Test; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.*; import static org.mockito.Mockito.*; public class TestNoop { private final WorkflowExecutor executor = mock(WorkflowExecutor.class); @Test public void should_do_nothing() { WorkflowModel workflow = new WorkflowModel(); Noop noopTask = new Noop(); TaskModel task = new TaskModel(); noopTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestSubWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.HashMap; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.execution.StartWorkflowInput; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class TestSubWorkflow { private WorkflowExecutor workflowExecutor; private SubWorkflow subWorkflow; private StartWorkflowOperation startWorkflowOperation; @Autowired private ObjectMapper objectMapper; @Before public void setup() { workflowExecutor = mock(WorkflowExecutor.class); startWorkflowOperation = mock(StartWorkflowOperation.class); subWorkflow = new SubWorkflow(objectMapper, startWorkflowOperation); } @Test public void testStartSubWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 3); task.setInputData(inputData); String workflowId = "workflow_1"; WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId(workflowId); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(3); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn(workflowId); when(workflowExecutor.getWorkflow(anyString(), eq(false))).thenReturn(workflow); workflow.setStatus(WorkflowModel.Status.RUNNING); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); workflow.setStatus(WorkflowModel.Status.TERMINATED); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); assertEquals(TaskModel.Status.CANCELED, task.getStatus()); workflow.setStatus(WorkflowModel.Status.COMPLETED); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); } @Test public void testStartSubWorkflowQueueFailure() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); task.setStatus(TaskModel.Status.SCHEDULED); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 3); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(3); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)) .thenThrow(new TransientException("QueueDAO failure")); subWorkflow.start(workflowInstance, task, workflowExecutor); assertNull("subWorkflowId should be null", task.getSubWorkflowId()); assertEquals(TaskModel.Status.SCHEDULED, task.getStatus()); assertTrue("Output data should be empty", task.getOutputData().isEmpty()); } @Test public void testStartSubWorkflowStartError() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); task.setStatus(TaskModel.Status.SCHEDULED); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 3); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(3); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); String failureReason = "non transient failure"; when(startWorkflowOperation.execute(startWorkflowInput)) .thenThrow(new NonTransientException(failureReason)); subWorkflow.start(workflowInstance, task, workflowExecutor); assertNull("subWorkflowId should be null", task.getSubWorkflowId()); assertEquals(TaskModel.Status.FAILED, task.getStatus()); assertEquals(failureReason, task.getReasonForIncompletion()); assertTrue("Output data should be empty", task.getOutputData().isEmpty()); } @Test public void testStartSubWorkflowWithEmptyWorkflowInput() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 3); Map workflowInput = new HashMap<>(); inputData.put("workflowInput", workflowInput); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(3); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); } @Test public void testStartSubWorkflowWithWorkflowInput() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 3); Map workflowInput = new HashMap<>(); workflowInput.put("test", "value"); inputData.put("workflowInput", workflowInput); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(3); startWorkflowInput.setWorkflowInput(workflowInput); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); } @Test public void testStartSubWorkflowTaskToDomain() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); Map taskToDomain = new HashMap<>() { { put("*", "unittest"); } }; TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); inputData.put("subWorkflowTaskToDomain", taskToDomain); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(taskToDomain); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); } @Test public void testExecuteSubWorkflowWithoutId() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); assertFalse(subWorkflow.execute(workflowInstance, task, workflowExecutor)); } @Test public void testExecuteWorkflowStatus() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); WorkflowModel subWorkflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); Map taskToDomain = new HashMap<>() { { put("*", "unittest"); } }; TaskModel task = new TaskModel(); Map outputData = new HashMap<>(); task.setOutputData(outputData); task.setSubWorkflowId("sub-workflow-id"); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); inputData.put("subWorkflowTaskToDomain", taskToDomain); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(taskToDomain); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); when(workflowExecutor.getWorkflow(eq("sub-workflow-id"), eq(false))) .thenReturn(subWorkflowInstance); subWorkflowInstance.setStatus(WorkflowModel.Status.RUNNING); assertFalse(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertNull(task.getStatus()); assertNull(task.getReasonForIncompletion()); subWorkflowInstance.setStatus(WorkflowModel.Status.PAUSED); assertFalse(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertNull(task.getStatus()); assertNull(task.getReasonForIncompletion()); subWorkflowInstance.setStatus(WorkflowModel.Status.COMPLETED); assertTrue(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull(task.getReasonForIncompletion()); subWorkflowInstance.setStatus(WorkflowModel.Status.FAILED); subWorkflowInstance.setReasonForIncompletion("unit1"); assertTrue(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertEquals(TaskModel.Status.FAILED, task.getStatus()); assertTrue(task.getReasonForIncompletion().contains("unit1")); subWorkflowInstance.setStatus(WorkflowModel.Status.TIMED_OUT); subWorkflowInstance.setReasonForIncompletion("unit2"); assertTrue(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertEquals(TaskModel.Status.TIMED_OUT, task.getStatus()); assertTrue(task.getReasonForIncompletion().contains("unit2")); subWorkflowInstance.setStatus(WorkflowModel.Status.TERMINATED); subWorkflowInstance.setReasonForIncompletion("unit3"); assertTrue(subWorkflow.execute(workflowInstance, task, workflowExecutor)); assertEquals(TaskModel.Status.CANCELED, task.getStatus()); assertTrue(task.getReasonForIncompletion().contains("unit3")); } @Test public void testCancelWithWorkflowId() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); WorkflowModel subWorkflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); task.setSubWorkflowId("sub-workflow-id"); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); when(workflowExecutor.getWorkflow(eq("sub-workflow-id"), eq(true))) .thenReturn(subWorkflowInstance); workflowInstance.setStatus(WorkflowModel.Status.TIMED_OUT); subWorkflow.cancel(workflowInstance, task, workflowExecutor); assertEquals(WorkflowModel.Status.TERMINATED, subWorkflowInstance.getStatus()); } @Test public void testCancelWithoutWorkflowId() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); WorkflowModel subWorkflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); TaskModel task = new TaskModel(); Map outputData = new HashMap<>(); task.setOutputData(outputData); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("UnitWorkFlow"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); when(workflowExecutor.getWorkflow(eq("sub-workflow-id"), eq(false))) .thenReturn(subWorkflowInstance); subWorkflow.cancel(workflowInstance, task, workflowExecutor); assertEquals(WorkflowModel.Status.RUNNING, subWorkflowInstance.getStatus()); } @Test public void testIsAsync() { assertTrue(subWorkflow.isAsync()); } @Test public void testStartSubWorkflowWithSubWorkflowDefinition() { WorkflowDef workflowDef = new WorkflowDef(); WorkflowModel workflowInstance = new WorkflowModel(); workflowInstance.setWorkflowDefinition(workflowDef); WorkflowDef subWorkflowDef = new WorkflowDef(); subWorkflowDef.setName("subWorkflow_1"); TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); Map inputData = new HashMap<>(); inputData.put("subWorkflowName", "UnitWorkFlow"); inputData.put("subWorkflowVersion", 2); inputData.put("subWorkflowDefinition", subWorkflowDef); task.setInputData(inputData); StartWorkflowInput startWorkflowInput = new StartWorkflowInput(); startWorkflowInput.setName("subWorkflow_1"); startWorkflowInput.setVersion(2); startWorkflowInput.setWorkflowInput(inputData); startWorkflowInput.setWorkflowDefinition(subWorkflowDef); startWorkflowInput.setTaskToDomain(workflowInstance.getTaskToDomain()); when(startWorkflowOperation.execute(startWorkflowInput)).thenReturn("workflow_1"); subWorkflow.start(workflowInstance, task, workflowExecutor); assertEquals("workflow_1", task.getSubWorkflowId()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestSystemTaskWorker.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.service.ExecutionService; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestSystemTaskWorker { private static final String TEST_TASK = "system_task"; private static final String ISOLATED_TASK = "system_task-isolated"; private AsyncSystemTaskExecutor asyncSystemTaskExecutor; private ExecutionService executionService; private QueueDAO queueDAO; private ConductorProperties properties; private SystemTaskWorker systemTaskWorker; @Before public void setUp() { asyncSystemTaskExecutor = mock(AsyncSystemTaskExecutor.class); executionService = mock(ExecutionService.class); queueDAO = mock(QueueDAO.class); properties = mock(ConductorProperties.class); when(properties.getSystemTaskWorkerThreadCount()).thenReturn(10); when(properties.getIsolatedSystemTaskWorkerThreadCount()).thenReturn(10); when(properties.getSystemTaskWorkerCallbackDuration()).thenReturn(Duration.ofSeconds(30)); when(properties.getSystemTaskWorkerPollInterval()).thenReturn(Duration.ofSeconds(30)); systemTaskWorker = new SystemTaskWorker( queueDAO, asyncSystemTaskExecutor, properties, executionService); systemTaskWorker.start(); } @After public void tearDown() { systemTaskWorker.queueExecutionConfigMap.clear(); systemTaskWorker.stop(); } @Test public void testGetExecutionConfigForSystemTask() { when(properties.getSystemTaskWorkerThreadCount()).thenReturn(5); systemTaskWorker = new SystemTaskWorker( queueDAO, asyncSystemTaskExecutor, properties, executionService); assertEquals( systemTaskWorker.getExecutionConfig("").getSemaphoreUtil().availableSlots(), 5); } @Test public void testGetExecutionConfigForIsolatedSystemTask() { when(properties.getIsolatedSystemTaskWorkerThreadCount()).thenReturn(7); systemTaskWorker = new SystemTaskWorker( queueDAO, asyncSystemTaskExecutor, properties, executionService); assertEquals( systemTaskWorker.getExecutionConfig("test-iso").getSemaphoreUtil().availableSlots(), 7); } @Test public void testPollAndExecuteSystemTask() throws Exception { when(queueDAO.pop(anyString(), anyInt(), anyInt())) .thenReturn(Collections.singletonList("taskId")); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(asyncSystemTaskExecutor) .execute(any(), anyString()); systemTaskWorker.pollAndExecute(new TestTask(), TEST_TASK); latch.await(); verify(asyncSystemTaskExecutor).execute(any(), anyString()); } @Test public void testBatchPollAndExecuteSystemTask() throws Exception { when(queueDAO.pop(anyString(), anyInt(), anyInt())).thenReturn(List.of("t1", "t1")); CountDownLatch latch = new CountDownLatch(2); doAnswer( invocation -> { latch.countDown(); return null; }) .when(asyncSystemTaskExecutor) .execute(any(), eq("t1")); systemTaskWorker.pollAndExecute(new TestTask(), TEST_TASK); latch.await(); verify(asyncSystemTaskExecutor, Mockito.times(2)).execute(any(), eq("t1")); } @Test public void testPollAndExecuteIsolatedSystemTask() throws Exception { when(queueDAO.pop(anyString(), anyInt(), anyInt())).thenReturn(List.of("isolated_taskId")); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(asyncSystemTaskExecutor) .execute(any(), eq("isolated_taskId")); systemTaskWorker.pollAndExecute(new IsolatedTask(), ISOLATED_TASK); latch.await(); verify(asyncSystemTaskExecutor, Mockito.times(1)).execute(any(), eq("isolated_taskId")); } @Test public void testPollException() { when(properties.getSystemTaskWorkerThreadCount()).thenReturn(1); when(queueDAO.pop(anyString(), anyInt(), anyInt())).thenThrow(RuntimeException.class); systemTaskWorker.pollAndExecute(new TestTask(), TEST_TASK); verify(asyncSystemTaskExecutor, Mockito.never()).execute(any(), anyString()); } @Test public void testBatchPollException() { when(properties.getSystemTaskWorkerThreadCount()).thenReturn(2); when(queueDAO.pop(anyString(), anyInt(), anyInt())).thenThrow(RuntimeException.class); systemTaskWorker.pollAndExecute(new TestTask(), TEST_TASK); verify(asyncSystemTaskExecutor, Mockito.never()).execute(any(), anyString()); } static class TestTask extends WorkflowSystemTask { public TestTask() { super(TEST_TASK); } } static class IsolatedTask extends WorkflowSystemTask { public IsolatedTask() { super(ISOLATED_TASK); } } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestSystemTaskWorkerCoordinator.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.time.Duration; import java.util.Collections; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.core.config.ConductorProperties; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TestSystemTaskWorkerCoordinator { private static final String TEST_QUEUE = "test"; private static final String EXECUTION_NAMESPACE_CONSTANT = "@exeNS"; private SystemTaskWorker systemTaskWorker; private ConductorProperties properties; @Before public void setUp() { systemTaskWorker = mock(SystemTaskWorker.class); properties = mock(ConductorProperties.class); when(properties.getSystemTaskWorkerPollInterval()).thenReturn(Duration.ofMillis(50)); when(properties.getSystemTaskWorkerExecutionNamespace()).thenReturn(""); } @Test public void testIsFromCoordinatorExecutionNameSpace() { doReturn("exeNS").when(properties).getSystemTaskWorkerExecutionNamespace(); SystemTaskWorkerCoordinator systemTaskWorkerCoordinator = new SystemTaskWorkerCoordinator( systemTaskWorker, properties, Collections.emptySet()); assertTrue( systemTaskWorkerCoordinator.isFromCoordinatorExecutionNameSpace( new TaskWithExecutionNamespace())); } static class TaskWithExecutionNamespace extends WorkflowSystemTask { public TaskWithExecutionNamespace() { super(TEST_QUEUE + EXECUTION_NAMESPACE_CONSTANT); } } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/execution/tasks/TestTerminate.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.execution.tasks; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.junit.Test; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.core.execution.tasks.Terminate.getTerminationStatusParameter; import static com.netflix.conductor.core.execution.tasks.Terminate.getTerminationWorkflowOutputParameter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; public class TestTerminate { private final WorkflowExecutor executor = mock(WorkflowExecutor.class); @Test public void should_fail_if_input_status_is_not_valid() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), "PAUSED"); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); } @Test public void should_fail_if_input_status_is_empty() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), ""); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); } @Test public void should_fail_if_input_status_is_null() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), null); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); } @Test public void should_complete_workflow_on_terminate_task_success() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); workflow.setOutput(Collections.singletonMap("output", "${task1.output.value}")); HashMap expectedOutput = new HashMap<>() { { put("output", "${task0.output.value}"); } }; Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), "COMPLETED"); input.put(getTerminationWorkflowOutputParameter(), "${task0.output.value}"); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(expectedOutput, task.getOutputData()); } @Test public void should_fail_workflow_on_terminate_task_success() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); workflow.setOutput(Collections.singletonMap("output", "${task1.output.value}")); HashMap expectedOutput = new HashMap<>() { { put("output", "${task0.output.value}"); } }; Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), "FAILED"); input.put(getTerminationWorkflowOutputParameter(), "${task0.output.value}"); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(expectedOutput, task.getOutputData()); } @Test public void should_fail_workflow_on_terminate_task_success_with_empty_output() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), "FAILED"); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertTrue(task.getOutputData().isEmpty()); } @Test public void should_fail_workflow_on_terminate_task_success_with_resolved_output() { WorkflowModel workflow = new WorkflowModel(); Terminate terminateTask = new Terminate(); HashMap expectedOutput = new HashMap<>() { { put("result", 1); } }; Map input = new HashMap<>(); input.put(getTerminationStatusParameter(), "FAILED"); input.put(getTerminationWorkflowOutputParameter(), expectedOutput); TaskModel task = new TaskModel(); task.getInputData().putAll(input); terminateTask.execute(workflow, task, executor); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/metadata/MetadataMapperServiceTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.metadata; import java.util.List; import java.util.Optional; import java.util.Set; import javax.validation.ConstraintViolationException; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.dao.MetadataDAO; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class MetadataMapperServiceTest { @TestConfiguration static class TestMetadataMapperServiceConfiguration { @Bean public MetadataDAO metadataDAO() { return mock(MetadataDAO.class); } @Bean public MetadataMapperService metadataMapperService(MetadataDAO metadataDAO) { return new MetadataMapperService(metadataDAO); } } @Autowired private MetadataDAO metadataDAO; @Autowired private MetadataMapperService metadataMapperService; @After public void cleanUp() { reset(metadataDAO); } @Test public void testMetadataPopulationOnSimpleTask() { String nameTaskDefinition = "task1"; TaskDef taskDefinition = createTaskDefinition(nameTaskDefinition); WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); when(metadataDAO.getTaskDef(nameTaskDefinition)).thenReturn(taskDefinition); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(1, workflowDefinition.getTasks().size()); WorkflowTask populatedWorkflowTask = workflowDefinition.getTasks().get(0); assertNotNull(populatedWorkflowTask.getTaskDefinition()); verify(metadataDAO).getTaskDef(nameTaskDefinition); } @Test public void testNoMetadataPopulationOnEmbeddedTaskDefinition() { String nameTaskDefinition = "task2"; TaskDef taskDefinition = createTaskDefinition(nameTaskDefinition); WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); workflowTask.setTaskDefinition(taskDefinition); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(1, workflowDefinition.getTasks().size()); WorkflowTask populatedWorkflowTask = workflowDefinition.getTasks().get(0); assertNotNull(populatedWorkflowTask.getTaskDefinition()); verifyNoInteractions(metadataDAO); } @Test public void testMetadataPopulationOnlyOnNecessaryWorkflowTasks() { String nameTaskDefinition1 = "task4"; TaskDef taskDefinition = createTaskDefinition(nameTaskDefinition1); WorkflowTask workflowTask1 = createWorkflowTask(nameTaskDefinition1); workflowTask1.setTaskDefinition(taskDefinition); String nameTaskDefinition2 = "task5"; WorkflowTask workflowTask2 = createWorkflowTask(nameTaskDefinition2); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask1, workflowTask2)); when(metadataDAO.getTaskDef(nameTaskDefinition2)).thenReturn(taskDefinition); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(2, workflowDefinition.getTasks().size()); List workflowTasks = workflowDefinition.getTasks(); assertNotNull(workflowTasks.get(0).getTaskDefinition()); assertNotNull(workflowTasks.get(1).getTaskDefinition()); verify(metadataDAO).getTaskDef(nameTaskDefinition2); verifyNoMoreInteractions(metadataDAO); } @Test public void testMetadataPopulationMissingDefinitions() { String nameTaskDefinition1 = "task4"; WorkflowTask workflowTask1 = createWorkflowTask(nameTaskDefinition1); String nameTaskDefinition2 = "task5"; WorkflowTask workflowTask2 = createWorkflowTask(nameTaskDefinition2); TaskDef taskDefinition = createTaskDefinition(nameTaskDefinition1); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask1, workflowTask2)); when(metadataDAO.getTaskDef(nameTaskDefinition1)).thenReturn(taskDefinition); when(metadataDAO.getTaskDef(nameTaskDefinition2)).thenReturn(null); try { metadataMapperService.populateTaskDefinitions(workflowDefinition); } catch (NotFoundException nfe) { fail("Missing TaskDefinitions are not defaulted"); } } @Test public void testVersionPopulationForSubworkflowTaskIfVersionIsNotAvailable() { String nameTaskDefinition = "taskSubworkflow6"; String workflowDefinitionName = "subworkflow"; int version = 3; WorkflowDef subWorkflowDefinition = createWorkflowDefinition("workflowDefinitionName"); subWorkflowDefinition.setVersion(version); WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); workflowTask.setWorkflowTaskType(TaskType.SUB_WORKFLOW); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName(workflowDefinitionName); workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); when(metadataDAO.getLatestWorkflowDef(workflowDefinitionName)) .thenReturn(Optional.of(subWorkflowDefinition)); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(1, workflowDefinition.getTasks().size()); List workflowTasks = workflowDefinition.getTasks(); SubWorkflowParams params = workflowTasks.get(0).getSubWorkflowParam(); assertEquals(workflowDefinitionName, params.getName()); assertEquals(version, params.getVersion().intValue()); verify(metadataDAO).getLatestWorkflowDef(workflowDefinitionName); verify(metadataDAO).getTaskDef(nameTaskDefinition); verifyNoMoreInteractions(metadataDAO); } @Test public void testNoVersionPopulationForSubworkflowTaskIfAvailable() { String nameTaskDefinition = "taskSubworkflow7"; String workflowDefinitionName = "subworkflow"; Integer version = 2; WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); workflowTask.setWorkflowTaskType(TaskType.SUB_WORKFLOW); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName(workflowDefinitionName); subWorkflowParams.setVersion(version); workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(1, workflowDefinition.getTasks().size()); List workflowTasks = workflowDefinition.getTasks(); SubWorkflowParams params = workflowTasks.get(0).getSubWorkflowParam(); assertEquals(workflowDefinitionName, params.getName()); assertEquals(version, params.getVersion()); verify(metadataDAO).getTaskDef(nameTaskDefinition); verifyNoMoreInteractions(metadataDAO); } @Test(expected = TerminateWorkflowException.class) public void testExceptionWhenWorkflowDefinitionNotAvailable() { String nameTaskDefinition = "taskSubworkflow8"; String workflowDefinitionName = "subworkflow"; WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); workflowTask.setWorkflowTaskType(TaskType.SUB_WORKFLOW); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName(workflowDefinitionName); workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); when(metadataDAO.getLatestWorkflowDef(workflowDefinitionName)).thenReturn(Optional.empty()); metadataMapperService.populateTaskDefinitions(workflowDefinition); verify(metadataDAO).getLatestWorkflowDef(workflowDefinitionName); } @Test(expected = IllegalArgumentException.class) public void testLookupWorkflowDefinition() { try { String workflowName = "test"; when(metadataDAO.getWorkflowDef(workflowName, 0)) .thenReturn(Optional.of(new WorkflowDef())); Optional optionalWorkflowDef = metadataMapperService.lookupWorkflowDefinition(workflowName, 0); assertTrue(optionalWorkflowDef.isPresent()); metadataMapperService.lookupWorkflowDefinition(null, 0); } catch (ConstraintViolationException ex) { Assert.assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); } } @Test(expected = IllegalArgumentException.class) public void testLookupLatestWorkflowDefinition() { String workflowName = "test"; when(metadataDAO.getLatestWorkflowDef(workflowName)) .thenReturn(Optional.of(new WorkflowDef())); Optional optionalWorkflowDef = metadataMapperService.lookupLatestWorkflowDefinition(workflowName); assertTrue(optionalWorkflowDef.isPresent()); metadataMapperService.lookupLatestWorkflowDefinition(null); } @Test public void testShouldNotPopulateTaskDefinition() { WorkflowTask workflowTask = createWorkflowTask(""); assertFalse(metadataMapperService.shouldPopulateTaskDefinition(workflowTask)); } @Test public void testShouldPopulateTaskDefinition() { WorkflowTask workflowTask = createWorkflowTask("test"); assertTrue(metadataMapperService.shouldPopulateTaskDefinition(workflowTask)); } @Test public void testMetadataPopulationOnSimpleTaskDefMissing() { String nameTaskDefinition = "task1"; WorkflowTask workflowTask = createWorkflowTask(nameTaskDefinition); when(metadataDAO.getTaskDef(nameTaskDefinition)).thenReturn(null); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); assertEquals(1, workflowDefinition.getTasks().size()); WorkflowTask populatedWorkflowTask = workflowDefinition.getTasks().get(0); assertNotNull(populatedWorkflowTask.getTaskDefinition()); } private WorkflowDef createWorkflowDefinition(String name) { WorkflowDef workflowDefinition = new WorkflowDef(); workflowDefinition.setName(name); return workflowDefinition; } private WorkflowTask createWorkflowTask(String name) { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName(name); workflowTask.setType(TaskType.SIMPLE.name()); return workflowTask; } private TaskDef createTaskDefinition(String name) { return new TaskDef(name); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowRepairService.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.reconciliation; import java.time.Duration; import java.util.HashMap; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.EventQueues; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.*; import com.netflix.conductor.core.operation.StartWorkflowOperation; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.*; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestWorkflowRepairService { private QueueDAO queueDAO; private ExecutionDAO executionDAO; private ConductorProperties properties; private WorkflowRepairService workflowRepairService; private SystemTaskRegistry systemTaskRegistry; @Before public void setUp() { executionDAO = mock(ExecutionDAO.class); queueDAO = mock(QueueDAO.class); properties = mock(ConductorProperties.class); systemTaskRegistry = mock(SystemTaskRegistry.class); workflowRepairService = new WorkflowRepairService(executionDAO, queueDAO, properties, systemTaskRegistry); } @Test public void verifyAndRepairSimpleTaskInScheduledState() { TaskModel task = new TaskModel(); task.setTaskType("SIMPLE"); task.setStatus(TaskModel.Status.SCHEDULED); task.setTaskId("abcd"); task.setCallbackAfterSeconds(60); when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); assertTrue(workflowRepairService.verifyAndRepairTask(task)); // Verify that a new queue message is pushed for sync system tasks that fails queue contains // check. verify(queueDAO, times(1)).push(anyString(), anyString(), anyLong()); } @Test public void verifySimpleTaskInProgressState() { TaskModel task = new TaskModel(); task.setTaskType("SIMPLE"); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setTaskId("abcd"); task.setCallbackAfterSeconds(60); when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); assertFalse(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue message is never pushed for simple task in IN_PROGRESS state verify(queueDAO, never()).containsMessage(anyString(), anyString()); verify(queueDAO, never()).push(anyString(), anyString(), anyLong()); } @Test public void verifyAndRepairSystemTask() { String taskType = "TEST_SYS_TASK"; TaskModel task = new TaskModel(); task.setTaskType(taskType); task.setStatus(TaskModel.Status.SCHEDULED); task.setTaskId("abcd"); task.setCallbackAfterSeconds(60); when(systemTaskRegistry.isSystemTask("TEST_SYS_TASK")).thenReturn(true); when(systemTaskRegistry.get(taskType)) .thenReturn( new WorkflowSystemTask("TEST_SYS_TASK") { @Override public boolean isAsync() { return true; } @Override public boolean isAsyncComplete(TaskModel task) { return false; } @Override public void start( WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { super.start(workflow, task, executor); } }); when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); assertTrue(workflowRepairService.verifyAndRepairTask(task)); // Verify that a new queue message is pushed for tasks that fails queue contains check. verify(queueDAO, times(1)).push(anyString(), anyString(), anyLong()); // Verify a system task in IN_PROGRESS state can be recovered. reset(queueDAO); task.setStatus(TaskModel.Status.IN_PROGRESS); assertTrue(workflowRepairService.verifyAndRepairTask(task)); // Verify that a new queue message is pushed for async System task in IN_PROGRESS state that // fails queue contains check. verify(queueDAO, times(1)).push(anyString(), anyString(), anyLong()); } @Test public void assertSyncSystemTasksAreNotCheckedAgainstQueue() { // Return a Switch task object to init WorkflowSystemTask registry. when(systemTaskRegistry.get(TASK_TYPE_DECISION)).thenReturn(new Decision()); when(systemTaskRegistry.isSystemTask(TASK_TYPE_DECISION)).thenReturn(true); when(systemTaskRegistry.get(TASK_TYPE_SWITCH)).thenReturn(new Switch()); when(systemTaskRegistry.isSystemTask(TASK_TYPE_SWITCH)).thenReturn(true); TaskModel task = new TaskModel(); task.setTaskType(TASK_TYPE_DECISION); task.setStatus(TaskModel.Status.SCHEDULED); assertFalse(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue contains is never checked for sync system tasks verify(queueDAO, never()).containsMessage(anyString(), anyString()); // Verify that queue message is never pushed for sync system tasks verify(queueDAO, never()).push(anyString(), anyString(), anyLong()); task = new TaskModel(); task.setTaskType(TASK_TYPE_SWITCH); task.setStatus(TaskModel.Status.SCHEDULED); assertFalse(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue contains is never checked for sync system tasks verify(queueDAO, never()).containsMessage(anyString(), anyString()); // Verify that queue message is never pushed for sync system tasks verify(queueDAO, never()).push(anyString(), anyString(), anyLong()); } @Test public void assertAsyncCompleteInProgressSystemTasksAreNotCheckedAgainstQueue() { TaskModel task = new TaskModel(); task.setTaskType(TASK_TYPE_EVENT); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setTaskId("abcd"); task.setCallbackAfterSeconds(60); task.setInputData(Map.of("asyncComplete", true)); WorkflowSystemTask workflowSystemTask = new Event( mock(EventQueues.class), mock(ParametersUtils.class), mock(ObjectMapper.class)); when(systemTaskRegistry.get(TASK_TYPE_EVENT)).thenReturn(workflowSystemTask); assertTrue(workflowSystemTask.isAsyncComplete(task)); assertFalse(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue message is never pushed for async complete system tasks verify(queueDAO, never()).containsMessage(anyString(), anyString()); verify(queueDAO, never()).push(anyString(), anyString(), anyLong()); } @Test public void assertAsyncCompleteScheduledSystemTasksAreCheckedAgainstQueue() { TaskModel task = new TaskModel(); task.setTaskType(TASK_TYPE_SUB_WORKFLOW); task.setStatus(TaskModel.Status.SCHEDULED); task.setTaskId("abcd"); task.setCallbackAfterSeconds(60); WorkflowSystemTask workflowSystemTask = new SubWorkflow(new ObjectMapper(), mock(StartWorkflowOperation.class)); when(systemTaskRegistry.get(TASK_TYPE_SUB_WORKFLOW)).thenReturn(workflowSystemTask); when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); assertTrue(workflowSystemTask.isAsyncComplete(task)); assertTrue(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue message is never pushed for async complete system tasks verify(queueDAO, times(1)).containsMessage(anyString(), anyString()); verify(queueDAO, times(1)).push(anyString(), anyString(), anyLong()); } @Test public void verifyAndRepairParentWorkflow() { WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowId("abcd"); workflow.setParentWorkflowId("parentWorkflowId"); when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(10)); when(executionDAO.getWorkflow("abcd", true)).thenReturn(workflow); when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); workflowRepairService.verifyAndRepairWorkflowTasks("abcd"); verify(queueDAO, times(1)).containsMessage(anyString(), anyString()); verify(queueDAO, times(1)).push(anyString(), anyString(), anyLong()); } @Test public void assertInProgressSubWorkflowSystemTasksAreCheckedAndRepaired() { String subWorkflowId = "subWorkflowId"; String taskId = "taskId"; TaskModel task = new TaskModel(); task.setTaskType(TASK_TYPE_SUB_WORKFLOW); task.setStatus(TaskModel.Status.IN_PROGRESS); task.setTaskId(taskId); task.setCallbackAfterSeconds(60); task.setSubWorkflowId(subWorkflowId); Map outputMap = new HashMap<>(); outputMap.put("subWorkflowId", subWorkflowId); task.setOutputData(outputMap); WorkflowModel subWorkflow = new WorkflowModel(); subWorkflow.setWorkflowId(subWorkflowId); subWorkflow.setStatus(WorkflowModel.Status.TERMINATED); subWorkflow.setOutput(Map.of("k1", "v1", "k2", "v2")); when(executionDAO.getWorkflow(subWorkflowId, false)).thenReturn(subWorkflow); assertTrue(workflowRepairService.verifyAndRepairTask(task)); // Verify that queue message is never pushed for async complete system tasks verify(queueDAO, never()).containsMessage(anyString(), anyString()); verify(queueDAO, never()).push(anyString(), anyString(), anyLong()); // Verify ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskModel.class); verify(executionDAO, times(1)).updateTask(argumentCaptor.capture()); assertEquals(taskId, argumentCaptor.getValue().getTaskId()); assertEquals(subWorkflowId, argumentCaptor.getValue().getSubWorkflowId()); assertEquals(TaskModel.Status.CANCELED, argumentCaptor.getValue().getStatus()); assertNotNull(argumentCaptor.getValue().getOutputData()); assertEquals(subWorkflowId, argumentCaptor.getValue().getOutputData().get("subWorkflowId")); assertEquals("v1", argumentCaptor.getValue().getOutputData().get("k1")); assertEquals("v2", argumentCaptor.getValue().getOutputData().get("k2")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.reconciliation; import java.time.Duration; import java.util.List; import java.util.Optional; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestWorkflowSweeper { private ConductorProperties properties; private WorkflowExecutor workflowExecutor; private WorkflowRepairService workflowRepairService; private QueueDAO queueDAO; private ExecutionDAOFacade executionDAOFacade; private WorkflowSweeper workflowSweeper; private int defaultPostPoneOffSetSeconds = 1800; @Before public void setUp() { properties = mock(ConductorProperties.class); workflowExecutor = mock(WorkflowExecutor.class); queueDAO = mock(QueueDAO.class); workflowRepairService = mock(WorkflowRepairService.class); executionDAOFacade = mock(ExecutionDAOFacade.class); workflowSweeper = new WorkflowSweeper( workflowExecutor, Optional.of(workflowRepairService), properties, queueDAO, executionDAOFacade); } @Test public void testPostponeDurationForHumanTaskType() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_HUMAN); taskModel.setStatus(Status.IN_PROGRESS); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForWaitTaskType() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForWaitTaskTypeWithLongWaitTime() { long waitTimeout = 65845; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); } @Test public void testPostponeDurationForWaitTaskTypeWithLessOneSecondWaitTime() { long waitTimeout = 180; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); } @Test public void testPostponeDurationForWaitTaskTypeWithZeroWaitTime() { long waitTimeout = 0; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); } @Test public void testPostponeDurationForTaskInProgress() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.IN_PROGRESS); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForTaskInProgressWithResponseTimeoutSet() { long responseTimeout = 200; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.IN_PROGRESS); taskModel.setResponseTimeoutSeconds(responseTimeout); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (responseTimeout + 1) * 1000); } @Test public void testPostponeDurationForTaskInScheduled() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); WorkflowDef workflowDef = new WorkflowDef(); workflowModel.setWorkflowDefinition(workflowDef); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.SCHEDULED); taskModel.setReferenceTaskName("task1"); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSet() { long workflowTimeout = 1800; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setTimeoutSeconds(workflowTimeout); workflowModel.setWorkflowDefinition(workflowDef); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.SCHEDULED); workflowModel.setTasks(List.of(taskModel)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); } @Test public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSetAndNoPollTimeout() { long workflowTimeout = 1800; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setTimeoutSeconds(workflowTimeout); workflowModel.setWorkflowDefinition(workflowDef); TaskDef taskDef = new TaskDef(); TaskModel taskModel = mock(TaskModel.class); workflowModel.setTasks(List.of(taskModel)); when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); } @Test public void testPostponeDurationForTaskInScheduledWithNoWorkflowTimeoutSetAndNoPollTimeout() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); WorkflowDef workflowDef = new WorkflowDef(); workflowModel.setWorkflowDefinition(workflowDef); TaskDef taskDef = new TaskDef(); TaskModel taskModel = mock(TaskModel.class); workflowModel.setTasks(List.of(taskModel)); when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForTaskInScheduledWithNoPollTimeoutSet() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskDef taskDef = new TaskDef(); WorkflowDef workflowDef = new WorkflowDef(); workflowModel.setWorkflowDefinition(workflowDef); TaskModel taskModel = mock(TaskModel.class); workflowModel.setTasks(List.of(taskModel)); when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), defaultPostPoneOffSetSeconds * 1000); } @Test public void testPostponeDurationForTaskInScheduledWithPollTimeoutSet() { int pollTimeout = 200; WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); TaskDef taskDef = new TaskDef(); taskDef.setPollTimeoutSeconds(pollTimeout); TaskModel taskModel = mock(TaskModel.class); ; workflowModel.setTasks(List.of(taskModel)); when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); when(properties.getWorkflowOffsetTimeout()) .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); verify(queueDAO) .setUnackTimeout( DECIDER_QUEUE, workflowModel.getWorkflowId(), (pollTimeout + 1) * 1000); } @Test public void testWorkflowOffsetJitter() { long offset = 45; for (int i = 0; i < 10; i++) { long offsetWithJitter = workflowSweeper.workflowOffsetWithJitter(offset); assertTrue(offsetWithJitter >= 30); assertTrue(offsetWithJitter <= 60); } } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/storage/DummyPayloadStorageTest.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.storage; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.Map; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class DummyPayloadStorageTest { private DummyPayloadStorage dummyPayloadStorage; private static final String TEST_STORAGE_PATH = "test-storage"; private ExternalStorageLocation location; private ObjectMapper objectMapper; public static final String MOCK_PAYLOAD = "{\n" + "\"output\": \"TEST_OUTPUT\",\n" + "}\n"; @Before public void setup() { dummyPayloadStorage = new DummyPayloadStorage(); objectMapper = new ObjectMapper(); location = dummyPayloadStorage.getLocation( ExternalPayloadStorage.Operation.WRITE, PayloadType.TASK_OUTPUT, TEST_STORAGE_PATH); try { byte[] payloadBytes = MOCK_PAYLOAD.getBytes("UTF-8"); dummyPayloadStorage.upload( location.getPath(), new ByteArrayInputStream(payloadBytes), payloadBytes.length); } catch (UnsupportedEncodingException unsupportedEncodingException) { } } @Test public void testGetLocationNotNull() { assertNotNull(location); } @Test public void testDownloadForValidPath() { try (InputStream inputStream = dummyPayloadStorage.download(location.getPath())) { Map payload = objectMapper.readValue( IOUtils.toString(inputStream, StandardCharsets.UTF_8), Map.class); assertTrue(payload.containsKey("output")); assertEquals(payload.get("output"), "TEST_OUTPUT"); } catch (Exception e) { assertTrue(e instanceof IOException); } } @Test public void testDownloadForInvalidPath() { InputStream inputStream = dummyPayloadStorage.download("testPath"); assertNull(inputStream); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/sync/local/LocalOnlyLockTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.sync.local; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.junit.After; import org.junit.Ignore; import org.junit.Test; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @Ignore // Test always times out in CI environment public class LocalOnlyLockTest { // Lock can be global since it uses global cache internally private final LocalOnlyLock localOnlyLock = new LocalOnlyLock(); @After public void tearDown() { // Clean caches between tests as they are shared globally localOnlyLock.cache().invalidateAll(); localOnlyLock.scheduledFutures().values().forEach(f -> f.cancel(false)); localOnlyLock.scheduledFutures().clear(); } @Test public void testLockUnlock() { final boolean a = localOnlyLock.acquireLock("a", 100, 10000, TimeUnit.MILLISECONDS); assertTrue(a); assertEquals(localOnlyLock.cache().estimatedSize(), 1); assertEquals(localOnlyLock.cache().get("a").isLocked(), true); assertEquals(localOnlyLock.scheduledFutures().size(), 1); localOnlyLock.releaseLock("a"); assertEquals(localOnlyLock.scheduledFutures().size(), 0); assertEquals(localOnlyLock.cache().get("a").isLocked(), false); localOnlyLock.deleteLock("a"); assertEquals(localOnlyLock.cache().estimatedSize(), 0); } @Test(timeout = 10 * 10_000) public void testLockTimeout() throws InterruptedException, ExecutionException { final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit( () -> { localOnlyLock.acquireLock("c", 100, 1000, TimeUnit.MILLISECONDS); }) .get(); assertTrue(localOnlyLock.acquireLock("d", 100, 1000, TimeUnit.MILLISECONDS)); assertFalse(localOnlyLock.acquireLock("c", 100, 1000, TimeUnit.MILLISECONDS)); assertEquals(localOnlyLock.scheduledFutures().size(), 2); executor.submit( () -> { localOnlyLock.releaseLock("c"); }) .get(); localOnlyLock.releaseLock("d"); assertEquals(localOnlyLock.scheduledFutures().size(), 0); } @Test(timeout = 10 * 10_000) public void testReleaseFromAnotherThread() throws InterruptedException, ExecutionException { final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit( () -> { localOnlyLock.acquireLock("c", 100, 10000, TimeUnit.MILLISECONDS); }) .get(); try { localOnlyLock.releaseLock("c"); } catch (IllegalMonitorStateException e) { // expected localOnlyLock.deleteLock("c"); return; } finally { executor.submit( () -> { localOnlyLock.releaseLock("c"); }) .get(); } fail(); } @Test(timeout = 10 * 10_000) public void testLockLeaseWithRelease() throws Exception { localOnlyLock.acquireLock("b", 1000, 1000, TimeUnit.MILLISECONDS); localOnlyLock.releaseLock("b"); // Wait for lease to run out and also call release Thread.sleep(2000); localOnlyLock.acquireLock("b"); assertEquals(true, localOnlyLock.cache().get("b").isLocked()); localOnlyLock.releaseLock("b"); } @Test public void testRelease() { localOnlyLock.releaseLock("x54as4d2;23'4"); localOnlyLock.releaseLock("x54as4d2;23'4"); assertEquals(false, localOnlyLock.cache().get("x54as4d2;23'4").isLocked()); } @Test(timeout = 10 * 10_000) public void testLockLeaseTime() throws InterruptedException { for (int i = 0; i < 10; i++) { final Thread thread = new Thread( () -> { localOnlyLock.acquireLock("a", 1000, 100, TimeUnit.MILLISECONDS); }); thread.start(); thread.join(); } localOnlyLock.acquireLock("a"); assertTrue(localOnlyLock.cache().get("a").isLocked()); localOnlyLock.releaseLock("a"); localOnlyLock.deleteLock("a"); } @Test public void testLockConfiguration() { new ApplicationContextRunner() .withPropertyValues("conductor.workflow-execution-lock.type=local_only") .withUserConfiguration(LocalOnlyLockConfiguration.class) .run( context -> { LocalOnlyLock lock = context.getBean(LocalOnlyLock.class); assertNotNull(lock); }); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtilsTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.unit.DataSize; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.model.TaskModel.Status.FAILED_WITH_TERMINAL_ERROR; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class ExternalPayloadStorageUtilsTest { private ExternalPayloadStorage externalPayloadStorage; private ExternalStorageLocation location; @Autowired private ObjectMapper objectMapper; // Subject private ExternalPayloadStorageUtils externalPayloadStorageUtils; @Rule public ExpectedException expectedException = ExpectedException.none(); @Before public void setup() { externalPayloadStorage = mock(ExternalPayloadStorage.class); ConductorProperties properties = mock(ConductorProperties.class); location = new ExternalStorageLocation(); location.setPath("some/test/path"); when(properties.getWorkflowInputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10L)); when(properties.getMaxWorkflowInputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10240L)); when(properties.getWorkflowOutputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10L)); when(properties.getMaxWorkflowOutputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10240L)); when(properties.getTaskInputPayloadSizeThreshold()).thenReturn(DataSize.ofKilobytes(10L)); when(properties.getMaxTaskInputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10240L)); when(properties.getTaskOutputPayloadSizeThreshold()).thenReturn(DataSize.ofKilobytes(10L)); when(properties.getMaxTaskOutputPayloadSizeThreshold()) .thenReturn(DataSize.ofKilobytes(10240L)); externalPayloadStorageUtils = new ExternalPayloadStorageUtils(externalPayloadStorage, properties, objectMapper); } @Test public void testDownloadPayload() throws IOException { String path = "test/payload"; Map payload = new HashMap<>(); payload.put("key1", "value1"); payload.put("key2", 200); byte[] payloadBytes = objectMapper.writeValueAsString(payload).getBytes(); when(externalPayloadStorage.download(path)) .thenReturn(new ByteArrayInputStream(payloadBytes)); Map result = externalPayloadStorageUtils.downloadPayload(path); assertNotNull(result); assertEquals(payload, result); } @SuppressWarnings("unchecked") @Test public void testUploadTaskPayload() throws IOException { AtomicInteger uploadCount = new AtomicInteger(0); InputStream stream = com.netflix.conductor.core.utils.ExternalPayloadStorageUtilsTest.class .getResourceAsStream("/payload.json"); Map payload = objectMapper.readValue(stream, Map.class); byte[] payloadBytes = objectMapper.writeValueAsString(payload).getBytes(); when(externalPayloadStorage.getLocation( ExternalPayloadStorage.Operation.WRITE, ExternalPayloadStorage.PayloadType.TASK_INPUT, "", payloadBytes)) .thenReturn(location); doAnswer( invocation -> { uploadCount.incrementAndGet(); return null; }) .when(externalPayloadStorage) .upload(anyString(), any(), anyLong()); TaskModel task = new TaskModel(); task.setInputData(payload); externalPayloadStorageUtils.verifyAndUpload( task, ExternalPayloadStorage.PayloadType.TASK_INPUT); assertTrue(StringUtils.isNotEmpty(task.getExternalInputPayloadStoragePath())); assertFalse(task.getInputData().isEmpty()); assertEquals(1, uploadCount.get()); assertNotNull(task.getExternalInputPayloadStoragePath()); } @SuppressWarnings("unchecked") @Test public void testUploadWorkflowPayload() throws IOException { AtomicInteger uploadCount = new AtomicInteger(0); InputStream stream = com.netflix.conductor.core.utils.ExternalPayloadStorageUtilsTest.class .getResourceAsStream("/payload.json"); Map payload = objectMapper.readValue(stream, Map.class); byte[] payloadBytes = objectMapper.writeValueAsString(payload).getBytes(); when(externalPayloadStorage.getLocation( ExternalPayloadStorage.Operation.WRITE, ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT, "", payloadBytes)) .thenReturn(location); doAnswer( invocation -> { uploadCount.incrementAndGet(); return null; }) .when(externalPayloadStorage) .upload(anyString(), any(), anyLong()); WorkflowModel workflow = new WorkflowModel(); WorkflowDef def = new WorkflowDef(); def.setName("name"); def.setVersion(1); workflow.setOutput(payload); workflow.setWorkflowDefinition(def); externalPayloadStorageUtils.verifyAndUpload( workflow, ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT); assertTrue(StringUtils.isNotEmpty(workflow.getExternalOutputPayloadStoragePath())); assertFalse(workflow.getOutput().isEmpty()); assertEquals(1, uploadCount.get()); assertNotNull(workflow.getExternalOutputPayloadStoragePath()); } @Test public void testUploadHelper() { AtomicInteger uploadCount = new AtomicInteger(0); String path = "some/test/path.json"; ExternalStorageLocation location = new ExternalStorageLocation(); location.setPath(path); when(externalPayloadStorage.getLocation(any(), any(), any(), any())).thenReturn(location); doAnswer( invocation -> { uploadCount.incrementAndGet(); return null; }) .when(externalPayloadStorage) .upload(anyString(), any(), anyLong()); assertEquals( path, externalPayloadStorageUtils.uploadHelper( new byte[] {}, 10L, ExternalPayloadStorage.PayloadType.TASK_OUTPUT)); assertEquals(1, uploadCount.get()); } @Test public void testFailTaskWithInputPayload() { TaskModel task = new TaskModel(); task.setInputData(new HashMap<>()); externalPayloadStorageUtils.failTask( task, ExternalPayloadStorage.PayloadType.TASK_INPUT, "error"); assertNotNull(task); assertTrue(task.getInputData().isEmpty()); assertEquals(FAILED_WITH_TERMINAL_ERROR, task.getStatus()); } @Test public void testFailTaskWithOutputPayload() { TaskModel task = new TaskModel(); task.setOutputData(new HashMap<>()); externalPayloadStorageUtils.failTask( task, ExternalPayloadStorage.PayloadType.TASK_OUTPUT, "error"); assertNotNull(task); assertTrue(task.getOutputData().isEmpty()); assertEquals(FAILED_WITH_TERMINAL_ERROR, task.getStatus()); } @Test public void testFailWorkflowWithInputPayload() { WorkflowModel workflow = new WorkflowModel(); workflow.setInput(new HashMap<>()); expectedException.expect(TerminateWorkflowException.class); externalPayloadStorageUtils.failWorkflow( workflow, ExternalPayloadStorage.PayloadType.TASK_INPUT, "error"); assertNotNull(workflow); assertTrue(workflow.getInput().isEmpty()); assertEquals(WorkflowModel.Status.FAILED, workflow.getStatus()); } @Test public void testFailWorkflowWithOutputPayload() { WorkflowModel workflow = new WorkflowModel(); workflow.setOutput(new HashMap<>()); expectedException.expect(TerminateWorkflowException.class); externalPayloadStorageUtils.failWorkflow( workflow, ExternalPayloadStorage.PayloadType.TASK_OUTPUT, "error"); assertNotNull(workflow); assertTrue(workflow.getOutput().isEmpty()); assertEquals(WorkflowModel.Status.FAILED, workflow.getStatus()); } @Test public void testShouldUpload() { Map payload = new HashMap<>(); payload.put("key1", "value1"); payload.put("key2", "value2"); TaskModel task = new TaskModel(); task.setInputData(payload); task.setOutputData(payload); WorkflowModel workflow = new WorkflowModel(); workflow.setInput(payload); workflow.setOutput(payload); assertTrue( externalPayloadStorageUtils.shouldUpload( task, ExternalPayloadStorage.PayloadType.TASK_INPUT)); assertTrue( externalPayloadStorageUtils.shouldUpload( task, ExternalPayloadStorage.PayloadType.TASK_OUTPUT)); assertTrue( externalPayloadStorageUtils.shouldUpload( task, ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT)); assertTrue( externalPayloadStorageUtils.shouldUpload( task, ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT)); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/utils/JsonUtilsTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class JsonUtilsTest { private JsonUtils jsonUtils; @Autowired private ObjectMapper objectMapper; @Before public void setup() { jsonUtils = new JsonUtils(objectMapper); } @Test public void testArray() { List list = new LinkedList<>(); Map map = new HashMap<>(); map.put("externalId", "[{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}]"); map.put("name", "conductor"); map.put("version", 2); list.add(map); //noinspection unchecked map = (Map) list.get(0); assertTrue(map.get("externalId") instanceof String); int before = list.size(); jsonUtils.expand(list); assertEquals(before, list.size()); //noinspection unchecked map = (Map) list.get(0); assertTrue(map.get("externalId") instanceof ArrayList); } @Test public void testMap() { Map map = new HashMap<>(); map.put("externalId", "{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}"); map.put("name", "conductor"); map.put("version", 2); assertTrue(map.get("externalId") instanceof String); jsonUtils.expand(map); assertTrue(map.get("externalId") instanceof LinkedHashMap); } @Test public void testMultiLevelMap() { Map parentMap = new HashMap<>(); parentMap.put("requestId", "abcde"); parentMap.put("status", "PROCESSED"); Map childMap = new HashMap<>(); childMap.put("path", "test/path"); childMap.put("type", "VIDEO"); Map grandChildMap = new HashMap<>(); grandChildMap.put("duration", "370"); grandChildMap.put("passed", "true"); childMap.put("metadata", grandChildMap); parentMap.put("asset", childMap); Object jsonObject = jsonUtils.expand(parentMap); assertNotNull(jsonObject); } // This test verifies that the types of the elements in the input are maintained upon expanding // the JSON object @Test public void testTypes() throws Exception { String map = "{\"requestId\":\"1375128656908832001\",\"workflowId\":\"fc147e1d-5408-4d41-b066-53cb2e551d0e\"," + "\"inner\":{\"num\":42,\"status\":\"READY\"}}"; jsonUtils.expand(map); Object jsonObject = jsonUtils.expand(map); assertNotNull(jsonObject); assertTrue(jsonObject instanceof LinkedHashMap); assertTrue(((LinkedHashMap) jsonObject).get("requestId") instanceof String); assertTrue(((LinkedHashMap) jsonObject).get("workflowId") instanceof String); assertTrue(((LinkedHashMap) jsonObject).get("inner") instanceof LinkedHashMap); assertTrue( ((LinkedHashMap) ((LinkedHashMap) jsonObject).get("inner")).get("num") instanceof Integer); assertTrue( ((LinkedHashMap) ((LinkedHashMap) jsonObject).get("inner")) .get("status") instanceof String); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/utils/ParametersUtilsTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) @SuppressWarnings("rawtypes") public class ParametersUtilsTest { private ParametersUtils parametersUtils; private JsonUtils jsonUtils; @Autowired private ObjectMapper objectMapper; @Before public void setup() { parametersUtils = new ParametersUtils(objectMapper); jsonUtils = new JsonUtils(objectMapper); } @Test public void testReplace() throws Exception { Map map = new HashMap<>(); map.put("name", "conductor"); map.put("version", 2); map.put("externalId", "{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}"); Map input = new HashMap<>(); input.put("k1", "${$.externalId}"); input.put("k4", "${name}"); input.put("k5", "${version}"); Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class); Map replaced = parametersUtils.replace(input, jsonObj); assertNotNull(replaced); assertEquals("{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}", replaced.get("k1")); assertEquals("conductor", replaced.get("k4")); assertEquals(2, replaced.get("k5")); } @Test public void testReplaceWithArrayExpand() { List list = new LinkedList<>(); Map map = new HashMap<>(); map.put("externalId", "[{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}]"); map.put("name", "conductor"); map.put("version", 2); list.add(map); jsonUtils.expand(list); Map input = new HashMap<>(); input.put("k1", "${$..externalId}"); input.put("k2", "${$[0].externalId[0].taskRefName}"); input.put("k3", "${__json_externalId.taskRefName}"); input.put("k4", "${$[0].name}"); input.put("k5", "${$[0].version}"); Map replaced = parametersUtils.replace(input, list); assertNotNull(replaced); assertEquals(replaced.get("k2"), "t001"); assertNull(replaced.get("k3")); assertEquals(replaced.get("k4"), "conductor"); assertEquals(replaced.get("k5"), 2); } @Test public void testReplaceWithMapExpand() { Map map = new HashMap<>(); map.put("externalId", "{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}"); map.put("name", "conductor"); map.put("version", 2); jsonUtils.expand(map); Map input = new HashMap<>(); input.put("k1", "${$.externalId}"); input.put("k2", "${externalId.taskRefName}"); input.put("k4", "${name}"); input.put("k5", "${version}"); Map replaced = parametersUtils.replace(input, map); assertNotNull(replaced); assertEquals("t001", replaced.get("k2")); assertNull(replaced.get("k3")); assertEquals("conductor", replaced.get("k4")); assertEquals(2, replaced.get("k5")); } @Test public void testReplaceConcurrent() throws ExecutionException, InterruptedException { ExecutorService executorService = Executors.newFixedThreadPool(2); AtomicReference generatedId = new AtomicReference<>("test-0"); Map input = new HashMap<>(); Map payload = new HashMap<>(); payload.put("event", "conductor:TEST_EVENT"); payload.put("someId", generatedId); input.put("payload", payload); input.put("name", "conductor"); input.put("version", 2); Map inputParams = new HashMap<>(); inputParams.put("k1", "${payload.someId}"); inputParams.put("k2", "${name}"); CompletableFuture.runAsync( () -> { for (int i = 0; i < 10000; i++) { generatedId.set("test-" + i); payload.put("someId", generatedId.get()); Object jsonObj = null; try { jsonObj = objectMapper.readValue( objectMapper.writeValueAsString(input), Object.class); } catch (JsonProcessingException e) { e.printStackTrace(); return; } Map replaced = parametersUtils.replace(inputParams, jsonObj); assertNotNull(replaced); assertEquals(generatedId.get(), replaced.get("k1")); assertEquals("conductor", replaced.get("k2")); assertNull(replaced.get("k3")); } }, executorService) .get(); executorService.shutdown(); } // Tests ParametersUtils with Map and List input values, and verifies input map is not mutated // by ParametersUtils. @Test public void testReplaceInputWithMapAndList() throws Exception { Map map = new HashMap<>(); map.put("name", "conductor"); map.put("version", 2); map.put("externalId", "{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}"); Map input = new HashMap<>(); input.put("k1", "${$.externalId}"); input.put("k2", "${name}"); input.put("k3", "${version}"); input.put("k4", "${}"); input.put("k5", "${ }"); Map mapValue = new HashMap<>(); mapValue.put("name", "${name}"); mapValue.put("version", "${version}"); input.put("map", mapValue); List listValue = new ArrayList<>(); listValue.add("${name}"); listValue.add("${version}"); input.put("list", listValue); Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class); Map replaced = parametersUtils.replace(input, jsonObj); assertNotNull(replaced); // Verify that values are replaced correctly. assertEquals("{\"taskRefName\":\"t001\",\"workflowId\":\"w002\"}", replaced.get("k1")); assertEquals("conductor", replaced.get("k2")); assertEquals(2, replaced.get("k3")); assertEquals("", replaced.get("k4")); assertEquals("", replaced.get("k5")); Map replacedMap = (Map) replaced.get("map"); assertEquals("conductor", replacedMap.get("name")); assertEquals(2, replacedMap.get("version")); List replacedList = (List) replaced.get("list"); assertEquals(2, replacedList.size()); assertEquals("conductor", replacedList.get(0)); assertEquals(2, replacedList.get(1)); // Verify that input map is not mutated assertEquals("${$.externalId}", input.get("k1")); assertEquals("${name}", input.get("k2")); assertEquals("${version}", input.get("k3")); Map inputMap = (Map) input.get("map"); assertEquals("${name}", inputMap.get("name")); assertEquals("${version}", inputMap.get("version")); List inputList = (List) input.get("list"); assertEquals(2, inputList.size()); assertEquals("${name}", inputList.get(0)); assertEquals("${version}", inputList.get(1)); } @Test public void testNestedPathExpressions() throws Exception { Map map = new HashMap<>(); map.put("name", "conductor"); map.put("index", 1); map.put("mapValue", "a"); map.put("recordIds", List.of(1, 2, 3)); map.put("map", Map.of("a", List.of(1, 2, 3), "b", List.of(2, 4, 5), "c", List.of(3, 7, 8))); Map input = new HashMap<>(); input.put("k1", "${recordIds[${index}]}"); input.put("k2", "${map.${mapValue}[${index}]}"); input.put("k3", "${map.b[${map.${mapValue}[${index}]}]}"); Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class); Map replaced = parametersUtils.replace(input, jsonObj); assertNotNull(replaced); assertEquals(2, replaced.get("k1")); assertEquals(2, replaced.get("k2")); assertEquals(5, replaced.get("k3")); } @Test public void testReplaceWithLineTerminators() throws Exception { Map map = new HashMap<>(); map.put("name", "conductor"); map.put("version", 2); Map input = new HashMap<>(); input.put("k1", "Name: ${name}; Version: ${version};"); input.put("k2", "Name: ${name};\nVersion: ${version};"); input.put("k3", "Name: ${name};\rVersion: ${version};"); input.put("k4", "Name: ${name};\r\nVersion: ${version};"); Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class); Map replaced = parametersUtils.replace(input, jsonObj); assertNotNull(replaced); assertEquals("Name: conductor; Version: 2;", replaced.get("k1")); assertEquals("Name: conductor;\nVersion: 2;", replaced.get("k2")); assertEquals("Name: conductor;\rVersion: 2;", replaced.get("k3")); assertEquals("Name: conductor;\r\nVersion: 2;", replaced.get("k4")); } @Test public void testReplaceWithEscapedTags() throws Exception { Map map = new HashMap<>(); map.put("someString", "conductor"); map.put("someNumber", 2); Map input = new HashMap<>(); input.put( "k1", "${$.someString} $${$.someNumber}${$.someNumber} ${$.someNumber}$${$.someString}"); input.put("k2", "$${$.someString}afterText"); input.put("k3", "beforeText$${$.someString}"); input.put("k4", "$${$.someString} afterText"); input.put("k5", "beforeText $${$.someString}"); Map mapValue = new HashMap<>(); mapValue.put("a", "${someString}"); mapValue.put("b", "${someNumber}"); mapValue.put("c", "$${someString} ${someNumber}"); input.put("map", mapValue); List listValue = new ArrayList<>(); listValue.add("${someString}"); listValue.add("${someNumber}"); listValue.add("${someString} $${someNumber}"); input.put("list", listValue); Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class); Map replaced = parametersUtils.replace(input, jsonObj); assertNotNull(replaced); // Verify that values are replaced correctly. assertEquals("conductor ${$.someNumber}2 2${$.someString}", replaced.get("k1")); assertEquals("${$.someString}afterText", replaced.get("k2")); assertEquals("beforeText${$.someString}", replaced.get("k3")); assertEquals("${$.someString} afterText", replaced.get("k4")); assertEquals("beforeText ${$.someString}", replaced.get("k5")); Map replacedMap = (Map) replaced.get("map"); assertEquals("conductor", replacedMap.get("a")); assertEquals(2, replacedMap.get("b")); assertEquals("${someString} 2", replacedMap.get("c")); List replacedList = (List) replaced.get("list"); assertEquals(3, replacedList.size()); assertEquals("conductor", replacedList.get(0)); assertEquals(2, replacedList.get(1)); assertEquals("conductor ${someNumber}", replacedList.get(2)); // Verify that input map is not mutated Map inputMap = (Map) input.get("map"); assertEquals("${someString}", inputMap.get("a")); assertEquals("${someNumber}", inputMap.get("b")); assertEquals("$${someString} ${someNumber}", inputMap.get("c")); // Verify that input list is not mutated List inputList = (List) input.get("list"); assertEquals(3, inputList.size()); assertEquals("${someString}", inputList.get(0)); assertEquals("${someNumber}", inputList.get(1)); assertEquals("${someString} $${someNumber}", inputList.get(2)); } @Test public void getWorkflowInputHandlesNullInputTemplate() { WorkflowDef workflowDef = new WorkflowDef(); Map inputParams = Map.of("key", "value"); Map workflowInput = parametersUtils.getWorkflowInput(workflowDef, inputParams); assertEquals("value", workflowInput.get("key")); } @Test public void getWorkflowInputFillsInTemplatedFields() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setInputTemplate(Map.of("other_key", "other_value")); Map inputParams = new HashMap<>(Map.of("key", "value")); Map workflowInput = parametersUtils.getWorkflowInput(workflowDef, inputParams); assertEquals("value", workflowInput.get("key")); assertEquals("other_value", workflowInput.get("other_key")); } @Test public void getWorkflowInputPreservesExistingFieldsIfPopulated() { WorkflowDef workflowDef = new WorkflowDef(); String keyName = "key"; workflowDef.setInputTemplate(Map.of(keyName, "templated_value")); Map inputParams = new HashMap<>(Map.of(keyName, "supplied_value")); Map workflowInput = parametersUtils.getWorkflowInput(workflowDef, inputParams); assertEquals("supplied_value", workflowInput.get(keyName)); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/utils/QueueUtilsTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import org.junit.Assert; import org.junit.Test; public class QueueUtilsTest { @Test public void queueNameWithTypeAndIsolationGroup() { String queueNameGenerated = QueueUtils.getQueueName("tType", null, "isolationGroup", null); String queueNameGeneratedOnlyType = QueueUtils.getQueueName("tType", null, null, null); String queueNameGeneratedWithAllValues = QueueUtils.getQueueName("tType", "domain", "iso", "eN"); Assert.assertEquals("tType-isolationGroup", queueNameGenerated); Assert.assertEquals("tType", queueNameGeneratedOnlyType); Assert.assertEquals("domain:tType@eN-iso", queueNameGeneratedWithAllValues); } @Test public void notIsolatedIfSeparatorNotPresent() { String notIsolatedQueue = "notIsolated"; Assert.assertFalse(QueueUtils.isIsolatedQueue(notIsolatedQueue)); } @Test public void testGetExecutionNameSpace() { String executionNameSpace = QueueUtils.getExecutionNameSpace("domain:queueName@eN-iso"); Assert.assertEquals(executionNameSpace, "eN"); } @Test public void testGetQueueExecutionNameSpaceEmpty() { Assert.assertEquals(QueueUtils.getExecutionNameSpace("queueName"), ""); } @Test public void testGetQueueExecutionNameSpaceWithIsolationGroup() { Assert.assertEquals( QueueUtils.getExecutionNameSpace("domain:test@executionNameSpace-isolated"), "executionNameSpace"); } @Test public void testGetQueueName() { Assert.assertEquals( "domain:taskType@eN-isolated", QueueUtils.getQueueName("taskType", "domain", "isolated", "eN")); } @Test public void testGetTaskType() { Assert.assertEquals("taskType", QueueUtils.getTaskType("domain:taskType-isolated")); } @Test public void testGetTaskTypeWithoutDomain() { Assert.assertEquals("taskType", QueueUtils.getTaskType("taskType-isolated")); } @Test public void testGetTaskTypeWithoutDomainAndWithoutIsolationGroup() { Assert.assertEquals("taskType", QueueUtils.getTaskType("taskType")); } @Test public void testGetTaskTypeWithoutDomainAndWithExecutionNameSpace() { Assert.assertEquals("taskType", QueueUtils.getTaskType("taskType@eN")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/core/utils/SemaphoreUtilTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.core.utils; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.IntStream; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @SuppressWarnings("ToArrayCallWithZeroLengthArrayArgument") public class SemaphoreUtilTest { @Test public void testBlockAfterAvailablePermitsExhausted() throws Exception { int threads = 5; ExecutorService executorService = Executors.newFixedThreadPool(threads); SemaphoreUtil semaphoreUtil = new SemaphoreUtil(threads); List> futuresList = new ArrayList<>(); IntStream.range(0, threads) .forEach( t -> futuresList.add( CompletableFuture.runAsync( () -> semaphoreUtil.acquireSlots(1), executorService))); CompletableFuture allFutures = CompletableFuture.allOf( futuresList.toArray(new CompletableFuture[futuresList.size()])); allFutures.get(); assertEquals(0, semaphoreUtil.availableSlots()); assertFalse(semaphoreUtil.acquireSlots(1)); executorService.shutdown(); } @Test public void testAllowsPollingWhenPermitBecomesAvailable() throws Exception { int threads = 5; ExecutorService executorService = Executors.newFixedThreadPool(threads); SemaphoreUtil semaphoreUtil = new SemaphoreUtil(threads); List> futuresList = new ArrayList<>(); IntStream.range(0, threads) .forEach( t -> futuresList.add( CompletableFuture.runAsync( () -> semaphoreUtil.acquireSlots(1), executorService))); CompletableFuture allFutures = CompletableFuture.allOf( futuresList.toArray(new CompletableFuture[futuresList.size()])); allFutures.get(); assertEquals(0, semaphoreUtil.availableSlots()); semaphoreUtil.completeProcessing(1); assertTrue(semaphoreUtil.availableSlots() > 0); assertTrue(semaphoreUtil.acquireSlots(1)); executorService.shutdown(); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/dao/ExecutionDAOTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import org.apache.commons.lang3.builder.EqualsBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import static org.junit.Assert.*; public abstract class ExecutionDAOTest { protected abstract ExecutionDAO getExecutionDAO(); protected ConcurrentExecutionLimitDAO getConcurrentExecutionLimitDAO() { return (ConcurrentExecutionLimitDAO) getExecutionDAO(); } @Rule public ExpectedException expectedException = ExpectedException.none(); @Test public void testTaskExceedsLimit() { TaskDef taskDefinition = new TaskDef(); taskDefinition.setName("task1"); taskDefinition.setConcurrentExecLimit(1); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("task1"); workflowTask.setTaskDefinition(taskDefinition); workflowTask.setTaskDefinition(taskDefinition); List tasks = new LinkedList<>(); for (int i = 0; i < 15; i++) { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(i + 1); task.setTaskId("t_" + i); task.setWorkflowInstanceId("workflow_" + i); task.setReferenceTaskName("task1"); task.setTaskDefName("task1"); tasks.add(task); task.setStatus(TaskModel.Status.SCHEDULED); task.setWorkflowTask(workflowTask); } getExecutionDAO().createTasks(tasks); assertFalse(getConcurrentExecutionLimitDAO().exceedsLimit(tasks.get(0))); tasks.get(0).setStatus(TaskModel.Status.IN_PROGRESS); getExecutionDAO().updateTask(tasks.get(0)); for (TaskModel task : tasks) { assertTrue(getConcurrentExecutionLimitDAO().exceedsLimit(task)); } } @Test public void testCreateTaskException() { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName("task1"); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Workflow instance id cannot be null"); getExecutionDAO().createTasks(List.of(task)); task.setWorkflowInstanceId(UUID.randomUUID().toString()); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Task reference name cannot be null"); getExecutionDAO().createTasks(List.of(task)); } @Test public void testCreateTaskException2() { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName("task1"); task.setWorkflowInstanceId(UUID.randomUUID().toString()); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Task reference name cannot be null"); getExecutionDAO().createTasks(Collections.singletonList(task)); } @Test public void testTaskCreateDups() { List tasks = new LinkedList<>(); String workflowId = UUID.randomUUID().toString(); for (int i = 0; i < 3; i++) { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(i + 1); task.setTaskId(workflowId + "_t" + i); task.setReferenceTaskName("t" + i); task.setRetryCount(0); task.setWorkflowInstanceId(workflowId); task.setTaskDefName("task" + i); task.setStatus(TaskModel.Status.IN_PROGRESS); tasks.add(task); } // Let's insert a retried task TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(workflowId + "_t" + 2); task.setReferenceTaskName("t" + 2); task.setRetryCount(1); task.setWorkflowInstanceId(workflowId); task.setTaskDefName("task" + 2); task.setStatus(TaskModel.Status.IN_PROGRESS); tasks.add(task); // Duplicate task! task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(workflowId + "_t" + 1); task.setReferenceTaskName("t" + 1); task.setRetryCount(0); task.setWorkflowInstanceId(workflowId); task.setTaskDefName("task" + 1); task.setStatus(TaskModel.Status.IN_PROGRESS); tasks.add(task); List created = getExecutionDAO().createTasks(tasks); assertEquals(tasks.size() - 1, created.size()); // 1 less Set srcIds = tasks.stream() .map(t -> t.getReferenceTaskName() + "." + t.getRetryCount()) .collect(Collectors.toSet()); Set createdIds = created.stream() .map(t -> t.getReferenceTaskName() + "." + t.getRetryCount()) .collect(Collectors.toSet()); assertEquals(srcIds, createdIds); List pending = getExecutionDAO().getPendingTasksByWorkflow("task0", workflowId); assertNotNull(pending); assertEquals(1, pending.size()); assertTrue(EqualsBuilder.reflectionEquals(tasks.get(0), pending.get(0))); List found = getExecutionDAO().getTasks(tasks.get(0).getTaskDefName(), null, 1); assertNotNull(found); assertEquals(1, found.size()); assertTrue(EqualsBuilder.reflectionEquals(tasks.get(0), found.get(0))); } @Test public void testTaskOps() { List tasks = new LinkedList<>(); String workflowId = UUID.randomUUID().toString(); for (int i = 0; i < 3; i++) { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(workflowId + "_t" + i); task.setReferenceTaskName("testTaskOps" + i); task.setRetryCount(0); task.setWorkflowInstanceId(workflowId); task.setTaskDefName("testTaskOps" + i); task.setStatus(TaskModel.Status.IN_PROGRESS); tasks.add(task); } for (int i = 0; i < 3; i++) { TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId("x" + workflowId + "_t" + i); task.setReferenceTaskName("testTaskOps" + i); task.setRetryCount(0); task.setWorkflowInstanceId("x" + workflowId); task.setTaskDefName("testTaskOps" + i); task.setStatus(TaskModel.Status.IN_PROGRESS); getExecutionDAO().createTasks(Collections.singletonList(task)); } List created = getExecutionDAO().createTasks(tasks); assertEquals(tasks.size(), created.size()); List pending = getExecutionDAO().getPendingTasksForTaskType(tasks.get(0).getTaskDefName()); assertNotNull(pending); assertEquals(2, pending.size()); // Pending list can come in any order. finding the one we are looking for and then // comparing TaskModel matching = pending.stream() .filter(task -> task.getTaskId().equals(tasks.get(0).getTaskId())) .findAny() .get(); assertTrue(EqualsBuilder.reflectionEquals(matching, tasks.get(0))); for (int i = 0; i < 3; i++) { TaskModel found = getExecutionDAO().getTask(workflowId + "_t" + i); assertNotNull(found); found.addOutput("updated", true); found.setStatus(TaskModel.Status.COMPLETED); getExecutionDAO().updateTask(found); } List taskIds = tasks.stream().map(TaskModel::getTaskId).collect(Collectors.toList()); List found = getExecutionDAO().getTasks(taskIds); assertEquals(taskIds.size(), found.size()); found.forEach( task -> { assertTrue(task.getOutputData().containsKey("updated")); assertEquals(true, task.getOutputData().get("updated")); boolean removed = getExecutionDAO().removeTask(task.getTaskId()); assertTrue(removed); }); found = getExecutionDAO().getTasks(taskIds); assertTrue(found.isEmpty()); } @Test public void testPending() { WorkflowDef def = new WorkflowDef(); def.setName("pending_count_test"); WorkflowModel workflow = createTestWorkflow(); workflow.setWorkflowDefinition(def); List workflowIds = generateWorkflows(workflow, 10); long count = getExecutionDAO().getPendingWorkflowCount(def.getName()); assertEquals(10, count); for (int i = 0; i < 10; i++) { getExecutionDAO().removeFromPendingWorkflow(def.getName(), workflowIds.get(i)); } count = getExecutionDAO().getPendingWorkflowCount(def.getName()); assertEquals(0, count); } @Test public void complexExecutionTest() { WorkflowModel workflow = createTestWorkflow(); int numTasks = workflow.getTasks().size(); String workflowId = getExecutionDAO().createWorkflow(workflow); assertEquals(workflow.getWorkflowId(), workflowId); List created = getExecutionDAO().createTasks(workflow.getTasks()); assertEquals(workflow.getTasks().size(), created.size()); WorkflowModel workflowWithTasks = getExecutionDAO().getWorkflow(workflow.getWorkflowId(), true); assertEquals(workflowId, workflowWithTasks.getWorkflowId()); assertEquals(numTasks, workflowWithTasks.getTasks().size()); WorkflowModel found = getExecutionDAO().getWorkflow(workflowId, false); assertTrue(found.getTasks().isEmpty()); workflow.getTasks().clear(); assertEquals(workflow, found); workflow.getInput().put("updated", true); getExecutionDAO().updateWorkflow(workflow); found = getExecutionDAO().getWorkflow(workflowId); assertNotNull(found); assertTrue(found.getInput().containsKey("updated")); assertEquals(true, found.getInput().get("updated")); List running = getExecutionDAO() .getRunningWorkflowIds( workflow.getWorkflowName(), workflow.getWorkflowVersion()); assertNotNull(running); assertTrue(running.isEmpty()); workflow.setStatus(WorkflowModel.Status.RUNNING); getExecutionDAO().updateWorkflow(workflow); running = getExecutionDAO() .getRunningWorkflowIds( workflow.getWorkflowName(), workflow.getWorkflowVersion()); assertNotNull(running); assertEquals(1, running.size()); assertEquals(workflow.getWorkflowId(), running.get(0)); List pending = getExecutionDAO() .getPendingWorkflowsByType( workflow.getWorkflowName(), workflow.getWorkflowVersion()); assertNotNull(pending); assertEquals(1, pending.size()); assertEquals(3, pending.get(0).getTasks().size()); pending.get(0).getTasks().clear(); assertEquals(workflow, pending.get(0)); workflow.setStatus(WorkflowModel.Status.COMPLETED); getExecutionDAO().updateWorkflow(workflow); running = getExecutionDAO() .getRunningWorkflowIds( workflow.getWorkflowName(), workflow.getWorkflowVersion()); assertNotNull(running); assertTrue(running.isEmpty()); List bytime = getExecutionDAO() .getWorkflowsByType( workflow.getWorkflowName(), System.currentTimeMillis(), System.currentTimeMillis() + 100); assertNotNull(bytime); assertTrue(bytime.isEmpty()); bytime = getExecutionDAO() .getWorkflowsByType( workflow.getWorkflowName(), workflow.getCreateTime() - 10, workflow.getCreateTime() + 10); assertNotNull(bytime); assertEquals(1, bytime.size()); } protected WorkflowModel createTestWorkflow() { WorkflowDef def = new WorkflowDef(); def.setName("Junit Workflow"); def.setVersion(3); def.setSchemaVersion(2); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.setCorrelationId("correlationX"); workflow.setCreatedBy("junit_tester"); workflow.setEndTime(200L); Map input = new HashMap<>(); input.put("param1", "param1 value"); input.put("param2", 100); workflow.setInput(input); Map output = new HashMap<>(); output.put("ouput1", "output 1 value"); output.put("op2", 300); workflow.setOutput(output); workflow.setOwnerApp("workflow"); workflow.setParentWorkflowId("parentWorkflowId"); workflow.setParentWorkflowTaskId("parentWFTaskId"); workflow.setReasonForIncompletion("missing recipe"); workflow.setReRunFromWorkflowId("re-run from id1"); workflow.setCreateTime(90L); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setWorkflowId(UUID.randomUUID().toString()); List tasks = new LinkedList<>(); TaskModel task = new TaskModel(); task.setScheduledTime(1L); task.setSeq(1); task.setTaskId(UUID.randomUUID().toString()); task.setReferenceTaskName("t1"); task.setWorkflowInstanceId(workflow.getWorkflowId()); task.setTaskDefName("task1"); TaskModel task2 = new TaskModel(); task2.setScheduledTime(2L); task2.setSeq(2); task2.setTaskId(UUID.randomUUID().toString()); task2.setReferenceTaskName("t2"); task2.setWorkflowInstanceId(workflow.getWorkflowId()); task2.setTaskDefName("task2"); TaskModel task3 = new TaskModel(); task3.setScheduledTime(2L); task3.setSeq(3); task3.setTaskId(UUID.randomUUID().toString()); task3.setReferenceTaskName("t3"); task3.setWorkflowInstanceId(workflow.getWorkflowId()); task3.setTaskDefName("task3"); tasks.add(task); tasks.add(task2); tasks.add(task3); workflow.setTasks(tasks); workflow.setUpdatedBy("junit_tester"); workflow.setUpdatedTime(800L); return workflow; } protected List generateWorkflows(WorkflowModel base, int count) { List workflowIds = new ArrayList<>(); for (int i = 0; i < count; i++) { String workflowId = UUID.randomUUID().toString(); base.setWorkflowId(workflowId); base.setCorrelationId("corr001"); base.setStatus(WorkflowModel.Status.RUNNING); getExecutionDAO().createWorkflow(base); workflowIds.add(workflowId); } return workflowIds; } } ================================================ FILE: core/src/test/java/com/netflix/conductor/dao/PollDataDAOTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.dao; import java.util.List; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.PollData; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public abstract class PollDataDAOTest { protected abstract PollDataDAO getPollDataDAO(); @Test public void testPollData() { getPollDataDAO().updateLastPollData("taskDef", null, "workerId1"); PollData pollData = getPollDataDAO().getPollData("taskDef", null); assertNotNull(pollData); assertTrue(pollData.getLastPollTime() > 0); assertEquals(pollData.getQueueName(), "taskDef"); assertNull(pollData.getDomain()); assertEquals(pollData.getWorkerId(), "workerId1"); getPollDataDAO().updateLastPollData("taskDef", "domain1", "workerId1"); pollData = getPollDataDAO().getPollData("taskDef", "domain1"); assertNotNull(pollData); assertTrue(pollData.getLastPollTime() > 0); assertEquals(pollData.getQueueName(), "taskDef"); assertEquals(pollData.getDomain(), "domain1"); assertEquals(pollData.getWorkerId(), "workerId1"); List pData = getPollDataDAO().getPollData("taskDef"); assertEquals(pData.size(), 2); pollData = getPollDataDAO().getPollData("taskDef", "domain2"); assertNull(pollData); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/metrics/WorkflowMonitorTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.metrics; import java.util.List; import java.util.Map; import java.util.Set; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.service.MetadataService; @RunWith(SpringRunner.class) public class WorkflowMonitorTest { @Mock private MetadataService metadataService; @Mock private QueueDAO queueDAO; @Mock private ExecutionDAOFacade executionDAOFacade; private WorkflowMonitor workflowMonitor; @Before public void beforeEach() { workflowMonitor = new WorkflowMonitor(metadataService, queueDAO, executionDAOFacade, 1000, Set.of()); } private WorkflowDef makeDef(String name, int version, String ownerApp) { WorkflowDef wd = new WorkflowDef(); wd.setName(name); wd.setVersion(version); wd.setOwnerApp(ownerApp); return wd; } @Test public void testPendingWorkflowDataMap() { WorkflowDef test1_1 = makeDef("test1", 1, null); WorkflowDef test1_2 = makeDef("test1", 2, "name1"); WorkflowDef test2_1 = makeDef("test2", 1, "first"); WorkflowDef test2_2 = makeDef("test2", 2, "mid"); WorkflowDef test2_3 = makeDef("test2", 3, "last"); final Map mapping = workflowMonitor.getPendingWorkflowToOwnerAppMap( List.of(test1_1, test1_2, test2_1, test2_2, test2_3)); Assert.assertEquals(2, mapping.keySet().size()); Assert.assertTrue(mapping.containsKey("test1")); Assert.assertTrue(mapping.containsKey("test2")); Assert.assertEquals("name1", mapping.get("test1")); Assert.assertEquals("last", mapping.get("test2")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/EventServiceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.Set; import javax.validation.ConstraintViolationException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.core.events.EventQueues; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class EventServiceTest { @TestConfiguration static class TestEventConfiguration { @Bean public EventService eventService() { MetadataService metadataService = mock(MetadataService.class); EventQueues eventQueues = mock(EventQueues.class); return new EventServiceImpl(metadataService, eventQueues); } } @Autowired private EventService eventService; @Test(expected = ConstraintViolationException.class) public void testAddEventHandler() { try { eventService.addEventHandler(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("EventHandler cannot be null.")); throw ex; } fail("eventService.addEventHandler did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateEventHandler() { try { eventService.updateEventHandler(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("EventHandler cannot be null.")); throw ex; } fail("eventService.updateEventHandler did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testRemoveEventHandlerStatus() { try { eventService.removeEventHandlerStatus(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("EventHandler name cannot be null or empty.")); throw ex; } fail("eventService.removeEventHandlerStatus did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testGetEventHandlersForEvent() { try { eventService.getEventHandlersForEvent(null, false); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Event cannot be null or empty.")); throw ex; } fail("eventService.getEventHandlersForEvent did not throw ConstraintViolationException !"); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/ExecutionServiceTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.dao.QueueDAO; import static junit.framework.TestCase.assertEquals; import static org.mockito.Mockito.when; @RunWith(SpringRunner.class) public class ExecutionServiceTest { @Mock private WorkflowExecutor workflowExecutor; @Mock private ExecutionDAOFacade executionDAOFacade; @Mock private QueueDAO queueDAO; @Mock private ConductorProperties conductorProperties; @Mock private ExternalPayloadStorage externalPayloadStorage; @Mock private SystemTaskRegistry systemTaskRegistry; private ExecutionService executionService; private Workflow workflow1; private Workflow workflow2; private Task taskWorkflow1; private Task taskWorkflow2; private final List sort = Collections.singletonList("Sort"); @Before public void setup() { when(conductorProperties.getTaskExecutionPostponeDuration()) .thenReturn(Duration.ofSeconds(60)); executionService = new ExecutionService( workflowExecutor, executionDAOFacade, queueDAO, conductorProperties, externalPayloadStorage, systemTaskRegistry); WorkflowDef workflowDef = new WorkflowDef(); workflow1 = new Workflow(); workflow1.setWorkflowId("wf1"); workflow1.setWorkflowDefinition(workflowDef); workflow2 = new Workflow(); workflow2.setWorkflowId("wf2"); workflow2.setWorkflowDefinition(workflowDef); taskWorkflow1 = new Task(); taskWorkflow1.setTaskId("task1"); taskWorkflow1.setWorkflowInstanceId("wf1"); taskWorkflow2 = new Task(); taskWorkflow2.setTaskId("task2"); taskWorkflow2.setWorkflowInstanceId("wf2"); } @Test public void workflowSearchTest() { when(executionDAOFacade.searchWorkflowSummary("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( new WorkflowSummary(workflow1), new WorkflowSummary(workflow2)))); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getWorkflow(workflow2.getWorkflowId(), false)) .thenReturn(workflow2); SearchResult searchResult = executionService.search("query", "*", 0, 2, sort); assertEquals(2, searchResult.getTotalHits()); assertEquals(2, searchResult.getResults().size()); assertEquals(workflow1.getWorkflowId(), searchResult.getResults().get(0).getWorkflowId()); assertEquals(workflow2.getWorkflowId(), searchResult.getResults().get(1).getWorkflowId()); } @Test public void workflowSearchV2Test() { when(executionDAOFacade.searchWorkflows("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( workflow1.getWorkflowId(), workflow2.getWorkflowId()))); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getWorkflow(workflow2.getWorkflowId(), false)) .thenReturn(workflow2); SearchResult searchResult = executionService.searchV2("query", "*", 0, 2, sort); assertEquals(2, searchResult.getTotalHits()); assertEquals(Arrays.asList(workflow1, workflow2), searchResult.getResults()); } @Test public void workflowSearchV2ExceptionTest() { when(executionDAOFacade.searchWorkflows("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( workflow1.getWorkflowId(), workflow2.getWorkflowId()))); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getWorkflow(workflow2.getWorkflowId(), false)) .thenThrow(new RuntimeException()); SearchResult searchResult = executionService.searchV2("query", "*", 0, 2, sort); assertEquals(1, searchResult.getTotalHits()); assertEquals(Collections.singletonList(workflow1), searchResult.getResults()); } @Test public void workflowSearchByTasksTest() { when(executionDAOFacade.searchTaskSummary("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( new TaskSummary(taskWorkflow1), new TaskSummary(taskWorkflow2)))); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getWorkflow(workflow2.getWorkflowId(), false)) .thenReturn(workflow2); SearchResult searchResult = executionService.searchWorkflowByTasks("query", "*", 0, 2, sort); assertEquals(2, searchResult.getTotalHits()); assertEquals(2, searchResult.getResults().size()); assertEquals(workflow1.getWorkflowId(), searchResult.getResults().get(0).getWorkflowId()); assertEquals(workflow2.getWorkflowId(), searchResult.getResults().get(1).getWorkflowId()); } @Test public void workflowSearchByTasksExceptionTest() { when(executionDAOFacade.searchTaskSummary("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( new TaskSummary(taskWorkflow1), new TaskSummary(taskWorkflow2)))); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getTask(workflow2.getWorkflowId())) .thenThrow(new RuntimeException()); SearchResult searchResult = executionService.searchWorkflowByTasks("query", "*", 0, 2, sort); assertEquals(1, searchResult.getTotalHits()); assertEquals(1, searchResult.getResults().size()); assertEquals(workflow1.getWorkflowId(), searchResult.getResults().get(0).getWorkflowId()); } @Test public void workflowSearchByTasksV2Test() { when(executionDAOFacade.searchTasks("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( taskWorkflow1.getTaskId(), taskWorkflow2.getTaskId()))); when(executionDAOFacade.getTask(taskWorkflow1.getTaskId())).thenReturn(taskWorkflow1); when(executionDAOFacade.getTask(taskWorkflow2.getTaskId())).thenReturn(taskWorkflow2); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); when(executionDAOFacade.getWorkflow(workflow2.getWorkflowId(), false)) .thenReturn(workflow2); SearchResult searchResult = executionService.searchWorkflowByTasksV2("query", "*", 0, 2, sort); assertEquals(2, searchResult.getTotalHits()); assertEquals(Arrays.asList(workflow1, workflow2), searchResult.getResults()); } @Test public void workflowSearchByTasksV2ExceptionTest() { when(executionDAOFacade.searchTasks("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( taskWorkflow1.getTaskId(), taskWorkflow2.getTaskId()))); when(executionDAOFacade.getTask(taskWorkflow1.getTaskId())).thenReturn(taskWorkflow1); when(executionDAOFacade.getTask(taskWorkflow2.getTaskId())) .thenThrow(new RuntimeException()); when(executionDAOFacade.getWorkflow(workflow1.getWorkflowId(), false)) .thenReturn(workflow1); SearchResult searchResult = executionService.searchWorkflowByTasksV2("query", "*", 0, 2, sort); assertEquals(1, searchResult.getTotalHits()); assertEquals(Collections.singletonList(workflow1), searchResult.getResults()); } @Test public void TaskSearchTest() { List taskList = Arrays.asList(new TaskSummary(taskWorkflow1), new TaskSummary(taskWorkflow2)); when(executionDAOFacade.searchTaskSummary("query", "*", 0, 2, sort)) .thenReturn(new SearchResult<>(2, taskList)); SearchResult searchResult = executionService.getSearchTasks("query", "*", 0, 2, "Sort"); assertEquals(2, searchResult.getTotalHits()); assertEquals(2, searchResult.getResults().size()); assertEquals(taskWorkflow1.getTaskId(), searchResult.getResults().get(0).getTaskId()); assertEquals(taskWorkflow2.getTaskId(), searchResult.getResults().get(1).getTaskId()); } @Test public void TaskSearchV2Test() { when(executionDAOFacade.searchTasks("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( taskWorkflow1.getTaskId(), taskWorkflow2.getTaskId()))); when(executionDAOFacade.getTask(taskWorkflow1.getTaskId())).thenReturn(taskWorkflow1); when(executionDAOFacade.getTask(taskWorkflow2.getTaskId())).thenReturn(taskWorkflow2); SearchResult searchResult = executionService.getSearchTasksV2("query", "*", 0, 2, "Sort"); assertEquals(2, searchResult.getTotalHits()); assertEquals(Arrays.asList(taskWorkflow1, taskWorkflow2), searchResult.getResults()); } @Test public void TaskSearchV2ExceptionTest() { when(executionDAOFacade.searchTasks("query", "*", 0, 2, sort)) .thenReturn( new SearchResult<>( 2, Arrays.asList( taskWorkflow1.getTaskId(), taskWorkflow2.getTaskId()))); when(executionDAOFacade.getTask(taskWorkflow1.getTaskId())).thenReturn(taskWorkflow1); when(executionDAOFacade.getTask(taskWorkflow2.getTaskId())) .thenThrow(new RuntimeException()); SearchResult searchResult = executionService.getSearchTasksV2("query", "*", 0, 2, "Sort"); assertEquals(1, searchResult.getTotalHits()); assertEquals(Collections.singletonList(taskWorkflow1), searchResult.getResults()); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/MetadataServiceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.*; import javax.validation.ConstraintViolationException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDefSummary; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.dao.MetadataDAO; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class MetadataServiceTest { @TestConfiguration static class TestMetadataConfiguration { @Bean public MetadataDAO metadataDAO() { return mock(MetadataDAO.class); } @Bean public ConductorProperties properties() { ConductorProperties properties = mock(ConductorProperties.class); when(properties.isOwnerEmailMandatory()).thenReturn(true); return properties; } @Bean public MetadataService metadataService( MetadataDAO metadataDAO, ConductorProperties properties) { EventHandlerDAO eventHandlerDAO = mock(EventHandlerDAO.class); when(metadataDAO.getAllWorkflowDefs()).thenReturn(mockWorkflowDefs()); return new MetadataServiceImpl(metadataDAO, eventHandlerDAO, properties); } private List mockWorkflowDefs() { // Returns list of workflowDefs in reverse version order. List retval = new ArrayList<>(); for (int i = 5; i > 0; i--) { WorkflowDef def = new WorkflowDef(); def.setCreateTime(new Date().getTime()); def.setVersion(i); def.setName("test_workflow_def"); retval.add(def); } return retval; } } @Autowired private MetadataDAO metadataDAO; @Autowired private MetadataService metadataService; @Test(expected = ConstraintViolationException.class) public void testRegisterTaskDefNoName() { TaskDef taskDef = new TaskDef(); try { metadataService.registerTaskDef(Collections.singletonList(taskDef)); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskDef name cannot be null or empty")); assertTrue(messages.contains("ownerEmail cannot be empty")); throw ex; } fail("metadataService.registerTaskDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testRegisterTaskDefNull() { try { metadataService.registerTaskDef(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskDefList cannot be empty or null")); throw ex; } fail("metadataService.registerTaskDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testRegisterTaskDefNoResponseTimeout() { try { TaskDef taskDef = new TaskDef(); taskDef.setName("somename"); taskDef.setOwnerEmail("sample@test.com"); taskDef.setResponseTimeoutSeconds(0); metadataService.registerTaskDef(Collections.singletonList(taskDef)); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue( messages.contains( "TaskDef responseTimeoutSeconds: 0 should be minimum 1 second")); throw ex; } fail("metadataService.registerTaskDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateTaskDefNameNull() { try { TaskDef taskDef = new TaskDef(); metadataService.updateTaskDef(taskDef); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskDef name cannot be null or empty")); assertTrue(messages.contains("ownerEmail cannot be empty")); throw ex; } fail("metadataService.updateTaskDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateTaskDefNull() { try { metadataService.updateTaskDef(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskDef cannot be null")); throw ex; } fail("metadataService.updateTaskDef did not throw ConstraintViolationException !"); } @Test(expected = NotFoundException.class) public void testUpdateTaskDefNotExisting() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setOwnerEmail("sample@test.com"); when(metadataDAO.getTaskDef(any())).thenReturn(null); metadataService.updateTaskDef(taskDef); } @Test(expected = NotFoundException.class) public void testUpdateTaskDefDaoException() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setOwnerEmail("sample@test.com"); when(metadataDAO.getTaskDef(any())).thenReturn(null); metadataService.updateTaskDef(taskDef); } @Test public void testRegisterTaskDef() { TaskDef taskDef = new TaskDef(); taskDef.setName("somename"); taskDef.setOwnerEmail("sample@test.com"); taskDef.setResponseTimeoutSeconds(60 * 60); metadataService.registerTaskDef(Collections.singletonList(taskDef)); verify(metadataDAO, times(1)).createTaskDef(any(TaskDef.class)); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefNull() { try { List workflowDefList = null; metadataService.updateWorkflowDef(workflowDefList); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDef list name cannot be null or empty")); throw ex; } fail("metadataService.updateWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefEmptyList() { try { List workflowDefList = new ArrayList<>(); metadataService.updateWorkflowDef(workflowDefList); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDefList is empty")); throw ex; } fail("metadataService.updateWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefWithNullWorkflowDef() { try { List workflowDefList = new ArrayList<>(); workflowDefList.add(null); metadataService.updateWorkflowDef(workflowDefList); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDef cannot be null")); throw ex; } fail("metadataService.updateWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefWithEmptyWorkflowDefName() { try { List workflowDefList = new ArrayList<>(); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName(null); workflowDef.setOwnerEmail(null); workflowDefList.add(workflowDef); metadataService.updateWorkflowDef(workflowDefList); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDef name cannot be null or empty")); assertTrue(messages.contains("WorkflowTask list cannot be empty")); assertTrue(messages.contains("ownerEmail cannot be empty")); throw ex; } fail("metadataService.updateWorkflowDef did not throw ConstraintViolationException !"); } @Test public void testUpdateWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("somename"); workflowDef.setOwnerEmail("sample@test.com"); List tasks = new ArrayList<>(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("hello"); workflowTask.setName("hello"); tasks.add(workflowTask); workflowDef.setTasks(tasks); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); metadataService.updateWorkflowDef(Collections.singletonList(workflowDef)); verify(metadataDAO, times(1)).updateWorkflowDef(workflowDef); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefWithCaseExpression() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("somename"); workflowDef.setOwnerEmail("sample@test.com"); List tasks = new ArrayList<>(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("hello"); workflowTask.setName("hello"); workflowTask.setType("DECISION"); WorkflowTask caseTask = new WorkflowTask(); caseTask.setTaskReferenceName("casetrue"); caseTask.setName("casetrue"); List caseTaskList = new ArrayList<>(); caseTaskList.add(caseTask); Map> decisionCases = new HashMap(); decisionCases.put("true", caseTaskList); workflowTask.setDecisionCases(decisionCases); workflowTask.setCaseExpression("1 >0abcd"); tasks.add(workflowTask); workflowDef.setTasks(tasks); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); BulkResponse bulkResponse = metadataService.updateWorkflowDef(Collections.singletonList(workflowDef)); } @Test(expected = ConstraintViolationException.class) public void testUpdateWorkflowDefWithJavscriptEvaluator() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("somename"); workflowDef.setOwnerEmail("sample@test.com"); List tasks = new ArrayList<>(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("hello"); workflowTask.setName("hello"); workflowTask.setType("SWITCH"); workflowTask.setEvaluatorType("javascript"); workflowTask.setExpression("1>abcd"); WorkflowTask caseTask = new WorkflowTask(); caseTask.setTaskReferenceName("casetrue"); caseTask.setName("casetrue"); List caseTaskList = new ArrayList<>(); caseTaskList.add(caseTask); Map> decisionCases = new HashMap(); decisionCases.put("true", caseTaskList); workflowTask.setDecisionCases(decisionCases); tasks.add(workflowTask); workflowDef.setTasks(tasks); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); BulkResponse bulkResponse = metadataService.updateWorkflowDef(Collections.singletonList(workflowDef)); } @Test(expected = ConstraintViolationException.class) public void testRegisterWorkflowDefNoName() { try { WorkflowDef workflowDef = new WorkflowDef(); metadataService.registerWorkflowDef(workflowDef); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDef name cannot be null or empty")); assertTrue(messages.contains("WorkflowTask list cannot be empty")); assertTrue(messages.contains("ownerEmail cannot be empty")); throw ex; } fail("metadataService.registerWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testValidateWorkflowDefNoName() { try { WorkflowDef workflowDef = new WorkflowDef(); metadataService.validateWorkflowDef(workflowDef); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowDef name cannot be null or empty")); assertTrue(messages.contains("WorkflowTask list cannot be empty")); assertTrue(messages.contains("ownerEmail cannot be empty")); throw ex; } fail("metadataService.validateWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testRegisterWorkflowDefInvalidName() { try { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("invalid:name"); workflowDef.setOwnerEmail("inavlid-email"); metadataService.registerWorkflowDef(workflowDef); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowTask list cannot be empty")); assertTrue( messages.contains( "Workflow name cannot contain the following set of characters: ':'")); assertTrue(messages.contains("ownerEmail should be valid email address")); throw ex; } fail("metadataService.registerWorkflowDef did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testValidateWorkflowDefInvalidName() { try { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("invalid:name"); workflowDef.setOwnerEmail("inavlid-email"); metadataService.validateWorkflowDef(workflowDef); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowTask list cannot be empty")); assertTrue( messages.contains( "Workflow name cannot contain the following set of characters: ':'")); assertTrue(messages.contains("ownerEmail should be valid email address")); throw ex; } fail("metadataService.validateWorkflowDef did not throw ConstraintViolationException !"); } @Test public void testRegisterWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("somename"); workflowDef.setSchemaVersion(2); workflowDef.setOwnerEmail("sample@test.com"); List tasks = new ArrayList<>(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("hello"); workflowTask.setName("hello"); tasks.add(workflowTask); workflowDef.setTasks(tasks); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); metadataService.registerWorkflowDef(workflowDef); verify(metadataDAO, times(1)).createWorkflowDef(workflowDef); assertEquals(2, workflowDef.getSchemaVersion()); } @Test public void testValidateWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("somename"); workflowDef.setSchemaVersion(2); workflowDef.setOwnerEmail("sample@test.com"); List tasks = new ArrayList<>(); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setTaskReferenceName("hello"); workflowTask.setName("hello"); tasks.add(workflowTask); workflowDef.setTasks(tasks); when(metadataDAO.getTaskDef(any())).thenReturn(new TaskDef()); metadataService.validateWorkflowDef(workflowDef); verify(metadataDAO, times(1)).createWorkflowDef(workflowDef); assertEquals(2, workflowDef.getSchemaVersion()); } @Test(expected = ConstraintViolationException.class) public void testUnregisterWorkflowDefNoName() { try { metadataService.unregisterWorkflowDef("", null); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Workflow name cannot be null or empty")); assertTrue(messages.contains("Version cannot be null")); throw ex; } fail("metadataService.unregisterWorkflowDef did not throw ConstraintViolationException !"); } @Test public void testUnregisterWorkflowDef() { metadataService.unregisterWorkflowDef("somename", 111); verify(metadataDAO, times(1)).removeWorkflowDef("somename", 111); } @Test(expected = ConstraintViolationException.class) public void testValidateEventNull() { try { metadataService.addEventHandler(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("EventHandler cannot be null")); throw ex; } fail("metadataService.addEventHandler did not throw ConstraintViolationException !"); } @Test(expected = ConstraintViolationException.class) public void testValidateEventNoEvent() { try { EventHandler eventHandler = new EventHandler(); metadataService.addEventHandler(eventHandler); } catch (ConstraintViolationException ex) { assertEquals(3, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Missing event handler name")); assertTrue(messages.contains("Missing event location")); assertTrue( messages.contains("No actions specified. Please specify at-least one action")); throw ex; } fail("metadataService.addEventHandler did not throw ConstraintViolationException !"); } @Test public void testWorkflowNamesAndVersions() { Map> namesAndVersions = metadataService.getWorkflowNamesAndVersions(); Iterator versions = namesAndVersions.get("test_workflow_def").iterator(); for (int i = 1; i <= 5; i++) { WorkflowDefSummary ver = versions.next(); assertEquals(i, ver.getVersion()); assertNotNull(ver.getCreateTime()); assertEquals("test_workflow_def", ver.getName()); } } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/TaskServiceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.List; import java.util.Set; import javax.validation.ConstraintViolationException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.dao.QueueDAO; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class TaskServiceTest { @TestConfiguration static class TestTaskConfiguration { @Bean public ExecutionService executionService() { return mock(ExecutionService.class); } @Bean public TaskService taskService(ExecutionService executionService) { QueueDAO queueDAO = mock(QueueDAO.class); return new TaskServiceImpl(executionService, queueDAO); } } @Autowired private TaskService taskService; @Autowired private ExecutionService executionService; @Test(expected = ConstraintViolationException.class) public void testPoll() { try { taskService.poll(null, null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testBatchPoll() { try { taskService.batchPoll(null, null, null, null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetTasks() { try { taskService.getTasks(null, null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetPendingTaskForWorkflow() { try { taskService.getPendingTaskForWorkflow(null, null); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); assertTrue(messages.contains("TaskReferenceName cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testUpdateTask() { try { taskService.updateTask(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskResult cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testUpdateTaskInValid() { try { TaskResult taskResult = new TaskResult(); taskService.updateTask(taskResult); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Workflow Id cannot be null or empty")); assertTrue(messages.contains("Task ID cannot be null or empty")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testAckTaskReceived() { try { taskService.ackTaskReceived(null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskId cannot be null or empty.")); throw ex; } } @Test public void testAckTaskReceivedMissingWorkerId() { String ack = taskService.ackTaskReceived("abc", null); assertNotNull(ack); } @Test(expected = ConstraintViolationException.class) public void testLog() { try { taskService.log(null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetTaskLogs() { try { taskService.getTaskLogs(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetTask() { try { taskService.getTask(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRemoveTaskFromQueue() { try { taskService.removeTaskFromQueue(null, null); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskId cannot be null or empty.")); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetPollData() { try { taskService.getPollData(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRequeuePendingTask() { try { taskService.requeuePendingTask(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("TaskType cannot be null or empty.")); throw ex; } } @Test public void testSearch() { SearchResult searchResult = new SearchResult<>(2, List.of(mock(TaskSummary.class), mock(TaskSummary.class))); when(executionService.getSearchTasks("query", "*", 0, 2, "Sort")).thenReturn(searchResult); assertEquals(searchResult, taskService.search(0, 2, "Sort", "*", "query")); } @Test public void testSearchV2() { SearchResult searchResult = new SearchResult<>(2, List.of(mock(Task.class), mock(Task.class))); when(executionService.getSearchTasksV2("query", "*", 0, 2, "Sort")) .thenReturn(searchResult); assertEquals(searchResult, taskService.searchV2(0, 2, "Sort", "*", "query")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/WorkflowBulkServiceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; import javax.validation.ConstraintViolationException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.core.execution.WorkflowExecutor; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class WorkflowBulkServiceTest { @TestConfiguration static class TestWorkflowBulkConfiguration { @Bean WorkflowExecutor workflowExecutor() { return mock(WorkflowExecutor.class); } @Bean public WorkflowBulkService workflowBulkService(WorkflowExecutor workflowExecutor) { return new WorkflowBulkServiceImpl(workflowExecutor); } } @Autowired private WorkflowExecutor workflowExecutor; @Autowired private WorkflowBulkService workflowBulkService; @Test(expected = ConstraintViolationException.class) public void testPauseWorkflowNull() { try { workflowBulkService.pauseWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testPauseWorkflowWithInvalidListSize() { try { List list = new ArrayList<>(1001); for (int i = 0; i < 1002; i++) { list.add("test"); } workflowBulkService.pauseWorkflow(list); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue( messages.contains( "Cannot process more than 1000 workflows. Please use multiple requests.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testResumeWorkflowNull() { try { workflowBulkService.resumeWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRestartWorkflowNull() { try { workflowBulkService.restart(null, false); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRetryWorkflowNull() { try { workflowBulkService.retry(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); throw ex; } } @Test public void testRetryWorkflowSuccessful() { // When workflowBulkService.retry(Collections.singletonList("anyId")); // Then verify(workflowExecutor).retry("anyId", false); } @Test(expected = ConstraintViolationException.class) public void testTerminateNull() { try { workflowBulkService.terminate(null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowIds list cannot be null.")); throw ex; } } } ================================================ FILE: core/src/test/java/com/netflix/conductor/service/WorkflowServiceTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.service; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.validation.ConstraintViolationException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.operation.StartWorkflowOperation; import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SuppressWarnings("SpringJavaAutowiredMembersInspection") @RunWith(SpringRunner.class) @EnableAutoConfiguration public class WorkflowServiceTest { @TestConfiguration static class TestWorkflowConfiguration { @Bean public WorkflowExecutor workflowExecutor() { return mock(WorkflowExecutor.class); } @Bean public StartWorkflowOperation startWorkflowOperation() { return mock(StartWorkflowOperation.class); } @Bean public ExecutionService executionService() { return mock(ExecutionService.class); } @Bean public MetadataService metadataService() { return mock(MetadataServiceImpl.class); } @Bean public WorkflowService workflowService( WorkflowExecutor workflowExecutor, ExecutionService executionService, MetadataService metadataService, StartWorkflowOperation startWorkflowOperation) { return new WorkflowServiceImpl( workflowExecutor, executionService, metadataService, startWorkflowOperation); } } @Autowired private WorkflowExecutor workflowExecutor; @Autowired private ExecutionService executionService; @Autowired private MetadataService metadataService; @Autowired private WorkflowService workflowService; @Test(expected = ConstraintViolationException.class) public void testStartWorkflowNull() { try { workflowService.startWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("StartWorkflowRequest cannot be null")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testGetWorkflowsNoName() { try { workflowService.getWorkflows("", "c123", true, true); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Workflow name cannot be null or empty")); throw ex; } } @Test public void testGetWorklfowsSingleCorrelationId() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); List workflowArrayList = Collections.singletonList(workflow); when(executionService.getWorkflowInstances( anyString(), anyString(), anyBoolean(), anyBoolean())) .thenReturn(workflowArrayList); assertEquals(workflowArrayList, workflowService.getWorkflows("test", "c123", true, true)); } @Test public void testGetWorklfowsMultipleCorrelationId() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); List workflowArrayList = Collections.singletonList(workflow); List correlationIdList = Collections.singletonList("c123"); Map> workflowMap = new HashMap<>(); workflowMap.put("c123", workflowArrayList); when(executionService.getWorkflowInstances( anyString(), anyString(), anyBoolean(), anyBoolean())) .thenReturn(workflowArrayList); assertEquals( workflowMap, workflowService.getWorkflows("test", true, true, correlationIdList)); } @Test public void testGetExecutionStatus() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); when(executionService.getExecutionStatus(anyString(), anyBoolean())).thenReturn(workflow); assertEquals(workflow, workflowService.getExecutionStatus("w123", true)); } @Test(expected = ConstraintViolationException.class) public void testGetExecutionStatusNoWorkflowId() { try { workflowService.getExecutionStatus("", true); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = NotFoundException.class) public void testNotFoundExceptionGetExecutionStatus() { when(executionService.getExecutionStatus(anyString(), anyBoolean())).thenReturn(null); workflowService.getExecutionStatus("w123", true); } @Test public void testDeleteWorkflow() { workflowService.deleteWorkflow("w123", false); verify(executionService, times(1)).removeWorkflow(anyString(), eq(false)); } @Test(expected = ConstraintViolationException.class) public void testInvalidDeleteWorkflow() { try { workflowService.deleteWorkflow(null, false); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test public void testArchiveWorkflow() { workflowService.deleteWorkflow("w123", true); verify(executionService, times(1)).removeWorkflow(anyString(), eq(true)); } @Test(expected = ConstraintViolationException.class) public void testInvalidArchiveWorkflow() { try { workflowService.deleteWorkflow(null, true); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testInvalidPauseWorkflow() { try { workflowService.pauseWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testInvalidResumeWorkflow() { try { workflowService.resumeWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testInvalidSkipTaskFromWorkflow() { try { SkipTaskRequest skipTaskRequest = new SkipTaskRequest(); workflowService.skipTaskFromWorkflow(null, null, skipTaskRequest); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId name cannot be null or empty.")); assertTrue(messages.contains("TaskReferenceName cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testInvalidWorkflowNameGetRunningWorkflows() { try { workflowService.getRunningWorkflows(null, 123, null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("Workflow name cannot be null or empty.")); throw ex; } } @Test public void testGetRunningWorkflowsTime() { workflowService.getRunningWorkflows("test", 1, 100L, 120L); verify(workflowExecutor, times(1)) .getWorkflows(anyString(), anyInt(), anyLong(), anyLong()); } @Test public void testGetRunningWorkflows() { workflowService.getRunningWorkflows("test", 1, null, null); verify(workflowExecutor, times(1)).getRunningWorkflowIds(anyString(), anyInt()); } @Test public void testDecideWorkflow() { workflowService.decideWorkflow("test"); verify(workflowExecutor, times(1)).decide(anyString()); } @Test public void testPauseWorkflow() { workflowService.pauseWorkflow("test"); verify(workflowExecutor, times(1)).pauseWorkflow(anyString()); } @Test public void testResumeWorkflow() { workflowService.resumeWorkflow("test"); verify(workflowExecutor, times(1)).resumeWorkflow(anyString()); } @Test public void testSkipTaskFromWorkflow() { workflowService.skipTaskFromWorkflow("test", "testTask", null); verify(workflowExecutor, times(1)).skipTaskFromWorkflow(anyString(), anyString(), isNull()); } @Test public void testRerunWorkflow() { RerunWorkflowRequest request = new RerunWorkflowRequest(); workflowService.rerunWorkflow("test", request); verify(workflowExecutor, times(1)).rerun(any(RerunWorkflowRequest.class)); } @Test(expected = ConstraintViolationException.class) public void testRerunWorkflowNull() { try { workflowService.rerunWorkflow(null, null); } catch (ConstraintViolationException ex) { assertEquals(2, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); assertTrue(messages.contains("RerunWorkflowRequest cannot be null.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRestartWorkflowNull() { try { workflowService.restartWorkflow(null, false); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testRetryWorkflowNull() { try { workflowService.retryWorkflow(null, false); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testResetWorkflowNull() { try { workflowService.resetWorkflow(null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test(expected = ConstraintViolationException.class) public void testTerminateWorkflowNull() { try { workflowService.terminateWorkflow(null, null); } catch (ConstraintViolationException ex) { assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue(messages.contains("WorkflowId cannot be null or empty.")); throw ex; } } @Test public void testRerunWorkflowReturnWorkflowId() { RerunWorkflowRequest request = new RerunWorkflowRequest(); String workflowId = "w123"; when(workflowExecutor.rerun(any(RerunWorkflowRequest.class))).thenReturn(workflowId); assertEquals(workflowId, workflowService.rerunWorkflow("test", request)); } @Test public void testRestartWorkflow() { workflowService.restartWorkflow("w123", false); verify(workflowExecutor, times(1)).restart(anyString(), anyBoolean()); } @Test public void testRetryWorkflow() { workflowService.retryWorkflow("w123", false); verify(workflowExecutor, times(1)).retry(anyString(), anyBoolean()); } @Test public void testResetWorkflow() { workflowService.resetWorkflow("w123"); verify(workflowExecutor, times(1)).resetCallbacksForWorkflow(anyString()); } @Test public void testTerminateWorkflow() { workflowService.terminateWorkflow("w123", "test"); verify(workflowExecutor, times(1)).terminateWorkflow(anyString(), anyString()); } @Test public void testSearchWorkflows() { Workflow workflow = new Workflow(); WorkflowDef def = new WorkflowDef(); def.setName("name"); def.setVersion(1); workflow.setWorkflowDefinition(def); workflow.setCorrelationId("c123"); WorkflowSummary workflowSummary = new WorkflowSummary(workflow); List listOfWorkflowSummary = Collections.singletonList(workflowSummary); SearchResult searchResult = new SearchResult<>(100, listOfWorkflowSummary); when(executionService.search("*", "*", 0, 100, Collections.singletonList("asc"))) .thenReturn(searchResult); assertEquals(searchResult, workflowService.searchWorkflows(0, 100, "asc", "*", "*")); assertEquals( searchResult, workflowService.searchWorkflows( 0, 100, Collections.singletonList("asc"), "*", "*")); } @Test public void testSearchWorkflowsV2() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); List listOfWorkflow = Collections.singletonList(workflow); SearchResult searchResult = new SearchResult<>(1, listOfWorkflow); when(executionService.searchV2("*", "*", 0, 100, Collections.singletonList("asc"))) .thenReturn(searchResult); assertEquals(searchResult, workflowService.searchWorkflowsV2(0, 100, "asc", "*", "*")); assertEquals( searchResult, workflowService.searchWorkflowsV2( 0, 100, Collections.singletonList("asc"), "*", "*")); } @Test public void testInvalidSizeSearchWorkflows() { ConstraintViolationException ex = assertThrows( ConstraintViolationException.class, () -> workflowService.searchWorkflows(0, 6000, "asc", "*", "*")); assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue( messages.contains( "Cannot return more than 5000 workflows. Please use pagination.")); } @Test public void testInvalidSizeSearchWorkflowsV2() { ConstraintViolationException ex = assertThrows( ConstraintViolationException.class, () -> workflowService.searchWorkflowsV2(0, 6000, "asc", "*", "*")); assertEquals(1, ex.getConstraintViolations().size()); Set messages = getConstraintViolationMessages(ex.getConstraintViolations()); assertTrue( messages.contains( "Cannot return more than 5000 workflows. Please use pagination.")); } @Test public void testSearchWorkflowsByTasks() { Workflow workflow = new Workflow(); WorkflowDef def = new WorkflowDef(); def.setName("name"); def.setVersion(1); workflow.setWorkflowDefinition(def); workflow.setCorrelationId("c123"); WorkflowSummary workflowSummary = new WorkflowSummary(workflow); List listOfWorkflowSummary = Collections.singletonList(workflowSummary); SearchResult searchResult = new SearchResult<>(100, listOfWorkflowSummary); when(executionService.searchWorkflowByTasks( "*", "*", 0, 100, Collections.singletonList("asc"))) .thenReturn(searchResult); assertEquals(searchResult, workflowService.searchWorkflowsByTasks(0, 100, "asc", "*", "*")); assertEquals( searchResult, workflowService.searchWorkflowsByTasks( 0, 100, Collections.singletonList("asc"), "*", "*")); } @Test public void testSearchWorkflowsByTasksV2() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); List listOfWorkflow = Collections.singletonList(workflow); SearchResult searchResult = new SearchResult<>(1, listOfWorkflow); when(executionService.searchWorkflowByTasksV2( "*", "*", 0, 100, Collections.singletonList("asc"))) .thenReturn(searchResult); assertEquals( searchResult, workflowService.searchWorkflowsByTasksV2(0, 100, "asc", "*", "*")); assertEquals( searchResult, workflowService.searchWorkflowsByTasksV2( 0, 100, Collections.singletonList("asc"), "*", "*")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/validations/WorkflowDefConstraintTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.validations; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import org.apache.bval.jsr.ApacheValidationProvider; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.dao.MetadataDAO; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class WorkflowDefConstraintTest { private static Validator validator; private static ValidatorFactory validatorFactory; private MetadataDAO mockMetadataDao; @BeforeClass public static void init() { validatorFactory = Validation.byProvider(ApacheValidationProvider.class) .configure() .buildValidatorFactory(); validator = validatorFactory.getValidator(); } @AfterClass public static void close() { validatorFactory.close(); } @Before public void setUp() { mockMetadataDao = Mockito.mock(MetadataDAO.class); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); ValidationContext.initialize(mockMetadataDao); } @Test public void testWorkflowTaskName() { TaskDef taskDef = new TaskDef(); // name is null ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); Validator validator = factory.getValidator(); Set> result = validator.validate(taskDef); assertEquals(2, result.size()); } @Test public void testWorkflowTaskSimple() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("sampleWorkflow"); workflowDef.setDescription("Sample workflow def"); workflowDef.setOwnerEmail("sample@test.com"); workflowDef.setVersion(2); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("fileLocation", "${workflow.input.fileLocation}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); Set> result = validator.validate(workflowDef); assertEquals(0, result.size()); } @Test /*Testcase to check inputParam is not valid */ public void testWorkflowTaskInvalidInputParam() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("sampleWorkflow"); workflowDef.setDescription("Sample workflow def"); workflowDef.setOwnerEmail("sample@test.com"); workflowDef.setVersion(2); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("fileLocation", "${work.input.fileLocation}"); workflowTask_1.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); validator = factory.getValidator(); when(mockMetadataDao.getTaskDef("work1")).thenReturn(new TaskDef()); Set> result = validator.validate(workflowDef); assertEquals(1, result.size()); assertEquals( result.iterator().next().getMessage(), "taskReferenceName: work for given task: task_1 input value: fileLocation of input parameter: ${work.input.fileLocation} is not defined in workflow definition."); } @Test public void testWorkflowTaskReferenceNameNotUnique() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("sampleWorkflow"); workflowDef.setDescription("Sample workflow def"); workflowDef.setOwnerEmail("sample@test.com"); workflowDef.setVersion(2); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map inputParam = new HashMap<>(); inputParam.put("fileLocation", "${task_2.input.fileLocation}"); workflowTask_1.setInputParameters(inputParam); WorkflowTask workflowTask_2 = new WorkflowTask(); workflowTask_2.setName("task_2"); workflowTask_2.setTaskReferenceName("task_1"); workflowTask_2.setType(TaskType.TASK_TYPE_SIMPLE); workflowTask_2.setInputParameters(inputParam); List tasks = new ArrayList<>(); tasks.add(workflowTask_1); tasks.add(workflowTask_2); workflowDef.setTasks(tasks); ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); validator = factory.getValidator(); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowDef); assertEquals(3, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "taskReferenceName: task_2 for given task: task_2 input value: fileLocation of input parameter: ${task_2.input.fileLocation} is not defined in workflow definition.")); assertTrue( validationErrors.contains( "taskReferenceName: task_2 for given task: task_1 input value: fileLocation of input parameter: ${task_2.input.fileLocation} is not defined in workflow definition.")); assertTrue( validationErrors.contains( "taskReferenceName: task_1 should be unique across tasks for a given workflowDefinition: sampleWorkflow")); } } ================================================ FILE: core/src/test/java/com/netflix/conductor/validations/WorkflowTaskTypeConstraintTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.validations; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.validation.ConstraintViolation; import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; import javax.validation.executable.ExecutableValidator; import org.apache.bval.jsr.ApacheValidationProvider; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.tasks.Terminate; import com.netflix.conductor.dao.MetadataDAO; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class WorkflowTaskTypeConstraintTest { private static Validator validator; private static ValidatorFactory validatorFactory; private MetadataDAO mockMetadataDao; @BeforeClass public static void init() { validatorFactory = Validation.byProvider(ApacheValidationProvider.class) .configure() .buildValidatorFactory(); validator = validatorFactory.getValidator(); } @AfterClass public static void close() { validatorFactory.close(); } @Before public void setUp() { mockMetadataDao = Mockito.mock(MetadataDAO.class); ValidationContext.initialize(mockMetadataDao); } @Test public void testWorkflowTaskMissingReferenceName() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setDynamicForkTasksParam("taskList"); workflowTask.setDynamicForkTasksInputParamName("ForkTaskInputParam"); workflowTask.setTaskReferenceName(null); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); assertEquals( result.iterator().next().getMessage(), "WorkflowTask taskReferenceName name cannot be empty or null"); } @Test public void testWorkflowTaskTestSetType() throws NoSuchMethodException { WorkflowTask workflowTask = createSampleWorkflowTask(); Method method = WorkflowTask.class.getMethod("setType", String.class); Object[] parameterValues = {""}; ExecutableValidator executableValidator = validator.forExecutables(); Set> result = executableValidator.validateParameters(workflowTask, method, parameterValues); assertEquals(1, result.size()); assertEquals( result.iterator().next().getMessage(), "WorkTask type cannot be null or empty"); } @Test public void testWorkflowTaskTypeEvent() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("EVENT"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); assertEquals( result.iterator().next().getMessage(), "sink field is required for taskType: EVENT taskName: encode"); } @Test public void testWorkflowTaskTypeDynamic() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("DYNAMIC"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); assertEquals( result.iterator().next().getMessage(), "dynamicTaskNameParam field is required for taskType: DYNAMIC taskName: encode"); } @Test public void testWorkflowTaskTypeDecision() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("DECISION"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "decisionCases should have atleast one task for taskType: DECISION taskName: encode")); assertTrue( validationErrors.contains( "caseValueParam or caseExpression field is required for taskType: DECISION taskName: encode")); } @Test public void testWorkflowTaskTypeDoWhile() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("DO_WHILE"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "loopExpression field is required for taskType: DO_WHILE taskName: encode")); assertTrue( validationErrors.contains( "loopover field is required for taskType: DO_WHILE taskName: encode")); } @Test public void testWorkflowTaskTypeWait() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("WAIT"); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); workflowTask.setInputParameters(Map.of("duration", "10s", "until", "2022-04-16")); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "Both 'duration' and 'until' specified. Please provide only one input")); } @Test public void testWorkflowTaskTypeDecisionWithCaseParam() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("DECISION"); workflowTask.setCaseExpression("$.valueCheck == null ? 'true': 'false'"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "decisionCases should have atleast one task for taskType: DECISION taskName: encode")); } @Test public void testWorkflowTaskTypeForJoinDynamic() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN_DYNAMIC"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "dynamicForkTasksInputParamName field is required for taskType: FORK_JOIN_DYNAMIC taskName: encode")); assertTrue( validationErrors.contains( "dynamicForkTasksParam field is required for taskType: FORK_JOIN_DYNAMIC taskName: encode")); } @Test public void testWorkflowTaskTypeForJoinDynamicLegacy() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN_DYNAMIC"); workflowTask.setDynamicForkJoinTasksParam("taskList"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeForJoinDynamicWithForJoinTaskParam() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN_DYNAMIC"); workflowTask.setDynamicForkJoinTasksParam("taskList"); workflowTask.setDynamicForkTasksInputParamName("ForkTaskInputParam"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "dynamicForkJoinTasksParam or combination of dynamicForkTasksInputParamName and dynamicForkTasksParam cam be used for taskType: FORK_JOIN_DYNAMIC taskName: encode")); } @Test public void testWorkflowTaskTypeForJoinDynamicValid() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN_DYNAMIC"); workflowTask.setDynamicForkTasksParam("ForkTasksParam"); workflowTask.setDynamicForkTasksInputParamName("ForkTaskInputParam"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeForJoinDynamicWithForJoinTaskParamAndInputTaskParam() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN_DYNAMIC"); workflowTask.setDynamicForkJoinTasksParam("taskList"); workflowTask.setDynamicForkTasksInputParamName("ForkTaskInputParam"); workflowTask.setDynamicForkTasksParam("ForkTasksParam"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "dynamicForkJoinTasksParam or combination of dynamicForkTasksInputParamName and dynamicForkTasksParam cam be used for taskType: FORK_JOIN_DYNAMIC taskName: encode")); } @Test public void testWorkflowTaskTypeHTTP() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("HTTP"); workflowTask.getInputParameters().put("http_request", "http://www.netflix.com"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeHTTPWithHttpParamMissing() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("HTTP"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "inputParameters.http_request field is required for taskType: HTTP taskName: encode")); } @Test public void testWorkflowTaskTypeHTTPWithHttpParamInTaskDef() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("HTTP"); TaskDef taskDef = new TaskDef(); taskDef.setName("encode"); taskDef.getInputTemplate().put("http_request", "http://www.netflix.com"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(taskDef); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeHTTPWithHttpParamInTaskDefAndWorkflowTask() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("HTTP"); workflowTask.getInputParameters().put("http_request", "http://www.netflix.com"); TaskDef taskDef = new TaskDef(); taskDef.setName("encode"); taskDef.getInputTemplate().put("http_request", "http://www.netflix.com"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(taskDef); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeFork() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("FORK_JOIN"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "forkTasks should have atleast one task for taskType: FORK_JOIN taskName: encode")); } @Test public void testWorkflowTaskTypeSubworkflowMissingSubworkflowParam() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("SUB_WORKFLOW"); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "subWorkflowParam field is required for taskType: SUB_WORKFLOW taskName: encode")); } @Test public void testWorkflowTaskTypeSubworkflow() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("SUB_WORKFLOW"); SubWorkflowParams subWorkflowTask = new SubWorkflowParams(); workflowTask.setSubWorkflowParam(subWorkflowTask); Set> result = validator.validate(workflowTask); assertEquals(2, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue(validationErrors.contains("SubWorkflowParams name cannot be null")); assertTrue(validationErrors.contains("SubWorkflowParams name cannot be empty")); } @Test public void testWorkflowTaskTypeTerminateWithoutTerminationStatus() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_TERMINATE); workflowTask.setName("terminate_task"); workflowTask.setInputParameters( Collections.singletonMap( Terminate.getTerminationWorkflowOutputParameter(), "blah")); List validationErrors = getErrorMessages(workflowTask); Assert.assertEquals(1, validationErrors.size()); Assert.assertEquals( "terminate task must have an terminationStatus parameter and must be set to COMPLETED or FAILED, taskName: terminate_task", validationErrors.get(0)); } @Test public void testWorkflowTaskTypeTerminateWithInvalidStatus() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_TERMINATE); workflowTask.setName("terminate_task"); workflowTask.setInputParameters( Collections.singletonMap(Terminate.getTerminationStatusParameter(), "blah")); List validationErrors = getErrorMessages(workflowTask); Assert.assertEquals(1, validationErrors.size()); Assert.assertEquals( "terminate task must have an terminationStatus parameter and must be set to COMPLETED or FAILED, taskName: terminate_task", validationErrors.get(0)); } @Test public void testWorkflowTaskTypeTerminateOptional() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_TERMINATE); workflowTask.setName("terminate_task"); workflowTask.setInputParameters( Collections.singletonMap(Terminate.getTerminationStatusParameter(), "COMPLETED")); workflowTask.setOptional(true); List validationErrors = getErrorMessages(workflowTask); Assert.assertEquals(1, validationErrors.size()); Assert.assertEquals( "terminate task cannot be optional, taskName: terminate_task", validationErrors.get(0)); } @Test public void testWorkflowTaskTypeTerminateValid() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType(TaskType.TASK_TYPE_TERMINATE); workflowTask.setName("terminate_task"); workflowTask.setInputParameters( Collections.singletonMap(Terminate.getTerminationStatusParameter(), "COMPLETED")); List validationErrors = getErrorMessages(workflowTask); Assert.assertEquals(0, validationErrors.size()); } @Test public void testWorkflowTaskTypeKafkaPublish() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("KAFKA_PUBLISH"); workflowTask.getInputParameters().put("kafka_request", "testInput"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeKafkaPublishWithRequestParamMissing() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("KAFKA_PUBLISH"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "inputParameters.kafka_request field is required for taskType: KAFKA_PUBLISH taskName: encode")); } @Test public void testWorkflowTaskTypeKafkaPublishWithKafkaParamInTaskDef() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("KAFKA_PUBLISH"); TaskDef taskDef = new TaskDef(); taskDef.setName("encode"); taskDef.getInputTemplate().put("kafka_request", "test_kafka_request"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(taskDef); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeKafkaPublishWithRequestParamInTaskDefAndWorkflowTask() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("KAFKA_PUBLISH"); workflowTask.getInputParameters().put("kafka_request", "http://www.netflix.com"); TaskDef taskDef = new TaskDef(); taskDef.setName("encode"); taskDef.getInputTemplate().put("kafka_request", "test Kafka Request"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(taskDef); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeJSONJQTransform() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("JSON_JQ_TRANSFORM"); workflowTask.getInputParameters().put("queryExpression", "."); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } @Test public void testWorkflowTaskTypeJSONJQTransformWithQueryParamMissing() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("JSON_JQ_TRANSFORM"); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(new TaskDef()); Set> result = validator.validate(workflowTask); assertEquals(1, result.size()); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); assertTrue( validationErrors.contains( "inputParameters.queryExpression field is required for taskType: JSON_JQ_TRANSFORM taskName: encode")); } @Test public void testWorkflowTaskTypeJSONJQTransformWithQueryParamInTaskDef() { WorkflowTask workflowTask = createSampleWorkflowTask(); workflowTask.setType("JSON_JQ_TRANSFORM"); TaskDef taskDef = new TaskDef(); taskDef.setName("encode"); taskDef.getInputTemplate().put("queryExpression", "."); when(mockMetadataDao.getTaskDef(anyString())).thenReturn(taskDef); Set> result = validator.validate(workflowTask); assertEquals(0, result.size()); } private List getErrorMessages(WorkflowTask workflowTask) { Set> result = validator.validate(workflowTask); List validationErrors = new ArrayList<>(); result.forEach(e -> validationErrors.add(e.getMessage())); return validationErrors; } private WorkflowTask createSampleWorkflowTask() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("encode"); workflowTask.setTaskReferenceName("encode"); workflowTask.setType("FORK_JOIN_DYNAMIC"); Map inputParam = new HashMap<>(); inputParam.put("fileLocation", "${workflow.input.fileLocation}"); workflowTask.setInputParameters(inputParam); return workflowTask; } } ================================================ FILE: core/src/test/resources/completed.json ================================================ { "ownerApp": "cpeworkflowtests", "createTime": 1547430586952, "updateTime": 1547430613550, "status": "COMPLETED", "endTime": 1547430613550, "workflowId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "tasks": [ { "taskType": "perf_task_1", "status": "COMPLETED", "inputData": { "mod": "0", "oddEven": "0" }, "referenceTaskName": "perf_task_1", "retryCount": 0, "seq": 1, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_1", "scheduledTime": 1547430586967, "startTime": 1547430589848, "endTime": 1547430589873, "updateTime": 1547430613560, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "485fdbdf-9f49-4879-9471-4722225e5613", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-0618a1a5e9526c9a1", "outputData": { "mod": "8", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_1", "taskReferenceName": "perf_task_1", "inputParameters": { "mod": "workflow.input.mod", "oddEven": "workflow.input.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389709, "createdBy": "CPEWORKFLOW", "name": "perf_task_1", "description": "perf_task_1", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 2881, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:49:867 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_1,1", "01/14/19, 01:49:49:867 : Starting to execute perf_task_1, id=485fdbdf-9f49-4879-9471-4722225e5613", "01/14/19, 01:49:49:867 : failure probability is 0.3066777 against 0.0", "01/14/19, 01:49:49:868 : Marking task completed" ] }, { "taskType": "perf_task_10", "status": "COMPLETED", "inputData": { "taskToExecute": "perf_task_10" }, "referenceTaskName": "perf_task_2", "retryCount": 0, "seq": 2, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_10", "scheduledTime": 1547430589900, "startTime": 1547430590465, "endTime": 1547430590499, "updateTime": 1547430613572, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "14988072-378d-4b6c-a596-09db9c88c5d1", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-07f2166099c597efe", "outputData": { "mod": "0", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_10", "taskReferenceName": "perf_task_2", "inputParameters": { "taskToExecute": "workflow.input.task2Name" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389226, "createdBy": "CPEWORKFLOW", "name": "perf_task_10", "description": "perf_task_10", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 565, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:50:489 : Starting to execute perf_task_10, id=14988072-378d-4b6c-a596-09db9c88c5d1", "01/14/19, 01:49:50:489 : failure probability is 0.040783882 against 0.0", "01/14/19, 01:49:50:489 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_2,1", "01/14/19, 01:49:50:490 : Marking task completed" ] }, { "taskType": "perf_task_3", "status": "COMPLETED", "inputData": { "mod": "0", "oddEven": "0" }, "referenceTaskName": "perf_task_3", "retryCount": 0, "seq": 3, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_3", "scheduledTime": 1547430590531, "startTime": 1547430591460, "endTime": 1547430591488, "updateTime": 1547430613582, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "91b6ba4c-c414-4cb1-a2e7-18edd7aa22fd", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-0618a1a5e9526c9a1", "outputData": { "mod": "9", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_3", "taskReferenceName": "perf_task_3", "inputParameters": { "mod": "perf_task_2.output.mod", "oddEven": "perf_task_2.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389814, "createdBy": "CPEWORKFLOW", "name": "perf_task_3", "description": "perf_task_3", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 929, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:51:477 : Starting to execute perf_task_3, id=91b6ba4c-c414-4cb1-a2e7-18edd7aa22fd", "01/14/19, 01:49:51:477 : failure probability is 0.9401053 against 0.0", "01/14/19, 01:49:51:477 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_3,1", "01/14/19, 01:49:51:479 : Marking task completed" ] }, { "taskType": "HTTP", "status": "COMPLETED", "inputData": { "http_request": { "uri": "/wfe_perf/workflow/_search?q=status:RUNNING&size=0&devint", "method": "GET", "vipAddress": "es_conductor.netflix.com" } }, "referenceTaskName": "get_es_1", "retryCount": 0, "seq": 4, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "get_from_es", "scheduledTime": 1547430591524, "startTime": 1547430591961, "endTime": 1547430592238, "updateTime": 1547430613601, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "b8095fef-0028-4fa3-a2a2-6e59c224bb7d", "callbackAfterSeconds": 0, "workerId": "i-01815a305a47fb626", "outputData": { "response": { "headers": { "Content-Length": [ "121" ], "Content-Type": [ "application/json; charset=UTF-8" ] }, "reasonPhrase": "OK", "body": { "took": 2, "timed_out": false, "_shards": { "total": 6, "successful": 6, "failed": 0 }, "hits": { "total": 0, "max_score": 0, "hits": [] } }, "statusCode": 200 } }, "workflowTask": { "name": "get_from_es", "taskReferenceName": "get_es_1", "type": "HTTP", "startDelay": 0, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "workflowPriority": 0, "queueWaitTime": 437, "taskDefinition": { "present": false }, "taskStatus": "COMPLETED", "logs": [] }, { "taskType": "DECISION", "status": "COMPLETED", "inputData": { "hasChildren": "true", "case": "1" }, "referenceTaskName": "oddEvenDecision", "retryCount": 0, "seq": 5, "correlationId": "1547430586940", "pollCount": 0, "taskDefName": "DECISION", "scheduledTime": 1547430592280, "startTime": 1547430592292, "endTime": 1547430592284, "updateTime": 1547430613614, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "5c2d843a-8320-4b6c-9765-e91bff433dba", "callbackAfterSeconds": 0, "outputData": { "caseOutput": [ "1" ] }, "workflowTask": { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "perf_task_3.output.oddEven" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "perf_task_4", "taskReferenceName": "perf_task_4", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390494, "createdBy": "CPEWORKFLOW", "name": "perf_task_4", "description": "perf_task_4", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "perf_task_4.output.dynamicTasks", "input": "perf_task_4.output.inputs" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "perf_task_5", "taskReferenceName": "perf_task_5", "inputParameters": { "mod": "perf_task_4.output.mod", "oddEven": "perf_task_4.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390611, "createdBy": "CPEWORKFLOW", "name": "perf_task_5", "description": "perf_task_5", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_6", "taskReferenceName": "perf_task_6", "inputParameters": { "mod": "perf_task_5.output.mod", "oddEven": "perf_task_5.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390789, "createdBy": "CPEWORKFLOW", "name": "perf_task_6", "description": "perf_task_6", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "1": [ { "name": "perf_task_7", "taskReferenceName": "perf_task_7", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390955, "createdBy": "CPEWORKFLOW", "name": "perf_task_7", "description": "perf_task_7", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_8", "taskReferenceName": "perf_task_8", "inputParameters": { "mod": "perf_task_7.output.mod", "oddEven": "perf_task_7.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391122, "createdBy": "CPEWORKFLOW", "name": "perf_task_8", "description": "perf_task_8", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_9", "taskReferenceName": "perf_task_9", "inputParameters": { "mod": "perf_task_8.output.mod", "oddEven": "perf_task_8.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391291, "createdBy": "CPEWORKFLOW", "name": "perf_task_9", "description": "perf_task_9", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "perf_task_8.output.mod" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "perf_task_12", "taskReferenceName": "perf_task_12", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389427, "createdBy": "CPEWORKFLOW", "name": "perf_task_12", "description": "perf_task_12", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_13", "taskReferenceName": "perf_task_13", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389276, "createdBy": "CPEWORKFLOW", "name": "perf_task_13", "description": "perf_task_13", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "1": [ { "name": "perf_task_15", "taskReferenceName": "perf_task_15", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388963, "createdBy": "CPEWORKFLOW", "name": "perf_task_15", "description": "perf_task_15", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_16", "taskReferenceName": "perf_task_16", "inputParameters": { "mod": "perf_task_15.output.mod", "oddEven": "perf_task_15.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389067, "createdBy": "CPEWORKFLOW", "name": "perf_task_16", "description": "perf_task_16", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "4": [ { "name": "perf_task_18", "taskReferenceName": "perf_task_18", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388904, "createdBy": "CPEWORKFLOW", "name": "perf_task_18", "description": "perf_task_18", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_19", "taskReferenceName": "perf_task_19", "inputParameters": { "mod": "perf_task_18.output.mod", "oddEven": "perf_task_18.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389173, "createdBy": "CPEWORKFLOW", "name": "perf_task_19", "description": "perf_task_19", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "5": [ { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390669, "createdBy": "CPEWORKFLOW", "name": "perf_task_21", "description": "perf_task_21", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "perf_task_21.output.mod", "oddEven": "perf_task_21.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391345, "createdBy": "CPEWORKFLOW", "name": "perf_task_22", "description": "perf_task_22", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ] }, "defaultCase": [ { "name": "perf_task_24", "taskReferenceName": "perf_task_24", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391074, "createdBy": "CPEWORKFLOW", "name": "perf_task_24", "description": "perf_task_24", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_25", "taskReferenceName": "perf_task_25", "inputParameters": { "mod": "perf_task_24.output.mod", "oddEven": "perf_task_24.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391177, "createdBy": "CPEWORKFLOW", "name": "perf_task_25", "description": "perf_task_25", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "startDelay": 0, "optional": false, "asyncComplete": false } ] }, "startDelay": 0, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 12, "taskDefinition": { "present": false }, "taskStatus": "COMPLETED", "logs": [] }, { "taskType": "perf_task_7", "status": "COMPLETED", "inputData": { "mod": "9", "oddEven": "1" }, "referenceTaskName": "perf_task_7", "retryCount": 0, "seq": 6, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_7", "scheduledTime": 1547430592287, "startTime": 1547430593603, "endTime": 1547430593641, "updateTime": 1547430613624, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "10efe69b-691f-49c6-9bce-42ba08ff4d2e", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "5", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_7", "taskReferenceName": "perf_task_7", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390955, "createdBy": "CPEWORKFLOW", "name": "perf_task_7", "description": "perf_task_7", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 1316, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:53:622 : Starting to execute perf_task_7, id=10efe69b-691f-49c6-9bce-42ba08ff4d2e", "01/14/19, 01:49:53:622 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_7,1", "01/14/19, 01:49:53:622 : failure probability is 0.62726057 against 0.0", "01/14/19, 01:49:53:625 : Marking task completed" ] }, { "taskType": "perf_task_8", "status": "COMPLETED", "inputData": { "mod": "5", "oddEven": "1" }, "referenceTaskName": "perf_task_8", "retryCount": 0, "seq": 7, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_8", "scheduledTime": 1547430593685, "startTime": 1547430594976, "endTime": 1547430595009, "updateTime": 1547430613634, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "51020906-8fe0-4993-9020-66a081847bf3", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "5", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_8", "taskReferenceName": "perf_task_8", "inputParameters": { "mod": "perf_task_7.output.mod", "oddEven": "perf_task_7.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391122, "createdBy": "CPEWORKFLOW", "name": "perf_task_8", "description": "perf_task_8", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 1291, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:54:994 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_8,1", "01/14/19, 01:49:54:994 : failure probability is 0.017497659 against 0.0", "01/14/19, 01:49:54:994 : Starting to execute perf_task_8, id=51020906-8fe0-4993-9020-66a081847bf3", "01/14/19, 01:49:54:995 : Marking task completed" ] }, { "taskType": "perf_task_9", "status": "COMPLETED", "inputData": { "mod": "5", "oddEven": "1" }, "referenceTaskName": "perf_task_9", "retryCount": 0, "seq": 8, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_9", "scheduledTime": 1547430595069, "startTime": 1547430596047, "endTime": 1547430596081, "updateTime": 1547430613642, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "c82cf62f-9f48-46c0-ae32-9bbfad57e71f", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "5", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_9", "taskReferenceName": "perf_task_9", "inputParameters": { "mod": "perf_task_8.output.mod", "oddEven": "perf_task_8.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391291, "createdBy": "CPEWORKFLOW", "name": "perf_task_9", "description": "perf_task_9", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 978, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:56:065 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_9,1", "01/14/19, 01:49:56:065 : Marking task completed", "01/14/19, 01:49:56:065 : Starting to execute perf_task_9, id=c82cf62f-9f48-46c0-ae32-9bbfad57e71f", "01/14/19, 01:49:56:065 : failure probability is 0.7340754 against 0.0" ] }, { "taskType": "DECISION", "status": "COMPLETED", "inputData": { "hasChildren": "true", "case": "5" }, "referenceTaskName": "modDecision", "retryCount": 0, "seq": 9, "correlationId": "1547430586940", "pollCount": 0, "taskDefName": "DECISION", "scheduledTime": 1547430596122, "startTime": 1547430596133, "endTime": 1547430596125, "updateTime": 1547430613650, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "597b18b6-6d99-4356-b205-dbe532fc7983", "callbackAfterSeconds": 0, "outputData": { "caseOutput": [ "5" ] }, "workflowTask": { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "perf_task_8.output.mod" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "perf_task_12", "taskReferenceName": "perf_task_12", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389427, "createdBy": "CPEWORKFLOW", "name": "perf_task_12", "description": "perf_task_12", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_13", "taskReferenceName": "perf_task_13", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389276, "createdBy": "CPEWORKFLOW", "name": "perf_task_13", "description": "perf_task_13", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "1": [ { "name": "perf_task_15", "taskReferenceName": "perf_task_15", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388963, "createdBy": "CPEWORKFLOW", "name": "perf_task_15", "description": "perf_task_15", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_16", "taskReferenceName": "perf_task_16", "inputParameters": { "mod": "perf_task_15.output.mod", "oddEven": "perf_task_15.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389067, "createdBy": "CPEWORKFLOW", "name": "perf_task_16", "description": "perf_task_16", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "4": [ { "name": "perf_task_18", "taskReferenceName": "perf_task_18", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388904, "createdBy": "CPEWORKFLOW", "name": "perf_task_18", "description": "perf_task_18", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_19", "taskReferenceName": "perf_task_19", "inputParameters": { "mod": "perf_task_18.output.mod", "oddEven": "perf_task_18.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389173, "createdBy": "CPEWORKFLOW", "name": "perf_task_19", "description": "perf_task_19", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "5": [ { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390669, "createdBy": "CPEWORKFLOW", "name": "perf_task_21", "description": "perf_task_21", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "perf_task_21.output.mod", "oddEven": "perf_task_21.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391345, "createdBy": "CPEWORKFLOW", "name": "perf_task_22", "description": "perf_task_22", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ] }, "defaultCase": [ { "name": "perf_task_24", "taskReferenceName": "perf_task_24", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391074, "createdBy": "CPEWORKFLOW", "name": "perf_task_24", "description": "perf_task_24", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_25", "taskReferenceName": "perf_task_25", "inputParameters": { "mod": "perf_task_24.output.mod", "oddEven": "perf_task_24.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391177, "createdBy": "CPEWORKFLOW", "name": "perf_task_25", "description": "perf_task_25", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "startDelay": 0, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 11, "taskDefinition": { "present": false }, "taskStatus": "COMPLETED", "logs": [] }, { "taskType": "perf_task_21", "status": "COMPLETED", "inputData": { "mod": "5", "oddEven": "1" }, "referenceTaskName": "perf_task_21", "retryCount": 0, "seq": 10, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_21", "scheduledTime": 1547430596128, "startTime": 1547430597361, "endTime": 1547430597400, "updateTime": 1547430613663, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "f44f4598-7623-46db-a513-75000ccf39b8", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "2", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390669, "createdBy": "CPEWORKFLOW", "name": "perf_task_21", "description": "perf_task_21", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 1233, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:49:57:378 : Starting to execute perf_task_21, id=f44f4598-7623-46db-a513-75000ccf39b8", "01/14/19, 01:49:57:378 : failure probability is 0.88135785 against 0.0", "01/14/19, 01:49:57:378 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_21,1", "01/14/19, 01:49:57:383 : Marking task completed" ] }, { "taskType": "SUB_WORKFLOW", "status": "COMPLETED", "inputData": { "workflowInput": {}, "subWorkflowId": "e18f09cb-9b3e-4296-bc77-87339d2eb34c", "subWorkflowName": "sub_flow_1", "subWorkflowVersion": 1 }, "referenceTaskName": "wf3", "retryCount": 0, "seq": 11, "correlationId": "1547430586940", "pollCount": 0, "taskDefName": "sub_workflow_x", "scheduledTime": 1547430606665, "startTime": 1547430597443, "endTime": 1547430606672, "updateTime": 1547430613674, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "37514448-8b14-4d5e-8483-0eabd89b73f6", "callbackAfterSeconds": 0, "outputData": { "subWorkflowId": "e18f09cb-9b3e-4296-bc77-87339d2eb34c", "mod": null, "oddEven": null, "es2statuses": [] }, "workflowTask": { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": -9222, "taskDefinition": { "present": false }, "taskStatus": "COMPLETED", "logs": [] }, { "taskType": "perf_task_22", "status": "COMPLETED", "inputData": { "mod": "2", "oddEven": "0" }, "referenceTaskName": "perf_task_22", "retryCount": 0, "seq": 12, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_22", "scheduledTime": 1547430606701, "startTime": 1547430607444, "endTime": 1547430607481, "updateTime": 1547430613684, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "f2448612-4960-4717-84f7-6686434733fe", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "2", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "perf_task_21.output.mod", "oddEven": "perf_task_21.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391345, "createdBy": "CPEWORKFLOW", "name": "perf_task_22", "description": "perf_task_22", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 743, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:50:07:462 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_22,1", "01/14/19, 01:50:07:462 : Marking task completed", "01/14/19, 01:50:07:462 : Starting to execute perf_task_22, id=f2448612-4960-4717-84f7-6686434733fe", "01/14/19, 01:50:07:462 : failure probability is 0.6165708 against 0.0" ] }, { "taskType": "perf_task_28", "status": "COMPLETED", "inputData": { "mod": "9", "oddEven": "1" }, "referenceTaskName": "perf_task_28", "retryCount": 0, "seq": 13, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_28", "scheduledTime": 1547430607541, "startTime": 1547430608584, "endTime": 1547430608631, "updateTime": 1547430613694, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "f44c0a56-ae5b-4aba-ac69-c9f48ad6ecfc", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-0618a1a5e9526c9a1", "outputData": { "mod": "8", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_28", "taskReferenceName": "perf_task_28", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390042, "createdBy": "CPEWORKFLOW", "name": "perf_task_28", "description": "perf_task_28", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 1043, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:50:08:605 : Starting to execute perf_task_28, id=f44c0a56-ae5b-4aba-ac69-c9f48ad6ecfc", "01/14/19, 01:50:08:605 : failure probability is 0.8953033 against 0.0", "01/14/19, 01:50:08:605 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_28,1", "01/14/19, 01:50:08:608 : Marking task completed" ] }, { "taskType": "perf_task_29", "status": "COMPLETED", "inputData": { "mod": "8", "oddEven": "0" }, "referenceTaskName": "perf_task_29", "retryCount": 0, "seq": 14, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_29", "scheduledTime": 1547430608681, "startTime": 1547430611220, "endTime": 1547430611262, "updateTime": 1547430613702, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "ff3961e9-a7cf-454e-a5a5-31d9582fc3be", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-075e5e67066be5d52", "outputData": { "mod": "0", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_29", "taskReferenceName": "perf_task_29", "inputParameters": { "mod": "perf_task_28.output.mod", "oddEven": "perf_task_28.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390098, "createdBy": "CPEWORKFLOW", "name": "perf_task_29", "description": "perf_task_29", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 2539, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:50:11:238 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_29,1", "01/14/19, 01:50:11:238 : Starting to execute perf_task_29, id=ff3961e9-a7cf-454e-a5a5-31d9582fc3be", "01/14/19, 01:50:11:238 : failure probability is 0.3055073 against 0.0", "01/14/19, 01:50:11:240 : Marking task completed" ] }, { "taskType": "perf_task_30", "status": "COMPLETED", "inputData": { "mod": "0", "oddEven": "0" }, "referenceTaskName": "perf_task_30", "retryCount": 0, "seq": 15, "correlationId": "1547430586940", "pollCount": 1, "taskDefName": "perf_task_30", "scheduledTime": 1547430611308, "startTime": 1547430613454, "endTime": 1547430613496, "updateTime": 1547430613712, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 300, "workflowInstanceId": "1be75865-00a1-4e2b-95c0-573c444d98d7", "workflowType": "performance_test_1", "taskId": "603a164f-3198-40ed-a5b6-7dd439349c25", "callbackAfterSeconds": 0, "workerId": "cpeworkflowtests-devint-i-0618a1a5e9526c9a1", "outputData": { "mod": "6", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_30", "taskReferenceName": "perf_task_30", "inputParameters": { "mod": "perf_task_29.output.mod", "oddEven": "perf_task_29.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069392094, "createdBy": "CPEWORKFLOW", "name": "perf_task_30", "description": "perf_task_30", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "queueWaitTime": 2146, "taskDefinition": { "present": true }, "taskStatus": "COMPLETED", "logs": [ "01/14/19, 01:50:13:473 : Starting to execute perf_task_30, id=603a164f-3198-40ed-a5b6-7dd439349c25", "01/14/19, 01:50:13:473 : Attempt 1be75865-00a1-4e2b-95c0-573c444d98d7,perf_task_30,1", "01/14/19, 01:50:13:473 : failure probability is 0.4859264 against 0.0", "01/14/19, 01:50:13:476 : Marking task completed" ] } ], "input": { "mod": "0", "oddEven": "0", "task2Name": "perf_task_10" }, "output": { "mod": "6", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": {}, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": null }, "joinOn": [], "sink": null, "optional": false, "taskDefinition": null, "rateLimited": null } ], "attempt": 1 }, "workflowType": "performance_test_1", "version": 1, "correlationId": "1547430586940", "schemaVersion": 1, "workflowDefinition": { "createTime": 1477681181098, "updateTime": 1484162039528, "name": "performance_test_1", "description": "performance_test_1", "version": 1, "tasks": [ { "name": "perf_task_1", "taskReferenceName": "perf_task_1", "inputParameters": { "mod": "workflow.input.mod", "oddEven": "workflow.input.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389709, "createdBy": "CPEWORKFLOW", "name": "perf_task_1", "description": "perf_task_1", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_10", "taskReferenceName": "perf_task_2", "inputParameters": { "taskToExecute": "workflow.input.task2Name" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389226, "createdBy": "CPEWORKFLOW", "name": "perf_task_10", "description": "perf_task_10", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_3", "taskReferenceName": "perf_task_3", "inputParameters": { "mod": "perf_task_2.output.mod", "oddEven": "perf_task_2.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389814, "createdBy": "CPEWORKFLOW", "name": "perf_task_3", "description": "perf_task_3", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "get_from_es", "taskReferenceName": "get_es_1", "type": "HTTP", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "perf_task_3.output.oddEven" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "perf_task_4", "taskReferenceName": "perf_task_4", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390494, "createdBy": "CPEWORKFLOW", "name": "perf_task_4", "description": "perf_task_4", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "perf_task_4.output.dynamicTasks", "input": "perf_task_4.output.inputs" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "perf_task_5", "taskReferenceName": "perf_task_5", "inputParameters": { "mod": "perf_task_4.output.mod", "oddEven": "perf_task_4.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390611, "createdBy": "CPEWORKFLOW", "name": "perf_task_5", "description": "perf_task_5", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_6", "taskReferenceName": "perf_task_6", "inputParameters": { "mod": "perf_task_5.output.mod", "oddEven": "perf_task_5.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390789, "createdBy": "CPEWORKFLOW", "name": "perf_task_6", "description": "perf_task_6", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "1": [ { "name": "perf_task_7", "taskReferenceName": "perf_task_7", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390955, "createdBy": "CPEWORKFLOW", "name": "perf_task_7", "description": "perf_task_7", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_8", "taskReferenceName": "perf_task_8", "inputParameters": { "mod": "perf_task_7.output.mod", "oddEven": "perf_task_7.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391122, "createdBy": "CPEWORKFLOW", "name": "perf_task_8", "description": "perf_task_8", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_9", "taskReferenceName": "perf_task_9", "inputParameters": { "mod": "perf_task_8.output.mod", "oddEven": "perf_task_8.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391291, "createdBy": "CPEWORKFLOW", "name": "perf_task_9", "description": "perf_task_9", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "perf_task_8.output.mod" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "perf_task_12", "taskReferenceName": "perf_task_12", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389427, "createdBy": "CPEWORKFLOW", "name": "perf_task_12", "description": "perf_task_12", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_13", "taskReferenceName": "perf_task_13", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389276, "createdBy": "CPEWORKFLOW", "name": "perf_task_13", "description": "perf_task_13", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "1": [ { "name": "perf_task_15", "taskReferenceName": "perf_task_15", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388963, "createdBy": "CPEWORKFLOW", "name": "perf_task_15", "description": "perf_task_15", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_16", "taskReferenceName": "perf_task_16", "inputParameters": { "mod": "perf_task_15.output.mod", "oddEven": "perf_task_15.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389067, "createdBy": "CPEWORKFLOW", "name": "perf_task_16", "description": "perf_task_16", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false } ], "4": [ { "name": "perf_task_18", "taskReferenceName": "perf_task_18", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069388904, "createdBy": "CPEWORKFLOW", "name": "perf_task_18", "description": "perf_task_18", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_19", "taskReferenceName": "perf_task_19", "inputParameters": { "mod": "perf_task_18.output.mod", "oddEven": "perf_task_18.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069389173, "createdBy": "CPEWORKFLOW", "name": "perf_task_19", "description": "perf_task_19", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "5": [ { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390669, "createdBy": "CPEWORKFLOW", "name": "perf_task_21", "description": "perf_task_21", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "perf_task_21.output.mod", "oddEven": "perf_task_21.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391345, "createdBy": "CPEWORKFLOW", "name": "perf_task_22", "description": "perf_task_22", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ] }, "defaultCase": [ { "name": "perf_task_24", "taskReferenceName": "perf_task_24", "inputParameters": { "mod": "perf_task_9.output.mod", "oddEven": "perf_task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391074, "createdBy": "CPEWORKFLOW", "name": "perf_task_24", "description": "perf_task_24", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "perf_task_12.output.mod", "oddEven": "perf_task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 }, "optional": false, "asyncComplete": false }, { "name": "perf_task_25", "taskReferenceName": "perf_task_25", "inputParameters": { "mod": "perf_task_24.output.mod", "oddEven": "perf_task_24.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069391177, "createdBy": "CPEWORKFLOW", "name": "perf_task_25", "description": "perf_task_25", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "startDelay": 0, "optional": false, "asyncComplete": false } ] }, "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "perf_task_28", "taskReferenceName": "perf_task_28", "inputParameters": { "mod": "perf_task_3.output.mod", "oddEven": "perf_task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390042, "createdBy": "CPEWORKFLOW", "name": "perf_task_28", "description": "perf_task_28", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_29", "taskReferenceName": "perf_task_29", "inputParameters": { "mod": "perf_task_28.output.mod", "oddEven": "perf_task_28.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069390098, "createdBy": "CPEWORKFLOW", "name": "perf_task_29", "description": "perf_task_29", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false }, { "name": "perf_task_30", "taskReferenceName": "perf_task_30", "inputParameters": { "mod": "perf_task_29.output.mod", "oddEven": "perf_task_29.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "optional": false, "taskDefinition": { "createTime": 1547069392094, "createdBy": "CPEWORKFLOW", "name": "perf_task_30", "description": "perf_task_30", "retryCount": 2, "timeoutSeconds": 600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "asyncComplete": false } ], "schemaVersion": 1, "restartable": true, "workflowStatusListenerEnabled": false }, "priority": 0, "workflowName": "performance_test_1", "workflowVersion": 1, "startTime": 1547430586952 } ================================================ FILE: core/src/test/resources/conditional_flow.json ================================================ { "name": "ConditionalTaskWF", "description": "ConditionalTaskWF", "version": 1, "tasks": [{ "name": "conditional", "taskReferenceName": "conditional", "inputParameters": { "case": "${workflow.input.param1}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "nested": [{ "name": "conditional2", "taskReferenceName": "conditional2", "inputParameters": { "case": "${workflow.input.param2}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "one": [{ "name": "junit_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_1", "description": "junit_task_1", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "junit_task_3", "taskReferenceName": "t3", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_3", "description": "junit_task_3", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ], "two": [{ "name": "junit_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp3": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_2", "description": "junit_task_2", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }] }, "startDelay": 0 }], "three": [{ "name": "junit_task_3", "taskReferenceName": "t31", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_3", "description": "junit_task_3", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }] }, "defaultCase": [{ "name": "junit_task_2", "taskReferenceName": "t21", "inputParameters": { "tp1": "${workflow.input.param1}", "tp3": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_2", "description": "junit_task_2", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }], "startDelay": 0 }, { "name": "finalcondition", "taskReferenceName": "tf", "inputParameters": { "finalCase": "{workflow.input.finalCase}" }, "type": "DECISION", "caseValueParam": "finalCase", "decisionCases": { "notify": [{ "name": "junit_task_4", "taskReferenceName": "junit_task_4", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_4", "description": "junit_task_4", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }] }, "startDelay": 0 } ], "inputParameters": [ "param1", "param2" ], "schemaVersion": 2, "ownerEmail": "unit@test.com" } ================================================ FILE: core/src/test/resources/conditional_flow_with_switch.json ================================================ { "name": "ConditionalTaskWF", "description": "ConditionalTaskWF", "version": 1, "tasks": [ { "name": "conditional", "taskReferenceName": "conditional", "inputParameters": { "case": "${workflow.input.param1}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "nested": [ { "name": "conditional2", "taskReferenceName": "conditional2", "inputParameters": { "case": "${workflow.input.param2}" }, "type": "SWITCH", "evaluatorType": "javascript", "expression": "$.case == 'one' ? 'one' : ($.case == 'two' ? 'two' : ($.case == 'three' ? 'three' : 'other'))", "decisionCases": { "one": [ { "name": "junit_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_1", "description": "junit_task_1", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "junit_task_3", "taskReferenceName": "t3", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_3", "description": "junit_task_3", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ], "two": [ { "name": "junit_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp3": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_2", "description": "junit_task_2", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ] }, "startDelay": 0 } ], "three": [ { "name": "junit_task_3", "taskReferenceName": "t31", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_3", "description": "junit_task_3", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ] }, "defaultCase": [ { "name": "junit_task_2", "taskReferenceName": "t21", "inputParameters": { "tp1": "${workflow.input.param1}", "tp3": "${workflow.input.param2}" }, "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_2", "description": "junit_task_2", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ], "startDelay": 0 }, { "name": "finalcondition", "taskReferenceName": "tf", "inputParameters": { "finalCase": "{workflow.input.finalCase}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "finalCase", "decisionCases": { "notify": [ { "name": "junit_task_4", "taskReferenceName": "junit_task_4", "type": "SIMPLE", "startDelay": 0, "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "junit_task_4", "description": "junit_task_4", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ] }, "startDelay": 0 } ], "inputParameters": [ "param1", "param2" ], "schemaVersion": 2, "ownerEmail": "unit@test.com" } ================================================ FILE: core/src/test/resources/payload.json ================================================ { "imageType": "TEST_SAMPLE", "filteredSourceList": { "TEST_SAMPLE": [ { "sourceId": "1413900_10830", "url": "file/location/a0bdc4d0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_50241", "url": "file/location/cd4e00a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-55ee8663-85c2-42d3-aca2-4076707e6d4e", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-14056154-1544-4350-81db-b3751fe44777", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-0b0ae5ea-d5c5-410c-adc9-bf16d2909c2e", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-08869779-614d-417c-bfea-36a3f8f199da", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-e117db45-1c48-45d0-b751-89386eb2d81d", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0221421-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/4a009209-002f-4b58-8b96-cb2198f8ba3c" }, { "sourceId": "f0252161-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/55b56298-5e7a-4949-b919-88c5c9557e8e" }, { "sourceId": "f038d070-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/3c4804f4-e826-436f-90c9-52b8d9266d52" }, { "sourceId": "f04e0621-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/689283a1-1816-48ef-83da-7f9ac874bf45" }, { "sourceId": "f04ddf10-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/586666ae-7321-445a-80b6-323c8c241ecd" }, { "sourceId": "f05950c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/31795cc4-2590-4b20-a617-deaa18301f99" }, { "sourceId": "1413900_46819", "url": "file/location/c74497a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_11177", "url": "file/location/a231c730-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48713", "url": "file/location/ca638ae0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48525", "url": "file/location/ca0c9140-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_73303", "url": "file/location/d5943a40-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_55202", "url": "file/location/d1a4d7a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-61413adf-3c10-4484-b25d-e238df898f45", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-addca397-f050-4339-ae86-9ba8c4e1b0d5", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e4de9810-0f69-4593-8926-01ed82cbebcb", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e16e2074-7af6-4700-ab05-ca41ba9c9ab4", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-341c86f8-57a5-40e1-8842-3eb41dd9f528", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-88c2ea9b-cef7-4120-8043-b92713d8fade", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3f6a731f-3c92-4677-9923-f80b8a6be632", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-1508b871-64de-47ce-8b07-76c5cb3f3e1e", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "generated-1406dce8-7b9c-4956-a7e8-78721c476ce9", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "f0206671-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35ebee36-3072-44c5-abb5-702a5a3b1a91" }, { "sourceId": "f01f5501-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d3a9133d-c681-4910-a769-8195526ae634" }, { "sourceId": "f022b060-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8fc1413d-170e-4644-a554-5e0c596b225c" }, { "sourceId": "f02fa8b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35bed0a2-7def-457b-bded-4f4d7d94f76e" }, { "sourceId": "f031f2a0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a5a2ea1f-8d13-429c-a44d-3057d21f608a" }, { "sourceId": "f0424650-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/1c599ffc-4f10-4c0b-8d9a-ae41c7256113" }, { "sourceId": "f04ec970-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8404a421-e1a6-41cf-af63-a35ccb474457" }, { "sourceId": "1413900_47197", "url": "file/location/c81b6fa0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-2a63c0c8-62ea-44a4-a33b-f0b3047e8b00", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-b27face7-3589-4209-944a-5153b20c5996", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-144675b3-9321-48d2-8b5b-e19a40d30ef2", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-8cbe821e-b1fb-48ce-beb5-735319af4db6", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-ecc4ea47-9bad-4b91-97c7-35f4ea6fb479", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-c1eb9ed0-8560-4e09-a748-f926edb7cdc2", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-6bed81fd-c777-4c61-8da1-0bb7f7cf0082", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-852e5510-dd5d-4900-a614-854148fcc716", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-f4dedcb7-37c9-4ba9-ab37-64ec9be7c882", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0259691-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/721bc0de-e75f-4386-8b2e-ca84eb653596" }, { "sourceId": "f02b3be1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d2043b17-8ce5-42ee-a5e4-81c68f0c4838" }, { "sourceId": "f02b62f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/63931561-3b5b-4ffe-af47-da2c9de94684" }, { "sourceId": "f0315660-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d99ed629-2885-4e4a-8a1b-22e487b875fa" }, { "sourceId": "f0306c00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6f8e673a-7003-44aa-96b9-e2ed8a4654ff" }, { "sourceId": "f033c760-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/627c00f9-14b3-4057-b6e2-0f962ad0308e" }, { "sourceId": "f03526f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fafabaf9-fe58-4a9a-b555-026521aeb2fe" }, { "sourceId": "f03acc41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6c9fed2c-558a-4db3-8360-659b5e8c46e4" }, { "sourceId": "f0463df1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e9fb83d2-5f14-4442-92b5-67e613f2e35f" }, { "sourceId": "f04fb3d0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e7a0f82f-be8d-4ada-a4b1-13e8165e08be" }, { "sourceId": "f05272f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9aba488a-22b3-4932-85a7-52c461203541" }, { "sourceId": "f0581841-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/457415f6-6d0c-4304-8533-0d5b43fac564" }, { "sourceId": "generated-8fefb48c-6fde-4fd6-8f33-a1f3f3b62105", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-30c61aa5-f5bd-4077-8c32-336b87acbe96", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-d5da37db-d486-46d4-8f7d-1e0710a77eb5", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-77af26fe-9e22-48af-99e3-f63f10fbe6de", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-2e807016-3d11-4b60-bec7-c380a608b67d", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-615d02e9-62c2-43ab-9df7-753b6b8e2c22", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3e1600fd-a626-4ee6-972b-5f0187e96c38", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-1dcb208c-6a58-4334-a60c-6fb54c8a2af5", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f024ac30-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0af2107b-4231-4d23-bef3-4e417ac6c5d3" }, { "sourceId": "f0282ea1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0f592681-fd23-4194-ae43-42f61c664485" }, { "sourceId": "f02c4d50-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ec46b9a3-99af-410a-af7d-726f8854909f" }, { "sourceId": "f02b8a00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aed7e5da-b524-4d41-b264-28ce615ec826" }, { "sourceId": "f02b14d1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/b88c9055-ab0d-4d27-a405-265ba2a15f0c" }, { "sourceId": "f03044f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb8c4df9-d59e-4ac3-880e-4ea94cd880a4" }, { "sourceId": "f034ffe1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/59f3fbe8-b300-4861-9b2f-dac7b15aea7d" }, { "sourceId": "f03c2bd0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/19a06d54-41ed-419d-9947-f10cd5f0d85c" }, { "sourceId": "f03fae41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a9a48a62-7d62-4f67-b281-cc6fdc1e722c" }, { "sourceId": "f0455390-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0aeffc0a-a5ad-46ff-abab-1b3bc6a5840a" }, { "sourceId": "f04b1ff1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9a08aaed-c125-48f7-9d1d-fd11266c2b12" }, { "sourceId": "f04cf4b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/17a6e0f9-aa64-411f-9af7-837c84f7443f" }, { "sourceId": "f0511360-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb633c73-cb33-4806-bc08-049024644856" }, { "sourceId": "f0538460-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a7012248-6769-42da-a6c8-d4b831f6efce" }, { "sourceId": "f058db91-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/bcf71522-6168-48c4-86c9-995bca60ae51" }, { "sourceId": "generated-adf005c4-95c1-4904-9968-09cc19a26bfe", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-c4d367a4-4cdc-412e-af79-09b227f2e3ba", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-48dba018-f884-49db-b87e-67274e244c8f", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "generated-26700b83-4892-420e-8b46-1ee21eba75fb", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-632f3198-c0dc-4348-974f-51684d4e443e", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-86e2dd1d-1aa4-4dbe-b37b-b488f5dd1c70", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f04134e0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ff8f59bf-7757-4d51-a7e4-619f3e8ffaf2" }, { "sourceId": "f04f65b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d66467d1-3ac6-4041-8d15-e722ee07231f" }, { "sourceId": "1413900_15255", "url": "file/location/a9e20260-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-e953493b-cbe3-4319-885e-00c82089c76c", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-65c54676-3adb-4ef0-b65e-8e2a49533cbf", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "f02ac6b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/21568877-07a5-411f-9715-5e92806c4448" }, { "sourceId": "f02fcfc1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/f3b1f1a2-48d3-475d-a607-2e5a1fe532e7" }, { "sourceId": "f03526f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/84a40c66-d925-4a4a-ba62-8491d26e29e9" }, { "sourceId": "f03e75c1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e84c00e8-a148-46cf-9a0b-431c4c2aeb08" }, { "sourceId": "f0429471-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/178de9fa-7cc8-457a-8fb6-5c080e6163ea" }, { "sourceId": "f047eba0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/18d153aa-e13b-4264-ae03-f3da75eb425b" }, { "sourceId": "f04fdae0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/7c843e53-8d87-47cf-bca5-1a02e7f5e33f" }, { "sourceId": "f0553210-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/26bacd65-9082-4d83-9506-90e5f1ccd16a" }, { "sourceId": "1413900_84904", "url": "file/location/d8f7b090-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-84adc784-8d7d-4088-ba51-16fde57fbc21", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-9e49c58b-0b33-4daf-a39a-8fc91e302328", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "f02dd3f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8937b328-8f0d-4762-8d1f-7d7bc80c3d2e" }, { "sourceId": "f03240c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aab6e386-4d59-4b40-b257-9aed12a45446" } ] } } ================================================ FILE: core/src/test/resources/test.json ================================================ { "ownerApp": "cpeworkflowtests", "createTime": 1505587453961, "updateTime": 1505588471071, "status": "RUNNING", "endTime": 0, "workflowId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "tasks": [ { "taskType": "perf_task_1", "status": "COMPLETED", "inputData": { "mod": "0", "oddEven": "0" }, "referenceTaskName": "perf_task_1", "retryCount": 0, "seq": 1, "correlationId": "1505587453950", "pollCount": 1, "taskDefName": "perf_task_1", "scheduledTime": 1505587453972, "startTime": 1505587455481, "endTime": 1505587455539, "updateTime": 1505587455539, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 3600, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "3a54e268-0054-4eab-aea2-e54d1b89896c", "callbackAfterSeconds": 0, "outputData": { "mod": "5", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_1", "taskReferenceName": "perf_task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, "queueWaitTime": 1509, "taskStatus": "COMPLETED" }, { "taskType": "perf_task_10", "status": "COMPLETED", "inputData": { "taskToExecute": "perf_task_10" }, "referenceTaskName": "perf_task_2", "retryCount": 0, "seq": 2, "correlationId": "1505587453950", "pollCount": 1, "taskDefName": "perf_task_10", "scheduledTime": 1505587455517, "startTime": 1505587457017, "endTime": 1505587457075, "updateTime": 1505587457075, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 3600, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "3731c3ee-f918-42b7-8bb3-fb016fc0ecae", "callbackAfterSeconds": 0, "outputData": { "mod": "1", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_10", "taskReferenceName": "perf_task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute", "startDelay": 0 }, "queueWaitTime": 1500, "taskStatus": "COMPLETED" }, { "taskType": "perf_task_3", "status": "COMPLETED", "inputData": { "mod": "1", "oddEven": "1" }, "referenceTaskName": "perf_task_3", "retryCount": 0, "seq": 3, "correlationId": "1505587453950", "pollCount": 1, "taskDefName": "perf_task_3", "scheduledTime": 1505587457064, "startTime": 1505587459498, "endTime": 1505587459560, "updateTime": 1505587459560, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 3600, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "738370d6-596f-4ae5-95bf-ca635c7f10dd", "callbackAfterSeconds": 0, "outputData": { "mod": "6", "oddEven": "0", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_3", "taskReferenceName": "perf_task_3", "inputParameters": { "mod": "${perf_task_2.output.mod}", "oddEven": "${perf_task_2.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, "queueWaitTime": 2434, "taskStatus": "COMPLETED" }, { "taskType": "HTTP", "status": "COMPLETED", "inputData": { "http_request": { "uri": "/wfe_perf/workflow/_search?q=status:RUNNING&size=0&beta", "method": "GET", "vipAddress": "es_cpe_wfe.us-east-1.cloud.netflix.com" } }, "referenceTaskName": "get_es_1", "retryCount": 0, "seq": 4, "correlationId": "1505587453950", "pollCount": 1, "taskDefName": "get_from_es", "scheduledTime": 1505587459547, "startTime": 1505587459996, "endTime": 1505587460250, "updateTime": 1505587460250, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "64b49d62-1dfb-4290-94d4-971b4d033f33", "callbackAfterSeconds": 0, "workerId": "i-04c53d07aba5b5e9c", "outputData": { "response": { "headers": { "Content-Length": [ "121" ], "Content-Type": [ "application/json; charset=UTF-8" ] }, "reasonPhrase": "OK", "body": { "took": 1, "timed_out": false, "_shards": { "total": 6, "successful": 6, "failed": 0 }, "hits": { "total": 1, "max_score": 0.0, "hits": [] } }, "statusCode": 200 } }, "workflowTask": { "name": "get_from_es", "taskReferenceName": "get_es_1", "type": "HTTP", "startDelay": 0 }, "queueWaitTime": 449, "taskStatus": "COMPLETED" }, { "taskType": "DECISION", "status": "COMPLETED", "inputData": { "hasChildren": "true", "case": "0" }, "referenceTaskName": "oddEvenDecision", "retryCount": 0, "seq": 5, "correlationId": "1505587453950", "pollCount": 0, "taskDefName": "DECISION", "scheduledTime": 1505587460216, "startTime": 1505587460241, "endTime": 1505587460274, "updateTime": 1505587460274, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "5a596a36-09eb-4a11-a952-01ab5a7c362f", "callbackAfterSeconds": 0, "outputData": { "caseOutput": [ "0" ] }, "workflowTask": { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${perf_task_3.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "perf_task_4", "taskReferenceName": "perf_task_4", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${perf_task_4.output.dynamicTasks}", "input": "${perf_task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input", "startDelay": 0 }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN", "startDelay": 0 }, { "name": "perf_task_5", "taskReferenceName": "perf_task_5", "inputParameters": { "mod": "${perf_task_4.output.mod}", "oddEven": "${perf_task_4.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_6", "taskReferenceName": "perf_task_6", "inputParameters": { "mod": "${perf_task_5.output.mod}", "oddEven": "${perf_task_5.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "1": [ { "name": "perf_task_7", "taskReferenceName": "perf_task_7", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_8", "taskReferenceName": "perf_task_8", "inputParameters": { "mod": "${perf_task_7.output.mod}", "oddEven": "${perf_task_7.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_9", "taskReferenceName": "perf_task_9", "inputParameters": { "mod": "${perf_task_8.output.mod}", "oddEven": "${perf_task_8.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "${perf_task_8.output.mod}" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "perf_task_12", "taskReferenceName": "perf_task_12", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_13", "taskReferenceName": "perf_task_13", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "1": [ { "name": "perf_task_15", "taskReferenceName": "perf_task_15", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_16", "taskReferenceName": "perf_task_16", "inputParameters": { "mod": "${perf_task_15.output.mod}", "oddEven": "${perf_task_15.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "4": [ { "name": "perf_task_18", "taskReferenceName": "perf_task_18", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_19", "taskReferenceName": "perf_task_19", "inputParameters": { "mod": "${perf_task_18.output.mod}", "oddEven": "${perf_task_18.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "5": [ { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "${perf_task_21.output.mod}", "oddEven": "${perf_task_21.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ] }, "defaultCase": [ { "name": "perf_task_24", "taskReferenceName": "perf_task_24", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "perf_task_25", "taskReferenceName": "perf_task_25", "inputParameters": { "mod": "${perf_task_24.output.mod}", "oddEven": "${perf_task_24.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "startDelay": 0 } ] }, "startDelay": 0 }, "queueWaitTime": 25, "taskStatus": "COMPLETED" }, { "taskType": "perf_task_4", "status": "COMPLETED", "inputData": { "mod": "6", "oddEven": "0" }, "referenceTaskName": "perf_task_4", "retryCount": 0, "seq": 6, "correlationId": "1505587453950", "pollCount": 1, "taskDefName": "perf_task_4", "scheduledTime": 1505587460234, "startTime": 1505587463699, "endTime": 1505587463718, "updateTime": 1505587463718, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 3600, "workflowInstanceId": "46e2d0d7-0809-40f2-9f22-bed9d41f6613", "taskId": "1bf3da08-9d16-4f8a-98c3-4a6efee0e03a", "callbackAfterSeconds": 0, "outputData": { "mod": "9", "oddEven": "1", "inputs": { "subflow_0": { "mod": 4, "oddEven": 0 }, "subflow_4": { "mod": 4, "oddEven": 0 }, "subflow_2": { "mod": 4, "oddEven": 0 } }, "dynamicTasks": [ { "name": null, "taskReferenceName": "subflow_0", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_2", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } }, { "name": null, "taskReferenceName": "subflow_4", "description": null, "inputParameters": null, "type": "SUB_WORKFLOW", "dynamicTaskNameParam": null, "caseValueParam": null, "caseExpression": null, "decisionCases": { }, "dynamicForkJoinTasksParam": null, "dynamicForkTasksParam": null, "dynamicForkTasksInputParamName": null, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": null, "optional": null, "subWorkflowParam": { "name": "sub_flow_1", "version": null } } ], "attempt": 1 }, "workflowTask": { "name": "perf_task_4", "taskReferenceName": "perf_task_4", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, "queueWaitTime": 3465, "taskStatus": "COMPLETED" } ], "input": { "mod": "0", "oddEven": "0", "task2Name": "perf_task_10" }, "workflowType": "performance_test_1", "version": 1, "correlationId": "1505587453950", "schemaVersion": 2, "taskToDomain": { "*": "beta" }, "startTime": 1505587453961, "workflowDefinition": { "createTime": 1477681181098, "updateTime": 1502738273998, "name": "performance_test_1", "description": "performance_test_1", "version": 1, "tasks": [ { "name": "perf_task_1", "taskReferenceName": "perf_task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "dyntask", "taskReferenceName": "perf_task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute", "startDelay": 0 }, { "name": "perf_task_3", "taskReferenceName": "perf_task_3", "inputParameters": { "mod": "${perf_task_2.output.mod}", "oddEven": "${perf_task_2.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "get_from_es", "taskReferenceName": "get_es_1", "type": "HTTP", "startDelay": 0 }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${perf_task_3.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "perf_task_4", "taskReferenceName": "perf_task_4", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${perf_task_4.output.dynamicTasks}", "input": "${perf_task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input", "startDelay": 0 }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN", "startDelay": 0 }, { "name": "perf_task_5", "taskReferenceName": "perf_task_5", "inputParameters": { "mod": "${perf_task_4.output.mod}", "oddEven": "${perf_task_4.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_6", "taskReferenceName": "perf_task_6", "inputParameters": { "mod": "${perf_task_5.output.mod}", "oddEven": "${perf_task_5.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "1": [ { "name": "perf_task_7", "taskReferenceName": "perf_task_7", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_8", "taskReferenceName": "perf_task_8", "inputParameters": { "mod": "${perf_task_7.output.mod}", "oddEven": "${perf_task_7.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_9", "taskReferenceName": "perf_task_9", "inputParameters": { "mod": "${perf_task_8.output.mod}", "oddEven": "${perf_task_8.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "${perf_task_8.output.mod}" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "perf_task_12", "taskReferenceName": "perf_task_12", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_13", "taskReferenceName": "perf_task_13", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "1": [ { "name": "perf_task_15", "taskReferenceName": "perf_task_15", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_16", "taskReferenceName": "perf_task_16", "inputParameters": { "mod": "${perf_task_15.output.mod}", "oddEven": "${perf_task_15.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "4": [ { "name": "perf_task_18", "taskReferenceName": "perf_task_18", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_19", "taskReferenceName": "perf_task_19", "inputParameters": { "mod": "${perf_task_18.output.mod}", "oddEven": "${perf_task_18.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "5": [ { "name": "perf_task_21", "taskReferenceName": "perf_task_21", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "perf_task_22", "taskReferenceName": "perf_task_22", "inputParameters": { "mod": "${perf_task_21.output.mod}", "oddEven": "${perf_task_21.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ] }, "defaultCase": [ { "name": "perf_task_24", "taskReferenceName": "perf_task_24", "inputParameters": { "mod": "${perf_task_9.output.mod}", "oddEven": "${perf_task_9.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${perf_task_12.output.mod}", "oddEven": "${perf_task_12.output.oddEven}" }, "type": "SUB_WORKFLOW", "startDelay": 0, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "perf_task_25", "taskReferenceName": "perf_task_25", "inputParameters": { "mod": "${perf_task_24.output.mod}", "oddEven": "${perf_task_24.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "startDelay": 0 } ] }, "startDelay": 0 }, { "name": "perf_task_28", "taskReferenceName": "perf_task_28", "inputParameters": { "mod": "${perf_task_3.output.mod}", "oddEven": "${perf_task_3.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_29", "taskReferenceName": "perf_task_29", "inputParameters": { "mod": "${perf_task_28.output.mod}", "oddEven": "${perf_task_28.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 }, { "name": "perf_task_30", "taskReferenceName": "perf_task_30", "inputParameters": { "mod": "${perf_task_29.output.mod}", "oddEven": "${perf_task_29.output.oddEven}" }, "type": "SIMPLE", "startDelay": 0 } ], "schemaVersion": 2 } } ================================================ FILE: dependencies.gradle ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ /* * Common place to define all the version dependencies */ ext { revActivation = '2.0.0' revAwaitility = '3.1.6' revAwsSdk = '1.12.535' revBval = '2.0.6' revCassandra = '3.10.2' revCassandraUnit = '3.11.2.0' revCommonsCompress = '1.24.0' revCommonsIo = '2.7' revDynoQueues = '2.0.20' revElasticSearch6 = '6.8.17' revEmbeddedRedis = '0.6' revEurekaClient = '1.10.10' revFasterXml = '2.15.0' revGroovy = '3.0.19' revGrpc = '1.57.+' revGuava = '32.1.2-jre' revHamcrestAllMatchers = '1.8' revHealth = '1.1.+' revJAXB = '2.3.3' revJAXRS = '2.1.1' revJedis = '3.3.0' revJersey = '1.19.4' revJerseyCommon = '2.22.2' revJsonPath = '2.4.0' revJq = '0.0.13' revJsr311Api = '1.1.1' revMockServerClient = '5.12.0' revOpenapi = '1.6.+' revOrkesQueues = '1.0.3' revPowerMock = '2.0.9' revProtoBuf = '3.24.3' revProtogenAnnotations = '1.0.0' revProtogenCodegen = '1.4.0' revRarefiedRedis = '0.0.17' revRedisson = '3.13.3' revRxJava = '1.2.2' revSpectator = '0.122.0' revSpock = '2.3-groovy-3.0' revSpotifyCompletableFutures = '0.3.3' revTestContainer = '1.19.1' } ================================================ FILE: docker/README.md ================================================ # Conductor Docker Builds ## Pre-built docker images Conductor server with support for the following backend: 1. Redis 2. Postgres 3. Mysql 4. Cassandra ### Docker File for Server and UI [Docker Image Source for Server with UI](server/Dockerfile) ### Configuration Guide for Conductor Server Conductor uses a persistent store for managing state. The choice of backend is quite flexible and can be configured at runtime using `conductor.db.type` property. Refer to the table below for various supported backend and required configurations to enable each of them. > [!IMPORTANT] > > See [config.properties](docker/server/config/config.properties) for the required properties for each of the backends. > > | Backend | Property | > |------------|------------------------------------| > | postgres | conductor.db.type=postgres | > | redis | conductor.db.type=redis_standalone | > | mysql | conductor.db.type=mysql | > | cassandra | conductor.db.type=cassandra | > Conductor using Elasticsearch for indexing the workflow data. Currently, Elasticsearch 6 and 7 are supported. We welcome community contributions for other indexing backends. **Note:** Docker images use Elasticsearch 7. ## Helm Charts TODO: Link to the helm charts ## Run Docker Compose Locally ### Use the docker-compose to bring up the local conductor server. | Docker Compose | Description | |--------------------------------------------------------------|----------------------------| | [docker-compose.yaml](docker-compose.yaml) | Redis + Elasticsearch 7 | | [docker-compose-postgres.yaml](docker-compose-postgres.yaml) | Postgres + Elasticsearch 7 | | [docker-compose-mysql.yaml](docker-compose-mysql.yaml) | Mysql + Elasticsearch 7 | ================================================ FILE: docker/ci/Dockerfile ================================================ FROM openjdk:17-jdk WORKDIR /workspace/conductor COPY . /workspace/conductor RUN ./gradlew clean build ================================================ FILE: docker/docker-compose-mysql.yaml ================================================ version: '2.3' services: conductor-server: environment: - CONFIG_PROP=config-mysql.properties image: conductor:server container_name: conductor-server build: context: ../ dockerfile: docker/server/Dockerfile networks: - internal ports: - 8080:8080 - 5000:5000 healthcheck: test: [ "CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health" ] interval: 60s timeout: 30s retries: 12 links: - conductor-elasticsearch:es - conductor-mysql:mysql - conductor-redis:rs depends_on: conductor-elasticsearch: condition: service_healthy conductor-mysql: condition: service_healthy conductor-redis: condition: service_healthy logging: driver: "json-file" options: max-size: "1k" max-file: "3" conductor-mysql: image: mysql:latest environment: MYSQL_ROOT_PASSWORD: 12345 MYSQL_DATABASE: conductor MYSQL_USER: conductor MYSQL_PASSWORD: conductor volumes: - type: volume source: conductor_mysql target: /var/lib/mysql networks: - internal ports: - 3306:3306 healthcheck: test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/3306' interval: 5s timeout: 5s retries: 12 conductor-redis: image: redis:6.2.3-alpine volumes: - ./redis.conf:/usr/local/etc/redis/redis.conf networks: - internal ports: - 7379:6379 healthcheck: test: [ "CMD", "redis-cli","ping" ] conductor-elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11 environment: - "ES_JAVA_OPTS=-Xms512m -Xmx1024m" - xpack.security.enabled=false - discovery.type=single-node volumes: - esdata-conductor:/usr/share/elasticsearch/data networks: - internal ports: - 9201:9200 healthcheck: test: curl http://localhost:9200/_cluster/health -o /dev/null interval: 5s timeout: 5s retries: 12 logging: driver: "json-file" options: max-size: "1k" max-file: "3" volumes: conductor_mysql: driver: local esdata-conductor: driver: local networks: internal: ================================================ FILE: docker/docker-compose-postgres.yaml ================================================ version: '2.3' services: conductor-server: environment: - CONFIG_PROP=config-postgres.properties image: conductor:server container_name: conductor-server build: context: ../ dockerfile: docker/server/Dockerfile networks: - internal ports: - 8080:8080 - 5000:5000 healthcheck: test: [ "CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health" ] interval: 60s timeout: 30s retries: 12 links: - conductor-elasticsearch:es - conductor-postgres:postgresdb depends_on: conductor-elasticsearch: condition: service_healthy conductor-postgres: condition: service_healthy logging: driver: "json-file" options: max-size: "1k" max-file: "3" conductor-postgres: image: postgres environment: - POSTGRES_USER=conductor - POSTGRES_PASSWORD=conductor volumes: - pgdata-conductor:/var/lib/postgresql/data networks: - internal ports: - 6432:5432 healthcheck: test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432' interval: 5s timeout: 5s retries: 12 logging: driver: "json-file" options: max-size: "1k" max-file: "3" conductor-elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11 environment: - "ES_JAVA_OPTS=-Xms512m -Xmx1024m" - xpack.security.enabled=false - discovery.type=single-node volumes: - esdata-conductor:/usr/share/elasticsearch/data networks: - internal ports: - 9201:9200 healthcheck: test: curl http://localhost:9200/_cluster/health -o /dev/null interval: 5s timeout: 5s retries: 12 logging: driver: "json-file" options: max-size: "1k" max-file: "3" volumes: pgdata-conductor: driver: local esdata-conductor: driver: local networks: internal: ================================================ FILE: docker/docker-compose.yaml ================================================ version: '2.3' services: conductor-server: environment: - CONFIG_PROP=config-redis.properties image: conductor:server container_name: conductor-server build: context: ../ dockerfile: docker/server/Dockerfile networks: - internal ports: - 8080:8080 - 5000:5000 healthcheck: test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"] interval: 60s timeout: 30s retries: 12 links: - conductor-elasticsearch:es - conductor-redis:rs depends_on: conductor-elasticsearch: condition: service_healthy conductor-redis: condition: service_healthy logging: driver: "json-file" options: max-size: "1k" max-file: "3" conductor-redis: image: redis:6.2.3-alpine volumes: - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf networks: - internal ports: - 7379:6379 healthcheck: test: [ "CMD", "redis-cli","ping" ] conductor-elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11 environment: - "ES_JAVA_OPTS=-Xms512m -Xmx1024m" - xpack.security.enabled=false - discovery.type=single-node volumes: - esdata-conductor:/usr/share/elasticsearch/data networks: - internal ports: - 9201:9200 healthcheck: test: curl http://localhost:9200/_cluster/health -o /dev/null interval: 5s timeout: 5s retries: 12 logging: driver: "json-file" options: max-size: "1k" max-file: "3" volumes: esdata-conductor: driver: local networks: internal: ================================================ FILE: docker/server/Dockerfile ================================================ # # conductor:server - Combined Netflix conductor server & UI # # =========================================================================================================== # 0. Builder stage # =========================================================================================================== FROM alpine:3.18 AS builder LABEL maintainer="Netflix OSS " # =========================================================================================================== # 0. Build Conductor Server # =========================================================================================================== # Install dependencies RUN apk add openjdk17 RUN apk add git RUN apk add --update nodejs npm yarn COPY . /conductor WORKDIR /conductor/ui # Include monaco sources into bundle (instead of using CDN) ENV REACT_APP_MONACO_EDITOR_USING_CDN=false RUN yarn install && cp -r node_modules/monaco-editor public/ && yarn build RUN ls -ltr RUN echo "Done building UI" # Checkout the community project WORKDIR / RUN mkdir server-build WORKDIR server-build RUN ls -ltr RUN git clone https://github.com/Netflix/conductor-community.git # Copy the project directly onto the image WORKDIR conductor-community RUN ls -ltr # Build the server on run RUN ./gradlew build -x test --stacktrace WORKDIR /server-build RUN ls -ltr RUN pwd # =========================================================================================================== # 1. Bin stage # =========================================================================================================== FROM alpine:3.18 LABEL maintainer="Netflix OSS " RUN apk add openjdk17 RUN apk add nginx # Make app folders RUN mkdir -p /app/config /app/logs /app/libs # Copy the compiled output to new image COPY docker/server/bin /app COPY docker/server/config /app/config COPY --from=builder /server-build/conductor-community/community-server/build/libs/*boot*.jar /app/libs/conductor-server.jar # Copy compiled UI assets to nginx www directory WORKDIR /usr/share/nginx/html RUN rm -rf ./* COPY --from=builder /conductor/ui/build . COPY --from=builder /conductor/docker/server/nginx/nginx.conf /etc/nginx/http.d/default.conf # Copy the files for the server into the app folders RUN chmod +x /app/startup.sh HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 CMD [ "/app/startup.sh" ] ENTRYPOINT [ "/bin/sh"] ================================================ FILE: docker/server/config/config-mysql.properties ================================================ # Database persistence type. conductor.db.type=mysql # mysql spring.datasource.url=jdbc:mysql://mysql:3306/conductor spring.datasource.username=conductor spring.datasource.password=conductor # Use redis queues conductor.queue.type=redis_standalone # Elastic search instance indexing is enabled. conductor.indexing.enabled=true conductor.elasticsearch.url=http://es:9200 conductor.elasticsearch.indexName=conductor conductor.elasticsearch.version=7 conductor.elasticsearch.clusterHealthColor=yellow # Additional modules for metrics collection exposed to Prometheus (optional) conductor.metrics-prometheus.enabled=true management.endpoints.web.exposure.include=prometheus # Load sample kitchen-sink workflow loadSample=true ================================================ FILE: docker/server/config/config-postgres.properties ================================================ # Database persistence type. conductor.db.type=postgres # postgres spring.datasource.url=jdbc:postgresql://postgresdb:5432/postgres spring.datasource.username=conductor spring.datasource.password=conductor # Elastic search instance indexing is enabled. conductor.indexing.enabled=true conductor.elasticsearch.url=http://es:9200 conductor.elasticsearch.indexName=conductor conductor.elasticsearch.version=7 conductor.elasticsearch.clusterHealthColor=yellow # Additional modules for metrics collection exposed to Prometheus (optional) conductor.metrics-prometheus.enabled=true management.endpoints.web.exposure.include=prometheus # Load sample kitchen-sink workflow loadSample=true ================================================ FILE: docker/server/config/config-redis.properties ================================================ # Database persistence type. # Below are the properties for redis conductor.db.type=redis_standalone conductor.redis.hosts=rs:6379:us-east-1c conductor.redis-lock.serverAddress=redis://rs:6379 conductor.redis.taskDefCacheRefreshInterval=1 conductor.redis.workflowNamespacePrefix=conductor conductor.redis.queueNamespacePrefix=conductor_queues #Use redis queues conductor.queue.type=redis_standalone # Elastic search instance indexing is enabled. conductor.indexing.enabled=true conductor.elasticsearch.url=http://es:9200 conductor.elasticsearch.indexName=conductor conductor.elasticsearch.version=7 conductor.elasticsearch.clusterHealthColor=yellow # Additional modules for metrics collection exposed to Prometheus (optional) conductor.metrics-prometheus.enabled=true management.endpoints.web.exposure.include=prometheus # Load sample kitchen sink workflow loadSample=true ================================================ FILE: docker/server/config/config.properties ================================================ # See README in the docker for configuration guide # db.type determines the type of database used # See various configurations below for the values conductor.db.type=SET_THIS # =====================================================# # Redis Configuration Properties # =====================================================# #conductor.db.type=redis_standalone # The last part MUST be us-east-1c, it is not used and is kept for backwards compatibility # conductor.redis.hosts=rs:6379:us-east-1c # # conductor.redis-lock.serverAddress=redis://rs:6379 # conductor.redis.taskDefCacheRefreshInterval=1 # conductor.redis.workflowNamespacePrefix=conductor # conductor.redis.queueNamespacePrefix=conductor_queues # =====================================================# # Postgres Configuration Properties # =====================================================# # conductor.db.type=postgres # spring.datasource.url=jdbc:postgresql://localhost:5432/postgres # spring.datasource.username=postgres # spring.datasource.password=postgres # Additionally you can use set the spring.datasource.XXX properties for connection pool size etc. # If you want to use Postgres as indexing store set the following # conductor.indexing.enabled=true # conductor.indexing.type=postgres # When using Elasticsearch 7 for indexing, set the following # conductor.indexing.enabled=true # conductor.elasticsearch.url=http://es:9200 # conductor.elasticsearch.version=7 # conductor.elasticsearch.indexName=conductor ================================================ FILE: docker/server/config/log4j-file-appender.properties ================================================ # # Copyright 2020 Netflix, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # log4j.rootLogger=INFO,console,file log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.appender.file=org.apache.log4j.RollingFileAppender log4j.appender.file.File=/app/logs/conductor.log log4j.appender.file.MaxFileSize=10MB log4j.appender.file.MaxBackupIndex=10 log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n # Dedicated file appender for metrics log4j.appender.fileMetrics=org.apache.log4j.RollingFileAppender log4j.appender.fileMetrics.File=/app/logs/metrics.log log4j.appender.fileMetrics.MaxFileSize=10MB log4j.appender.fileMetrics.MaxBackupIndex=10 log4j.appender.fileMetrics.layout=org.apache.log4j.PatternLayout log4j.appender.fileMetrics.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.logger.ConductorMetrics=INFO,console,fileMetrics log4j.additivity.ConductorMetrics=false ================================================ FILE: docker/server/config/log4j.properties ================================================ # # Copyright 2017 Netflix, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set root logger level to DEBUG and its only appender to A1. log4j.rootLogger=INFO, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n logging.logger.com.netflix.dyno.queues.redis.RedisDynoQueue=ERROR ================================================ FILE: docker/server/config/redis.conf ================================================ appendonly yes ================================================ FILE: docker/server/nginx/nginx.conf ================================================ server { listen 5000; server_name conductor; server_tokens off; location / { add_header Referrer-Policy "strict-origin"; add_header X-Frame-Options "SAMEORIGIN"; add_header X-Content-Type-Options "nosniff"; add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; # This would be the directory where your React app's static files are stored at root /usr/share/nginx/html; try_files $uri /index.html; } location /api { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; proxy_pass http://localhost:8080/api; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; proxy_redirect off; } location /actuator { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; proxy_pass http://localhost:8080/actuator; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; proxy_redirect off; } location /swagger-ui { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; proxy_pass http://localhost:8080/swagger-ui; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; proxy_redirect off; } } ================================================ FILE: docker/ui/Dockerfile ================================================ # # conductor:ui - Netflix Conductor UI # FROM node:20-alpine LABEL maintainer="Netflix OSS " # Install the required packages for the node build # to run on alpine RUN apk update && apk add --no-cache python3 py3-pip make g++ # A directory within the virtualized Docker environment # Becomes more relevant when using Docker Compose later WORKDIR /usr/src/app # Copies package.json to Docker environment in a separate layer as a performance optimization COPY ./ui/package.json ./ # Installs all node packages. Cached unless package.json changes RUN yarn install && mkdir -p public && cp -r node_modules/monaco-editor public/ # Copies everything else over to Docker environment # node_modules excluded in .dockerignore. COPY ./ui . # Include monaco sources into bundle (instead of using CDN) ENV REACT_APP_MONACO_EDITOR_USING_CDN=false CMD [ "yarn", "start" ] ================================================ FILE: docker/ui/README.md ================================================ # Docker ## Conductor UI This Dockerfile create the conductor:ui image ## Building the image Run the following commands from the project root. `docker build -f docker/ui/Dockerfile -t conductor:ui .` ## Running the conductor server - With localhost conductor server: `docker run -p 5000:5000 -d -t conductor:ui` - With external conductor server: `docker run -p 5000:5000 -d -t -e "WF_SERVER=http://conductor-server:8080" conductor:ui` ================================================ FILE: docs/docs/apispec.md ================================================ # API Specification ## Task & Workflow Metadata | Endpoint | Description | Input | |------------------------------------------|:---------------------------------|-------------------------------------------------------------| | `GET /metadata/taskdefs` | Get all the task definitions | n/a | | `GET /metadata/taskdefs/{taskType}` | Retrieve task definition | Task Name | | `POST /metadata/taskdefs` | Register new task definitions | List of [Task Definitions](/configuration/taskdef.html) | | `PUT /metadata/taskdefs` | Update a task definition | A [Task Definition](/configuration/taskdef.html) | | `DELETE /metadata/taskdefs/{taskType}` | Delete a task definition | Task Name | ||| | `GET /metadata/workflow` | Get all the workflow definitions | n/a | | `POST /metadata/workflow` | Register new workflow | [Workflow Definition](/configuration/workflowdef.html) | | `PUT /metadata/workflow` | Register/Update new workflows | List of [Workflow Definition](/configuration/workflowdef.html) | | `GET /metadata/workflow/{name}?version=` | Get the workflow definitions | workflow name, version (optional) | ||| ## Start A Workflow ### With Input only See [Start Workflow Request](/gettingstarted/startworkflow.html). #### Output Id of the workflow (GUID) ### With Input and Task Domains ``` POST /workflow { //JSON payload for Start workflow request } ``` #### Start workflow request JSON for start workflow request ``` { "name": "myWorkflow", // Name of the workflow "version": 1, // Version “correlationId”: “corr1”, // correlation Id "priority": 1, // Priority "input": { // Input map. }, "taskToDomain": { // Task to domain map } } ``` #### Output Id of the workflow (GUID) ## Retrieve Workflows | Endpoint | Description | |-----------------------------------------------------------------------------|-----------------------------------------------| | `GET /workflow/{workflowId}?includeTasks=true | false` |Get Workflow State by workflow Id. If includeTasks is set, then also includes all the tasks executed and scheduled.| | `GET /workflow/running/{name}` | Get all the running workflows of a given type | | `GET /workflow/running/{name}/correlated/{correlationId}?includeClosed=true | false&includeTasks=true |false`|Get all the running workflows filtered by correlation Id. If includeClosed is set, also includes workflows that have completed running.| | `GET /workflow/search` | Search for workflows. See Below. | ## Search for Workflows Conductor uses Elasticsearch for indexing workflow execution and is used by search APIs. `GET /workflow/search?start=&size=&sort=&freeText=&query=` | Parameter | Description | |-----------|------------------------------------------------------------------------------------------------------------------| | start | Page number. Defaults to 0 | | size | Number of results to return | | sort | Sorting. Format is: `ASC:` or `DESC:` to sort in ascending or descending order by a field | | freeText | Elasticsearch supported query. e.g. workflowType:"name_of_workflow" | | query | SQL like where clause. e.g. workflowType = 'name_of_workflow'. Optional if freeText is provided. | ### Output Search result as described below: ```json { "totalHits": 0, "results": [ { "workflowType": "string", "version": 0, "workflowId": "string", "correlationId": "string", "startTime": "string", "updateTime": "string", "endTime": "string", "status": "RUNNING", "input": "string", "output": "string", "reasonForIncompletion": "string", "executionTime": 0, "event": "string" } ] } ``` ## Manage Workflows | Endpoint | Description | |-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| | `PUT /workflow/{workflowId}/pause` | Pause. No further tasks will be scheduled until resumed. Currently running tasks are not paused. | | `PUT /workflow/{workflowId}/resume` | Resume normal operations after a pause. | | `POST /workflow/{workflowId}/rerun` | See Below. | | `POST /workflow/{workflowId}/restart` | Restart workflow execution from the start. Current execution history is wiped out. | | `POST /workflow/{workflowId}/retry` | Retry the last failed task. | | `PUT /workflow/{workflowId}/skiptask/{taskReferenceName}` | See below. | | `DELETE /workflow/{workflowId}` | Terminates the running workflow. | | `DELETE /workflow/{workflowId}/remove` | Deletes the workflow from system. Use with caution. | ### Rerun Re-runs a completed workflow from a specific task. `POST /workflow/{workflowId}/rerun` ```json { "reRunFromWorkflowId": "string", "workflowInput": {}, "reRunFromTaskId": "string", "taskInput": {} } ``` ###Skip Task Skips a task execution (specified as `taskReferenceName` parameter) in a running workflow and continues forward. Optionally updating task's input and output as specified in the payload. `PUT /workflow/{workflowId}/skiptask/{taskReferenceName}?workflowId=&taskReferenceName=` ```json { "taskInput": {}, "taskOutput": {} } ``` ## Manage Tasks | Endpoint | Description | |-------------------------------------------------------|-------------------------------------------------------| | `GET /tasks/{taskId}` | Get task details. | | `GET /tasks/queue/all` | List the pending task sizes. | | `GET /tasks/queue/all/verbose` | Same as above, includes the size per shard | | `GET /tasks/queue/sizes?taskType=&taskType=&taskType` | Return the size of pending tasks for given task types | ||| ## Polling, Ack and Update Task These are critical endpoints used to poll for task, send ack (after polling) and finally updating the task result by worker. | Endpoint | Description | |---------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `GET /tasks/poll/{taskType}?workerid=&domain=` | Poll for a task. `workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | | `GET /tasks/poll/batch/{taskType}?count=&timeout=&workerid=&domain` | Poll for a task in a batch specified by `count`. This is a long poll and the connection will wait until `timeout` or if there is at-least 1 item available, whichever comes first.`workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | | `POST /tasks` | Update the result of task execution. See the schema below. | ### Schema for updating Task Result ```json { "workflowInstanceId": "Workflow Instance Id", "taskId": "ID of the task to be updated", "reasonForIncompletion" : "If failed, reason for failure", "callbackAfterSeconds": 0, "status": "IN_PROGRESS|FAILED|COMPLETED", "outputData": { //JSON document representing Task execution output } } ``` !!!Info "Acknowledging tasks after poll" If the worker fails to ack the task after polling, the task is re-queued and put back in queue and is made available during subsequent poll. ================================================ FILE: docs/docs/architecture/overview.md ================================================ # Overview ![Architecture diagram](/img/conductor-architecture.png) The API and storage layers are pluggable and provide ability to work with different backends and queue service providers. ## Runtime Model Conductor follows RPC based communication model where workers are running on a separate machine from the server. Workers communicate with server over HTTP based endpoints and employs polling model for managing work queues. ![Runtime Model of Conductor](/img/overview.png) **Notes** * Workers are remote systems that communicate over HTTP with the conductor servers. * Task Queues are used to schedule tasks for workers. We use [dyno-queues][1] internally but it can easily be swapped with SQS or similar pub-sub mechanism. * conductor-redis-persistence module uses [Dynomite][2] for storing the state and metadata along with [Elasticsearch][3] for indexing backend. * See section under extending backend for implementing support for different databases for storage and indexing. [1]: https://github.com/Netflix/dyno-queues [2]: https://github.com/Netflix/dynomite [3]: https://www.elastic.co ================================================ FILE: docs/docs/architecture/tasklifecycle.md ================================================ ## Task state transitions The figure below depicts the state transitions that a task can go through within a workflow execution. ![Task_States](/img/task_states.png) ## Retries and Failure Scenarios ### Task failure and retries Retries for failed task executions of each task can be configured independently. retryCount, retryDelaySeconds and retryLogic can be used to configure the retry mechanism. ![Task Failure](/img/TaskFailure.png) 1. Worker (W1) polls for task T1 from the Conductor server and receives the task. 2. Upon processing this task, the worker determines that the task execution is a failure and reports this to the server with FAILED status after 10 seconds. 3. The server will persist this FAILED execution of T1. A new execution of task T1 will be created and scheduled to be polled. This task will be available to be polled after 5 (retryDelaySeconds) seconds. ### Timeout seconds Timeout is the maximum amount of time that the task must reach a terminal state in, else the task will be marked as TIMED_OUT. ![Task Timeout](/img/TimeoutSeconds.png) **0 seconds** -> Worker polls for task T1 from the Conductor server and receives the task. T1 is put into IN_PROGRESS status by the server. Worker starts processing the task but is unable to process the task at this time. Worker updates the server with T1 set to IN_PROGRESS status and a callback of 9 seconds. Server puts T1 back in the queue but makes it invisible and the worker continues to poll for the task but does not receive T1 for 9 seconds. **9,18 seconds** -> Worker receives T1 from the server and is still unable to process the task and updates the server with a callback of 9 seconds. **27 seconds** -> Worker polls and receives task T1 from the server and is now able to process this task. **30 seconds** (T1 timeout) -> Server marks T1 as TIMED_OUT because it is not in a terminal state after first being moved to IN_PROGRESS status. Server schedules a new task based on the retry count. **32 seconds** -> Worker completes processing of T1 and updates the server with COMPLETED status. Server will ignore this update since T1 has already been moved to a terminal status (TIMED_OUT). ### Response timeout seconds Response timeout is the time within which the worker must respond to the server with an update for the task, else the task will be marked as TIMED_OUT. ![Response Timeout](/img/ResponseTimeoutSeconds.png) **0 seconds** -> Worker polls for the task T1 from the Conductor server and receives the task. T1 is put into IN_PROGRESS status by the server. Worker starts processing the task but the worker instance dies during this execution. **20 seconds** (T1 responseTimeout) -> Server marks T1 as TIMED_OUT since the task has not been updated by the worker within the configured responseTimeoutSeconds (20). A new instance of task T1 is scheduled as per the retry configuration. **25 seconds** -> The retried instance of T1 is available to be polled by the worker, after the retryDelaySeconds (5) has elapsed. ================================================ FILE: docs/docs/bestpractices.md ================================================ ## Response Timeout - Configure the responseTimeoutSeconds of each task to be > 0. - Should be less than or equal to timeoutSeconds. ## Payload sizes - Configure your workflows such that conductor is not used as a persistence store. - Ensure that the output data in the task result set in your worker is used by your workflow for execution. If the values in the output payloads are not used by subsequent tasks in your workflow, this data should not be sent back to conductor in the task result. - In cases where the output data of your task is used within subsequent tasks in your workflow but is substantially large (> 100KB), consider uploading this data to an object store (S3 or similar) and set the location to the object in your task output. The subsequent tasks can then download this data from the given location and use it during execution. ================================================ FILE: docs/docs/configuration/eventhandlers.md ================================================ # Event Handlers Eventing in Conductor provides for loose coupling between workflows and support for producing and consuming events from external systems. This includes: 1. Being able to produce an event (message) in an external system like SQS or internal to Conductor. 2. Start a workflow when a specific event occurs that matches the provided criteria. Conductor provides SUB_WORKFLOW task that can be used to embed a workflow inside parent workflow. Eventing supports provides similar capability without explicitly adding dependencies and provides **fire-and-forget** style integrations. ## Event Task Event task provides ability to publish an event (message) to either Conductor or an external eventing system like SQS. Event tasks are useful for creating event based dependencies for workflows and tasks. See [Event Task](/reference-docs/event-task.html) for documentation. ## Event Handler Event handlers are listeners registered that executes an action when a matching event occurs. The supported actions are: 1. Start a Workflow 2. Fail a Task 3. Complete a Task Event Handlers can be configured to listen to Conductor Events or an external event like SQS. ### Configuration Event Handlers are configured via ```/event/``` APIs. #### Structure: ```json { "name" : "descriptive unique name", "event": "event_type:event_location", "condition": "boolean condition", "actions": ["see examples below"] } ``` #### Condition Condition is an expression that MUST evaluate to a boolean value. A Javascript like syntax is supported that can be used to evaluate condition based on the payload. Actions are executed only when the condition evaluates to `true`. **Examples** Given the following payload in the message: ```json { "fileType": "AUDIO", "version": 3, "metadata": { "length": 300, "codec": "aac" } } ``` |Expression|Result| |---|---| |`$.version > 1`|true| |`$.version > 10`|false| |`$.metadata.length == 300`|true| ### Actions **Start A Workflow** ```json { "action": "start_workflow", "start_workflow": { "name": "WORKFLOW_NAME", "version": "", "input": { "param1": "${param1}" } } } ``` **Complete Task*** ```json { "action": "complete_task", "complete_task": { "workflowId": "${workflowId}", "taskRefName": "task_1", "output": { "response": "${result}" } }, "expandInlineJSON": true } ``` **Fail Task*** ```json { "action": "fail_task", "fail_task": { "workflowId": "${workflowId}", "taskRefName": "task_1", "output": { "response": "${result}" } }, "expandInlineJSON": true } ``` Input for starting a workflow and output when completing / failing task follows the same [expressions](/configuration/workflowdef.html#wiring-inputs-and-outputs) used for wiring workflow inputs. !!!info "Expanding stringified JSON elements in payload" `expandInlineJSON` property, when set to true will expand the inlined stringified JSON elements in the payload to JSON documents and replace the string value with JSON document. This feature allows such elements to be used with JSON path expressions. ## Extending Provide the implementation of [EventQueueProvider](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/events/EventQueueProvider.java). SQS Queue Provider: [SQSEventQueueProvider.java ](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/core/events/sqs/SQSEventQueueProvider.java) ================================================ FILE: docs/docs/configuration/isolationgroups.md ================================================ # Isolation Groups Consider an HTTP task where the latency of an API is high, task queue piles up effecting execution of other HTTP tasks which have low latency. We can isolate the execution of such tasks to have predictable performance using `isolationgroupId`, a property of task definition. When we set isolationGroupId, the executor `SystemTaskWorkerCoordinator` will allocate an isolated queue and an isolated thread pool for execution of those tasks. If no `isolationgroupId` is specified in task definition, then fallback is default behaviour where the executor executes the task in shared thread-pool for all tasks. ## Example ** Task Definition ** ```json { "name": "encode_task", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "sourceRequestId", "qcElementType" ], "outputKeys": [ "state", "skipped", "result" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 3600, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "rateLimitPerFrequency": 50, "isolationgroupId": "myIsolationGroupId" } ``` ** Workflow Definition ** ```json { "name": "encode_and_deploy", "description": "Encodes a file and deploys to CDN", "version": 1, "tasks": [ { "name": "encode", "taskReferenceName": "encode", "type": "HTTP", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } } } ], "outputParameters": { "cdn_url": "${d1.output.location}" }, "failureWorkflow": "cleanup_encode_resources", "restartable": true, "workflowStatusListenerEnabled": true, "schemaVersion": 2 } ``` - puts `encode` in `HTTP-myIsolationGroupId` queue, and allocates a new thread pool for this for execution. Note: To enable this feature, the `workflow.isolated.system.task.enable` property needs to be made `true`,its default value is `false` The property `workflow.isolated.system.task.worker.thread.count` sets the thread pool size for isolated tasks; default is `1`. isolationGroupId is currently supported only in HTTP and kafka Task. ### Execution Name Space `executionNameSpace` A property of taskdef can be used to provide JVM isolation to task execution and scale executor deployments horizontally. Limitation of using isolationGroupId is that we need to scale executors vertically as the executor allocates a new thread pool per `isolationGroupId`. Also, since the executor runs the tasks in the same JVM, task execution is not isolated completely. To support JVM isolation, and also allow the executors to scale horizontally, we can use `executionNameSpace` property in taskdef. Executor consumes tasks whose executionNameSpace matches with the configuration property `workflow.system.task.worker.executionNameSpace` If the property is not set, the executor executes tasks without any executionNameSpace set. ```json { "name": "encode_task", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "sourceRequestId", "qcElementType" ], "outputKeys": [ "state", "skipped", "result" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 3600, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "rateLimitPerFrequency": 50, "executionNameSpace": "myExecutionNameSpace" } ``` #### Example Workflow task ```json { "name": "encode_and_deploy", "description": "Encodes a file and deploys to CDN", "version": 1, "tasks": [ { "name": "encode", "taskReferenceName": "encode", "type": "HTTP", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } } } ], "outputParameters": { "cdn_url": "${d1.output.location}" }, "failureWorkflow": "cleanup_encode_resources", "restartable": true, "workflowStatusListenerEnabled": true, "schemaVersion": 2 } ``` - `encode` task is executed by the executor deployment whose `workflow.system.task.worker.executionNameSpace` property is `myExecutionNameSpace` `executionNameSpace` can be used along with `isolationGroupId` If the above task contains a isolationGroupId `myIsolationGroupId`, the tasks will be scheduled in a queue HTTP@myExecutionNameSpace-myIsolationGroupId, and have a new threadpool for execution in the deployment group with myExecutionNameSpace ================================================ FILE: docs/docs/configuration/sysoperator.md ================================================ # System Operators Operators are built-in primitives in Conductor that allow you to define the control flow in the workflow. Operators are similar to programming constructs such as for loops, decisions, etc. Conductor has support for most of the programing primitives allowing you to define the most advanced workflows. ## Supported Operators Conductor supports the following programming language constructs: | Language Construct | Conductor Operator | |----------------------------------|-------------------------------------------------------------| | Do/While or Loops | [Do While Task](/reference-docs/do-while-task.html) | | Dynamic Fork | [Dynamic Fork Task](/reference-docs/dynamic-fork-task.html) | | Fork / Parallel execution | [Fork Task](/reference-docs/fork-task.html) | | Join | [Join Task](/reference-docs/join-task.html) | | Sub Process / Sub-Flow | [Sub Workflow Task](/reference-docs/sub-workflow-task.html) | | Switch//Decision/if..then...else | [Switch Task](/reference-docs/switch-task.html) | | Terminate | [Terminate Task](/reference-docs/terminate-task.html) | | Variables | [Variable Task](/reference-docs/set-variable-task.html) | | Wait | [Wait Task](/reference-docs/wait-task.html) | ================================================ FILE: docs/docs/configuration/systask.md ================================================ # System Tasks System Tasks (Workers) are built-in tasks that are general purpose and re-usable. They run on the Conductor servers. Such tasks allow you to get started without having to write custom workers. ## Available System Tasks Conductor has the following set of system tasks available. | Task | Description | Use Case | |-----------------------|--------------------------------------------------------|------------------------------------------------------------------------------------| | Event Publishing | [Event Task](/reference-docs/event-task.html) | External eventing system integration. e.g. amqp, sqs, nats | | HTTP | [HTTP Task](/reference-docs/http-task.html) | Invoke any HTTP(S) endpoints | | Inline Code Execution | [Inline Task](/reference-docs/inline-task.html) | Execute arbitrary lightweight javascript code | | JQ Transform | [JQ Task](/reference-docs/json-jq-transform-task.html) | Use JQ to transform task input/output | | Kafka Publish | [Kafka Task](/reference-docs/kafka-publish-task.html) | Publish messages to Kafka | | Name | Description | |--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| | joinOn | List of task reference names, which the EXCLUSIVE_JOIN will lookout for to capture output. From above example, this could be ["T2", "T3"] | | defaultExclusiveJoinTask | Task reference name, whose output should be used incase the decision case is undefined. From above example, this could be ["T1"] | **Example** ``` json { "name": "exclusive_join", "taskReferenceName": "exclusiveJoin", "type": "EXCLUSIVE_JOIN", "joinOn": [ "task2", "task3" ], "defaultExclusiveJoinTask": [ "task1" ] } ``` ## Wait A wait task is implemented as a gate that remains in ```IN_PROGRESS``` state unless marked as ```COMPLETED``` or ```FAILED``` by an external trigger. To use a wait task, set the task type as ```WAIT``` **Parameters:** None required. **External Triggers for Wait Task** Task Resource endpoint can be used to update the status of a task to a terminate state. Contrib module provides SQS integration where an external system can place a message in a pre-configured queue that the server listens on. As the messages arrive, they are marked as ```COMPLETED``` or ```FAILED```. **SQS Queues** * SQS queues used by the server to update the task status can be retrieve using the following API: ``` GET /queue ``` * When updating the status of the task, the message needs to conform to the following spec: * Message has to be a valid JSON string. * The message JSON should contain a key named ```externalId``` with the value being a JSONified string that contains the following keys: * ```workflowId```: Id of the workflow * ```taskRefName```: Task reference name that should be updated. * Each queue represents a specific task status and tasks are marked accordingly. e.g. message coming to a ```COMPLETED``` queue marks the task status as ```COMPLETED```. * Tasks' output is updated with the message. **Example SQS Payload:** ```json { "some_key": "valuex", "externalId": "{\"taskRefName\":\"TASK_REFERENCE_NAME\",\"workflowId\":\"WORKFLOW_ID\"}" } ``` ## Dynamic Task Dynamic Task allows to execute one of the registered Tasks dynamically at run-time. It accepts the task name to execute in inputParameters. **Parameters:** |name|description| |---|---| | dynamicTaskNameParam|Name of the parameter from the task input whose value is used to schedule the task. e.g. if the value of the parameter is ABC, the next task scheduled is of type 'ABC'.| **Example** ``` json { "name": "user_task", "taskReferenceName": "t1", "inputParameters": { "files": "${workflow.input.files}", "taskToExecute": "${workflow.input.user_supplied_task}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" } ``` If the workflow is started with input parameter user_supplied_task's value as __user_task_2__, Conductor will schedule __user_task_2__ when scheduling this dynamic task. ## Inline Task Inline Task helps execute ad-hoc logic at Workflow run-time, using any evaluator engine. Supported evaluators are `value-param` evaluator which simply translates the input parameter to output and `javascript` evaluator that evaluates Javascript expression. This is particularly helpful in running simple evaluations in Conductor server, over creating Workers. **Parameters:** |name|type|description|notes| |---|---|---|---| |evaluatorType|String|Type of the evaluator. Supported evaluators: `value-param`, `javascript` which evaluates javascript expression.| |expression|String|Expression associated with the type of evaluator. For `javascript` evaluator, Javascript evaluation engine is used to evaluate expression defined as a string. Must return a value.|Must be non-empty.| Besides `expression`, any value is accessible as `$.value` for the `expression` to evaluate. **Outputs:** |name|type|description| |---|---|---| |result|Map|Contains the output returned by the evaluator based on the `expression`| The task output can then be referenced in downstream tasks like: ```"${inline_test.output.result.testvalue}"``` **Example** ``` json { "name": "INLINE_TASK", "taskReferenceName": "inline_test", "type": "INLINE", "inputParameters": { "inlineValue": "${workflow.input.inlineValue}", "evaluatorType": "javascript", "expression": "function scriptFun(){if ($.inlineValue == 1){ return {testvalue: true} } else { return {testvalue: false} }} scriptFun();" } } ``` ## Terminate Task Task that can terminate a workflow with a given status and modify the workflow's output with a given parameter. It can act as a "return" statement for conditions where you simply want to terminate your workflow. For example, if you have a decision where the first condition is met, you want to execute some tasks, otherwise you want to finish your workflow. **Parameters:** |name|type| description | notes | |---|---|---------------------------------------------------|-------------------------| |terminationStatus|String| can only accept "COMPLETED" or "FAILED" | task cannot be optional | |terminationReason|String| reason for incompletion to be set in the workflow | optional | |workflowOutput|Any| Expected workflow output | optional | **Outputs:** |name|type|description| |---|---|---| |output|Map|The content of `workflowOutput` from the inputParameters. An empty object if `workflowOutput` is not set.| ```json { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "COMPLETED", "terminationReason": "", "workflowOutput": "${task0.output}" }, "type": "TERMINATE", "startDelay": 0, "optional": false } ``` ## Kafka Publish Task A kafka Publish task is used to push messages to another microservice via kafka **Parameters:** The task expects an input parameter named ```kafka_request``` as part of the task's input with the following details: |name|description| |---|---| | bootStrapServers |bootStrapServers for connecting to given kafka.| |key|Key to be published| |keySerializer | Serializer used for serializing the key published to kafka. One of the following can be set :
    1. org.apache.kafka.common.serialization.IntegerSerializer
    2. org.apache.kafka.common.serialization.LongSerializer
    3. org.apache.kafka.common.serialization.StringSerializer.
    Default is String serializer | |value| Value published to kafka| |requestTimeoutMs| Request timeout while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.request.timeout.ms`. If the property is not set the value defaults to 100 ms | |maxBlockMs| maxBlockMs while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.max.block.ms`. If the property is not set the value defaults to 500 ms | |headers|A map of additional kafka headers to be sent along with the request.| |topic|Topic to publish| The producer created in the kafka task is cached. By default the cache size is 10 and expiry time is 120000 ms. To change the defaults following can be modified kafka.publish.producer.cache.size,kafka.publish.producer.cache.time.ms respectively. **Kafka Task Output** Task status transitions to COMPLETED **Example** Task sample ```json { "name": "call_kafka", "taskReferenceName": "call_kafka", "inputParameters": { "kafka_request": { "topic": "userTopic", "value": "Message to publish", "bootStrapServers": "localhost:9092", "headers": { "x-Auth":"Auth-key" }, "key": "123", "keySerializer": "org.apache.kafka.common.serialization.IntegerSerializer" } }, "type": "KAFKA_PUBLISH" } ``` The task is marked as ```FAILED``` if the message could not be published to the Kafka queue. ## Do While Task Sequentially execute a list of task as long as a condition is true. The list of tasks is executed first, before the condition is checked (even for the first iteration). When scheduled, each task of this loop will see its `taskReferenceName` concatenated with `__i`, with `i` being the iteration number, starting at 1. Warning: `taskReferenceName` containing arithmetic operators must not be used. Each task output is stored as part of the `DO_WHILE` task, indexed by the iteration value (see example below), allowing the condition to reference the output of a task for a specific iteration (eg. ```$.LoopTask['iteration]['first_task']```) The `DO_WHILE` task is set to `FAILED` as soon as one of the loopTask fails. In such case retry, iteration starts from 1. Limitations: - Domain or isolation group execution is unsupported; - Nested `DO_WHILE` is unsupported; - `SUB_WORKFLOW` is unsupported; - Since loopover tasks will be executed in loop inside scope of parent do while task, crossing branching outside of DO_WHILE task is not respected. Branching inside loopover task is supported. **Parameters:** |name|type|description| ================================================ FILE: docs/docs/configuration/taskdef.md ================================================ # Task Definition Tasks are the building blocks of workflow in Conductor. A task can be an operator, system task or custom code written in any programming language. A typical Conductor workflow is a list of tasks that are executed until completion or the termination of the workflow. Conductor maintains a registry of worker tasks. A task MUST be registered before being used in a workflow. **Example** ``` json { "name": "encode_task", "retryCount": 3, "timeoutSeconds": 1200, "pollTimeoutSeconds": 3600, "inputKeys": [ "sourceRequestId", "qcElementType" ], "outputKeys": [ "state", "skipped", "result" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 1200, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "rateLimitPerFrequency": 50, "ownerEmail": "foo@bar.com", "description": "Sample Encoding task" } ``` | Field | Description | Notes | |----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| | name | Task Type. Unique name of the Task that resonates with it's function. | Unique | | description | Description of the task | optional | | retryCount | No. of retries to attempt when a Task is marked as failure | defaults to 3 | | retryLogic | Mechanism for the retries | [Retry Logic values](#retry-logic) | | retryDelaySeconds | Time to wait before retries | defaults to 60 seconds | | timeoutPolicy | Task's timeout policy | [timeout policy values](#timeout-policy) | | timeoutSeconds | Time in seconds, after which the task is marked as `TIMED_OUT` if not completed after transitioning to `IN_PROGRESS` status for the first time | No timeouts if set to 0 | | pollTimeoutSeconds | Time in seconds, after which the task is marked as `TIMED_OUT` if not polled by a worker | No timeouts if set to 0 | | responseTimeoutSeconds | Must be greater than 0 and less than timeoutSeconds. The task is rescheduled if not updated with a status after this time (heartbeat mechanism). Useful when the worker polls for the task but fails to complete due to errors/network failure. | defaults to 3600 | | backoffScaleFactor | Must be greater than 0. Scale factor for linearity of the backoff | defaults to 1 | | inputKeys | Array of keys of task's expected input. Used for documenting task's input. See [Using inputKeys and outputKeys](#using-inputkeys-and-outputkeys). | optional | | outputKeys | Array of keys of task's expected output. Used for documenting task's output | optional | | inputTemplate | See [Using inputTemplate](#using-inputtemplate) below. | optional | | concurrentExecLimit | Number of tasks that can be executed at any given time. | optional | | rateLimitFrequencyInSeconds, rateLimitPerFrequency | See [Task Rate limits](#task-rate-limits) below. | optional | ### Retry Logic * FIXED : Reschedule the task after the ```retryDelaySeconds``` * EXPONENTIAL_BACKOFF : Reschedule after ```retryDelaySeconds 2^(attemptNumber)``` * LINEAR_BACKOFF : Reschedule after ```retryDelaySeconds * backoffRate * attemptNumber``` ### Timeout Policy * RETRY : Retries the task again * TIME_OUT_WF : Workflow is marked as TIMED_OUT and terminated * ALERT_ONLY : Registers a counter (task_timeout) ### Task Concurrent Execution Limits * `concurrentExecLimit` limits the number of simultaneous Task executions at any point. **Example:** If you have 1000 task executions waiting in the queue, and 1000 workers polling this queue for tasks, but if you have set `concurrentExecLimit` to 10, only 10 tasks would be given to workers (which would lead to starvation). If any of the workers finishes execution, a new task(s) will be removed from the queue, while still keeping the current execution count to 10. ### Task Rate limits > Note: Rate limiting is only supported for the Redis-persistence module and is not available with other persistence layers. * `rateLimitFrequencyInSeconds` and `rateLimitPerFrequency` should be used together. * `rateLimitFrequencyInSeconds` sets the "frequency window", i.e the `duration` to be used in `events per duration`. Eg: 1s, 5s, 60s, 300s etc. * `rateLimitPerFrequency`defines the number of Tasks that can be given to Workers per given "frequency window". **Example:** Let's set `rateLimitFrequencyInSeconds = 5`, and `rateLimitPerFrequency = 12`. This means, our frequency window is of 5 seconds duration, and for each frequency window, Conductor would only give 12 tasks to workers. So, in a given minute, Conductor would only give 12*(60/5) = 144 tasks to workers irrespective of the number of workers that are polling for the task. Note that unlike `concurrentExecLimit`, rate limiting doesn't take into account tasks already in progress/completed. Even if all the previous tasks are executed within 1 sec, or would take a few days, the new tasks are still given to workers at configured frequency, 144 tasks per minute in above example. ### Using inputKeys and outputKeys * `inputKeys` and `outputKeys` can be considered as parameters and return values for the Task. * Consider the task Definition as being represented by an interface: ```(value1, value2 .. valueN) someTaskDefinition(key1, key2 .. keyN);``` * However, these parameters are not strictly enforced at the moment. Both `inputKeys` and `outputKeys` act as a documentation for task re-use. The tasks in workflow need not define all of the keys in the task definition. * In the future, this can be extended to be a strict template that all task implementations must adhere to, just like interfaces in programming languages. ### Using inputTemplate * `inputTemplate` allows to define default values, which can be overridden by values provided in Workflow. * Eg: In your Task Definition, you can define your inputTemplate as: ```json "inputTemplate": { "url": "https://some_url:7004" } ``` * Now, in your workflow Definition, when using above task, you can use the default `url` or override with something else in the task's `inputParameters`. ```json "inputParameters": { "url": "${workflow.input.some_new_url}" } ``` ================================================ FILE: docs/docs/configuration/taskdomains.md ================================================ # Task Domains Task domains helps support task development. The idea is same “task definition” can be implemented in different “domains”. A domain is some arbitrary name that the developer controls. So when the workflow is started, the caller can specify, out of all the tasks in the workflow, which tasks need to run in a specific domain, this domain is then used to poll for task on the client side to execute it. As an example if a workflow (WF1) has 3 tasks T1, T2, T3. The workflow is deployed and working fine, which means there are T2 workers polling and executing. If you modify T2 and run it locally there is no guarantee that your modified T2 worker will get the task that you are looking for as it coming from the general T2 queue. “Task Domain” feature solves this problem by splitting the T2 queue by domains, so when the app polls for task T2 in a specific domain, it get the correct task. When starting a workflow multiple domains can be specified as a fall backs, for example "domain1,domain2". Conductor keeps track of last polling time for each task, so in this case it checks if the there are any active workers for "domain1" then the task is put in "domain1", if not then the same check is done for the next domain in sequence "domain2" and so on. If no workers are active for the domains provided: - If `NO_DOMAIN` is provided as last token in list of domains, then no domain is set. - Else, task will be added to last inactive domain in list of domains, hoping that workers would soon be available for that domain. Also, a `*` token can be used to apply domains for all tasks. This can be overridden by providing task specific mappings along with `*`. For example, the below configuration: ```json "taskToDomain": { "*": "mydomain", "some_task_x":"NO_DOMAIN", "some_task_y": "someDomain, NO_DOMAIN", "some_task_z": "someInactiveDomain1, someInactiveDomain2" } ``` - puts `some_task_x` in default queue (no domain). - puts `some_task_y` in `someDomain` domain, if available or in default otherwise. - puts `some_task_z` in `someInactiveDomain2`, even though workers are not available yet. - and puts all other tasks in `mydomain` (even if workers are not available). Note that this "fall back" type domain strings can only be used when starting the workflow, when polling from the client only one domain is used. Also, `NO_DOMAIN` token should be used last. ## How to use Task Domains ### Change the poll call The poll call must now specify the domain. #### Java Client If you are using the java client then a simple property change will force TaskRunnerConfigurer to pass the domain to the poller. ``` conductor.worker.T2.domain=mydomain //Task T2 needs to poll for domain "mydomain" ``` #### REST call `GET /tasks/poll/batch/T2?workerid=myworker&domain=mydomain` `GET /tasks/poll/T2?workerid=myworker&domain=mydomain` ### Change the start workflow call When starting the workflow, make sure the task to domain mapping is passes #### Java Client ``` Map input = new HashMap<>(); input.put("wf_input1", "one”); Map taskToDomain = new HashMap<>(); taskToDomain.put("T2", "mydomain"); // Other options ... // taskToDomain.put("*", "mydomain, NO_DOMAIN") // taskToDomain.put("T2", "mydomain, fallbackDomain1, fallbackDomain2") StartWorkflowRequest swr = new StartWorkflowRequest(); swr.withName(“myWorkflow”) .withCorrelationId(“corr1”) .withVersion(1) .withInput(input) .withTaskToDomain(taskToDomain); wfclient.startWorkflow(swr); ``` #### REST call `POST /workflow` ```json { "name": "myWorkflow", "version": 1, "correlatonId": "corr1" "input": { "wf_input1": "one" }, "taskToDomain": { "*": "mydomain", "some_task_x":"NO_DOMAIN", "some_task_y": "someDomain, NO_DOMAIN" } } ``` ================================================ FILE: docs/docs/configuration/workerdef.md ================================================ # Worker Definition A worker is responsible for executing a task. Operator and System tasks are handled by the Conductor server, while user defined tasks needs to have a worker created that awaits the work to be scheduled by the server for it to be executed. Workers can be implemented in any language, and Conductor provides support for Java, Golang and Python worker framework that provides features such as polling threads, metrics and server communication that makes creating workers easy. Each worker embodies Microservice design pattern and follows certain basic principles: 1. Workers are stateless and do not implement a workflow specific logic. 2. Each worker executes a very specific task and produces well defined output given specific inputs. 3. Workers are meant to be idempotent (or should handle cases where the task that partially executed gets rescheduled due to timeouts etc.) 4. Workers do not implement the logic to handle retries etc, that is taken care by the Conductor server. ================================================ FILE: docs/docs/configuration/workflowdef.md ================================================ # Workflow Definition ## What are Workflows? At a high level, a workflow is the Conductor primitive that encompasses the definition and flow of your business logic. A workflow is a collection (graph) of tasks and sub-workflows. A workflow definition specifies the order of execution of these [Tasks](taskdef.md). It also specifies how data/state is passed from one task to the other (using the input/output parameters). These are then combined to give you the final result. This orchestration of Tasks can happen in a hybrid ecosystem that includes microservices, serverless functions, and monolithic applications. They can also span across any public cloud and on-premise data center footprints. In addition, the orchestration of tasks can be across any programming language since Conductor is also language agnostic. One key benefit of this approach is that you can build a complex application using simple and granular tasks that do not need to be aware of or keep track of the state of your application's execution flow. Conductor keeps track of the state, calls tasks in the right order (sequentially or in parallel, as defined by you), retry calls if needed, handle failure scenarios gracefully, and outputs the final result. Leveraging workflows in Conductor enables developers to truly focus on their core mission - building their application code in the languages of their choice. Conductor does the heavy lifting associated with ensuring high reliability, transactional consistency, and long durability of their workflows. Simply put, wherever your application's component lives and whichever languages they were written in, you can build a workflow in Conductor to orchestrate their execution in a reliable & scalable manner. ## What does a Workflow look like? Let's start with a basic workflow and understand what are the different aspects of it. In particular, we will talk about two stages of a workflow, *defining* a workflow and *executing* a workflow ### Simple Workflow Example Assume your business logic is to simply to get some shipping information and then do the shipping. You start by logically partitioning them into two tasks: * **shipping_info** * **shipping_task** First we would build these two task definitions. Let's assume that ```shipping info``` takes an account number, and returns a name and address. **Example** ```json { "name": "mail_a_box", "description": "shipping Workflow", "version": 1, "tasks": [ { "name": "shipping_info", "taskReferenceName": "shipping_info_ref", "inputParameters": { "account": "${workflow.input.accountNumber}" }, "type": "SIMPLE" }, { "name": "shipping_task", "taskReferenceName": "shipping_task_ref", "inputParameters": { "name": "${shipping_info_ref.output.name}", "streetAddress": "${shipping_info_ref.output.streetAddress}", "city": "${shipping_info_ref.output.city}", "state": "${shipping_info_ref.output.state}", "zipcode": "${shipping_info_ref.output.zipcode}", }, "type": "SIMPLE" } ], "outputParameters": { "trackingNumber": "${shipping_task_ref.output.trackinNumber}" }, "failureWorkflow": "shipping_issues", "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "conductor@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ``` The mail_a_box workflow has 2 tasks: 1. The first task takes the provided account number, and outputs an address. 2. The 2nd task takes the address info and generates a shipping label. Upon completion of the 2 tasks, the workflow outputs the tracking number generated in the 2nd task. If the workflow fails, a second workflow named ```shipping_issues``` is run. ## Fields in a Workflow | Field | Description | Notes | |:------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------| | name | Name of the workflow || | description | Description of the workflow | optional | | version | Numeric field used to identify the version of the schema. Use incrementing numbers | When starting a workflow execution, if not specified, the definition with highest version is used | | tasks | An array of task definitions. | [Task properties](#tasks-within-workflow) | | inputParameters | List of input parameters. Used for documenting the required inputs to workflow | optional | | inputTemplate | Default input values. See [Using inputTemplate](#using-inputtemplate) | optional | | outputParameters | JSON template used to generate the output of the workflow | If not specified, the output is defined as the output of the _last_ executed task | | failureWorkflow | String; Workflow to be run on current Workflow failure. Useful for cleanup or post actions on failure. | optional | | schemaVersion | Current Conductor Schema version. schemaVersion 1 is discontinued. | Must be 2 | | restartable | Boolean flag to allow Workflow restarts | defaults to true | | workflowStatusListenerEnabled | If true, every workflow that gets terminated or completed will send a notification. See [workflow notifictions](#workflow-notifications) | optional (false by default) | ## Tasks within Workflow ```tasks``` property in a workflow execution defines an array of tasks to be executed in that order. | Field | Description | Notes | |:------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------| | name | Name of the task. MUST be registered as a task with Conductor before starting the workflow || | taskReferenceName | Alias used to refer the task within the workflow. MUST be unique within workflow. || | type | Type of task. SIMPLE for tasks executed by remote workers, or one of the system task types || | description | Description of the task | optional | | optional | true or false. When set to true - workflow continues even if the task fails. The status of the task is reflected as `COMPLETED_WITH_ERRORS` | Defaults to `false` | | inputParameters | JSON template that defines the input given to the task | See [Wiring Inputs and Outputs](#wiring-inputs-and-outputs) for details | | domain | See [Task Domains](/configuration/taskdomains.html) for more information. | optional | In addition to these parameters, System Tasks have their own parameters. Checkout [System Tasks](/configuration/systask.html) for more information. ## Wiring Inputs and Outputs Workflows are supplied inputs by client when a new execution is triggered. Workflow input is a JSON payload that is available via ```${workflow.input...}``` expressions. Each task in the workflow is given input based on the ```inputParameters``` template configured in workflow definition. ```inputParameters``` is a JSON fragment with value containing parameters for mapping values from input or output of a workflow or another task during the execution. Syntax for mapping the values follows the pattern as: __${SOURCE.input/output.JSONPath}__ | field | description | |--------------|--------------------------------------------------------------------------| | SOURCE | can be either "workflow" or any of the task reference name | | input/output | refers to either the input or output of the source | | JSONPath | JSON path expression to extract JSON fragment from source's input/output | !!! note "JSON Path Support" Conductor supports [JSONPath](http://goessner.net/articles/JsonPath/) specification and uses Java implementation from [here](https://github.com/jayway/JsonPath). !!! note "Escaping expressions" To escape an expression, prefix it with an extra _$_ character (ex.: ```$${workflow.input...}```). **Example** Consider a task with input configured to use input/output parameters from workflow and a task named __loc_task__. ```json { "inputParameters": { "movieId": "${workflow.input.movieId}", "url": "${workflow.input.fileLocation}", "lang": "${loc_task.output.languages[0]}", "http_request": { "method": "POST", "url": "http://example.com/${loc_task.output.fileId}/encode", "body": { "recipe": "${workflow.input.recipe}", "params": { "width": 100, "height": 100 } }, "headers": { "Accept": "application/json", "Content-Type": "application/json" } } } } ``` Consider the following as the _workflow input_ ```json { "movieId": "movie_123", "fileLocation":"s3://moviebucket/file123", "recipe":"png" } ``` And the output of the _loc_task_ as the following; ```json { "fileId": "file_xxx_yyy_zzz", "languages": ["en","ja","es"] } ``` When scheduling the task, Conductor will merge the values from workflow input and loc_task's output and create the input to the task as follows: ```json { "movieId": "movie_123", "url": "s3://moviebucket/file123", "lang": "en", "http_request": { "method": "POST", "url": "http://example.com/file_xxx_yyy_zzz/encode", "body": { "recipe": "png", "params": { "width": 100, "height": 100 } }, "headers": { "Accept": "application/json", "Content-Type": "application/json" } } } ``` ### Using inputTemplate * `inputTemplate` allows to define default values, which can be overridden by values provided in Workflow. * Eg: In your Workflow Definition, you can define your inputTemplate as: ```json "inputTemplate": { "url": "https://some_url:7004" } ``` And `url` would be `https://some_url:7004` if no `url` was provided as input to your workflow. ## Workflow notifications Conductor can be configured to publish notifications to external systems upon completion/termination of workflows. See [extending conductor](/extend.html) for details. ================================================ FILE: docs/docs/css/custom.css ================================================ :root { /*--main-text-color: #212121;*/ --brand-blue: #1976d2; --brand-dark-blue: #242A36; --caption-color: #4f4f4f; --brand-lt-blue: #f0f5fb; --brand-gray: rgb(118, 118, 118); --brand-lt-gray: rgb(203,204,207); --brand-red: #e50914; } body { color: var(--brand-dark-blue); font-family: "Roboto", sans-serif; font-weight: 400; } body::before { background: none; display: none; } body > .container { padding-top: 30px; } .bg-primary { background: #fff !important; } /* Navbar */ .navbar { box-shadow: 0 4px 8px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%); padding-left: 30px; padding-right: 30px; height: 80px; } .navbar-brand { background-image: url(/img/logo.svg); background-size: cover; color: transparent !important; padding: 0; text-shadow: none; margin-top: -6px; height: 37px; width: 175px; } .navbar-nav { margin-left: 50px; } .navbar-nav > .navitem, .navbar-nav > .dropdown { margin-left: 30px; } .navbar-nav > li .nav-link{ font-size: 15px; } .navbar-nav .nav-link { color: #242A36 !important; font-family: "Inter"; font-weight: 700; } .navbar-nav.ml-auto > li:first-child { display: none; } .navbar-nav.ml-auto .nav-link{ font-size: 0px; } .navbar-nav.ml-auto .nav-link .fa{ font-size: 30px; } .navbar-nav .dropdown-item { color: var(--brand-dark-blue); font-family: "Inter"; font-weight: 500; font-size: 14px; background-color: transparent; } .navbar-nav .dropdown-menu > li:hover { background-color: var(--brand-blue); } .navbar-nav .dropdown-menu > li:hover > .dropdown-item { color: #fff; } .navbar-nav .dropdown-submenu:hover > .dropdown-item { background-color: var(--brand-blue); } .navbar-nav .dropdown-menu li { margin: 0px; padding-top: 5px; padding-bottom: 5px; } .navbar-nav .dropdown-item.active { background-color: transparent; } .brand-darkblue { background: #242A36 !important; } .brand-gray { background: rgb(245,245,245); } .brand-blue { background: #1976D2; } .brand-white { background: #fff; } .logo { height: 444px; } /* Fonts */ h1, h2, h3, h4, h5, h6 { color: var(--brand-dark-blue); margin-bottom: 20px; } h1:first-child { margin-top: 0; } h1 { font-family: "Inter", sans-serif; font-size: 32px; font-weight: 700; margin-top: 50px; } h2 { font-family: "Inter", sans-serif; font-size: 24px; font-weight: 700; margin-top: 40px; } h3 { font-family: "Roboto", sans-serif; font-size: 20px; font-weight: 500; margin-top: 30px; } h4 { font-family: "Roboto", sans-serif; font-size: 18px; font-weight: 400; margin-top: 20px; } .main li { margin-bottom: 15px; } .btn { font-family: "Roboto", sans-serif; font-size: 14px; } .btn-primary { background: #1976D2; border: none; } .hero { padding-top: 100px; padding-bottom: 100px; } .hero .heading { font-size: 56px; font-weight: 900; line-height: 68px; } .hero .btn { font-size: 16px; padding: 10px 20px; } .hero .illustration { margin-left: 35px; } .bullets .heading, .module .heading { font-family: "Inter", sans-serif; font-size: 26px; font-weight: 700; } .bullets .row { margin-bottom: 60px; } .bullets .caption { padding-top: 10px; padding-right: 30px; } .icon { height: 25px; margin-right: 5px; vertical-align: -3px; } .caption { font-weight: 400; font-size: 17px; line-height: 24px; color: var(--caption-color); } .module { margin-top: 80px; margin-bottom: 80px; padding-top: 50px; padding-bottom: 50px; } .module .caption { padding-top: 10px; padding-right: 80px; } .module .screenshot { width: 600px; height: 337px; box-shadow:inset 0 1px 0 rgba(255,255,255,.6), 0 22px 70px 4px rgba(0,0,0,0.56), 0 0 0 1px rgba(0, 0, 0, 0.0); border-radius: 5px; background-size: cover; } /* Footer */ footer { margin: 0px; padding: 0px !important; text-align: left; font-weight: 400; } .footer { background-color: var(--brand-dark-blue); padding: 50px 0px; color: #fff; font-size: 14px; margin-top: 50px; } .footer a { color: var(--brand-lt-gray); } .footer .subhead { font-weight: 700; color: #fff; font-size: 15px; margin-bottom: 10px; } .footer .red { color: var(--brand-red); } .footer .fr { text-align: right; } /* TOC menu */ .toc ul { list-style: none; padding: 0px; } .toc > ul > li li { padding-left: 15px; font-weight: 400; font-size: 14px; } .toc > ul > li { font-size: 15px; font-weight: 500; } .toc .toc-link { margin-bottom: 5px; display: block; color: var(--brand-dark-blue); } .toc .toc-link.active { font-weight: 700; } /* Homepage Overrides */ .homepage > .container { max-width: none; } .homepage .toc { display: none; } /* Comparison block */ .compare { background-color: var(--brand-lt-blue); padding-top: 80px; padding-bottom: 80px; margin: 0px -15px; } .compare .heading { margin-bottom: 30px; margin-top: 0px; } .compare .bubble { background: #fff; border-radius: 10px; padding: 30px; height: 100%; } .compare .caption { font-size: 15px; line-height: 22px; } ================================================ FILE: docs/docs/extend.md ================================================ # Extending Conductor ## Backend Conductor provides a pluggable backend. The current implementation uses Dynomite. There are 4 interfaces that need to be implemented for each backend: ```java //Store for workflow and task definitions com.netflix.conductor.dao.MetadataDAO ``` ```java //Store for workflow executions com.netflix.conductor.dao.ExecutionDAO ``` ```java //Index for workflow executions com.netflix.conductor.dao.IndexDAO ``` ```java //Queue provider for tasks com.netflix.conductor.dao.QueueDAO ``` It is possible to mix and match different implementations for each of these. For example, SQS for queueing and a relational store for others. ## System Tasks To create system tasks follow the steps below: * Extend ```com.netflix.conductor.core.execution.tasks.WorkflowSystemTask``` * Instantiate the new class as part of the startup (eager singleton) * Implement the ```TaskMapper``` [interface](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/execution/mapper/TaskMapper.java) * Add this implementation to the map identified by [TaskMappers](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/config/CoreModule.java#L70) ## External Payload Storage To configure conductor to externalize the storage of large payloads: * Implement the `ExternalPayloadStorage` [interface](https://github.com/Netflix/conductor/blob/master/common/src/main/java/com/netflix/conductor/common/utils/ExternalPayloadStorage.java). * Add the storage option to the enum [here](https://github.com/Netflix/conductor/blob/master/server/src/main/java/com/netflix/conductor/bootstrap/ModulesProvider.java#L39). * Set this JVM system property ```workflow.external.payload.storage``` to the value of the enum element added above. * Add a binding similar to [this](https://github.com/Netflix/conductor/blob/master/server/src/main/java/com/netflix/conductor/bootstrap/ModulesProvider.java#L120-L127). ## Workflow Status Listener To provide a notification mechanism upon completion/termination of workflows: * Implement the ```WorkflowStatusListener``` [interface](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/execution/WorkflowStatusListener.java) * This can be configured to plugin custom notification/eventing upon workflows reaching a terminal state. ## Locking Service By default, Conductor Server module loads Zookeeper lock module. If you'd like to provide your own locking implementation module, for eg., with Dynomite and Redlock: * Implement ```Lock``` interface. * Add a binding similar to [this](https://github.com/Netflix/conductor/blob/master/server/src/main/java/com/netflix/conductor/bootstrap/ModulesProvider.java#L115-L129) * Enable locking service: ```conductor.app.workflowExecutionLockEnabled: true``` ================================================ FILE: docs/docs/externalpayloadstorage.md ================================================ # External Payload Storage !!!warning The external payload storage is currently only implemented to be used to by the Java client. Client libraries in other languages need to be modified to enable this. Contributions are welcomed. ## Context Conductor can be configured to enforce barriers on the size of workflow and task payloads for both input and output. These barriers can be used as safeguards to prevent the usage of conductor as a data persistence system and to reduce the pressure on its datastore. ## Barriers Conductor typically applies two kinds of barriers: * Soft Barrier * Hard Barrier #### Soft Barrier The soft barrier is used to alleviate pressure on the conductor datastore. In some special workflow use-cases, the size of the payload is warranted enough to be stored as part of the workflow execution. In such cases, conductor externalizes the storage of such payloads to S3 and uploads/downloads to/from S3 as needed during the execution. This process is completely transparent to the user/worker process. #### Hard Barrier The hard barriers are enforced to safeguard the conductor backend from the pressure of having to persist and deal with voluminous data which is not essential for workflow execution. In such cases, conductor will reject such payloads and will terminate/fail the workflow execution with the reasonForIncompletion set to an appropriate error message detailing the payload size. ## Usage ### Barriers setup Set the following properties to the desired values in the JVM system properties: | Property | Description | default value | | -- | -- | -- | | conductor.app.workflowInputPayloadSizeThreshold | Soft barrier for workflow input payload in KB | 5120 | | conductor.app.maxWorkflowInputPayloadSizeThreshold | Hard barrier for workflow input payload in KB | 10240 | | conductor.app.workflowOutputPayloadSizeThreshold | Soft barrier for workflow output payload in KB | 5120 | | conductor.app.maxWorkflowOutputPayloadSizeThreshold | Hard barrier for workflow output payload in KB | 10240 | | conductor.app.taskInputPayloadSizeThreshold | Soft barrier for task input payload in KB | 3072 | | conductor.app.maxTaskInputPayloadSizeThreshold | Hard barrier for task input payload in KB | 10240 | | conductor.app.taskOutputPayloadSizeThreshold | Soft barrier for task output payload in KB | 3072 | | conductor.app.maxTaskOutputPayloadSizeThreshold | Hard barrier for task output payload in KB | 10240 | ### Amazon S3 Conductor provides an implementation of [Amazon S3](https://aws.amazon.com/s3/) used to externalize large payload storage. Set the following property in the JVM system properties: ``` conductor.external-payload-storage.type=S3 ``` !!!note This [implementation](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/utils/S3PayloadStorage.java#L44-L45) assumes that S3 access is configured on the instance. Set the following properties to the desired values in the JVM system properties: | Property | Description | default value | | --- | --- | --- | | conductor.external-payload-storage.s3.bucketName | S3 bucket where the payloads will be stored | | | conductor.external-payload-storage.s3.signedUrlExpirationDuration | The expiration time in seconds of the signed url for the payload | 5 | The payloads will be stored in the bucket configured above in a `UUID.json` file at locations determined by the type of the payload. See [here](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/utils/S3PayloadStorage.java#L149-L167) for information about how the object key is determined. ### Azure Blob Storage ProductLive provides an implementation of [Azure Blob Storage](https://azure.microsoft.com/services/storage/blobs/) used to externalize large payload storage. To build conductor with azure blob feature read the [README.md](https://github.com/Netflix/conductor/blob/master/azureblob-storage/README.md) in `azureblob-storage` module !!!note This implementation assumes that you have an [Azure Blob Storage account's connection string or SAS Token](https://github.com/Azure/azure-sdk-for-java/blob/master/sdk/storage/azure-storage-blob/README.md). If you want signed url to expired you must specify a Connection String. Set the following properties to the desired values in the JVM system properties: | Property | Description | default value | | --- | --- | --- | | workflow.external.payload.storage.azure_blob.connection_string | Azure Blob Storage connection string. Required to sign Url. | | | workflow.external.payload.storage.azure_blob.endpoint | Azure Blob Storage endpoint. Optional if connection_string is set. | | | workflow.external.payload.storage.azure_blob.sas_token | Azure Blob Storage SAS Token. Must have permissions `Read` and `Write` on Resource `Object` on Service `Blob`. Optional if connection_string is set. | | | workflow.external.payload.storage.azure_blob.container_name | Azure Blob Storage container where the payloads will be stored | `conductor-payloads` | | workflow.external.payload.storage.azure_blob.signedurlexpirationseconds | The expiration time in seconds of the signed url for the payload | 5 | | workflow.external.payload.storage.azure_blob.workflow_input_path | Path prefix where workflows input will be stored with an random UUID filename | workflow/input/ | | workflow.external.payload.storage.azure_blob.workflow_output_path | Path prefix where workflows output will be stored with an random UUID filename | workflow/output/ | | workflow.external.payload.storage.azure_blob.task_input_path | Path prefix where tasks input will be stored with an random UUID filename | task/input/ | | workflow.external.payload.storage.azure_blob.task_output_path | Path prefix where tasks output will be stored with an random UUID filename | task/output/ | The payloads will be stored as done in [Amazon S3](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/utils/S3PayloadStorage.java#L149-L167). ### PostgreSQL Storage Frinx provides an implementation of [PostgreSQL Storage](https://www.postgresql.org/) used to externalize large payload storage. !!!note This implementation assumes that you have an [PostgreSQL database server with all required credentials](https://jdbc.postgresql.org/documentation/94/connect.html). Set the following properties to your application.properties: | Property | Description | default value | |-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------| | conductor.external-payload-storage.postgres.conductor-url | URL, that can be used to pull the json configurations, that will be downloaded from PostgreSQL to the conductor server. For example: for local development it is `http://localhost:8080` | `""` | | conductor.external-payload-storage.postgres.url | PostgreSQL database connection URL. Required to connect to database. | | | conductor.external-payload-storage.postgres.username | Username for connecting to PostgreSQL database. Required to connect to database. | | | conductor.external-payload-storage.postgres.password | Password for connecting to PostgreSQL database. Required to connect to database. | | | conductor.external-payload-storage.postgres.table-name | The PostgreSQL schema and table name where the payloads will be stored | `external.external_payload` | | conductor.external-payload-storage.postgres.max-data-rows | Maximum count of data rows in PostgreSQL database. After overcoming this limit, the oldest data will be deleted. | Long.MAX_VALUE (9223372036854775807L) | | conductor.external-payload-storage.postgres.max-data-days | Maximum count of days of data age in PostgreSQL database. After overcoming limit, the oldest data will be deleted. | 0 | | conductor.external-payload-storage.postgres.max-data-months | Maximum count of months of data age in PostgreSQL database. After overcoming limit, the oldest data will be deleted. | 0 | | conductor.external-payload-storage.postgres.max-data-years | Maximum count of years of data age in PostgreSQL database. After overcoming limit, the oldest data will be deleted. | 1 | The maximum date age for fields in the database will be: `years + months + days` The payloads will be stored in PostgreSQL database with key (externalPayloadPath) `UUID.json` and you can generate URI for this data using `external-postgres-payload-resource` rest controller. To make this URI work correctly, you must correctly set the conductor-url property. ================================================ FILE: docs/docs/faq.md ================================================ # Frequently asked Questions ### How do you schedule a task to be put in the queue after some time (e.g. 1 hour, 1 day etc.) After polling for the task update the status of the task to `IN_PROGRESS` and set the `callbackAfterSeconds` value to the desired time. The task will remain in the queue until the specified second before worker polling for it will receive it again. If there is a timeout set for the task, and the `callbackAfterSeconds` exceeds the timeout value, it will result in task being TIMED_OUT. ### How long can a workflow be in running state? Can I have a workflow that keeps running for days or months? Yes. As long as the timeouts on the tasks are set to handle long running workflows, it will stay in running state. ### My workflow fails to start with missing task error Ensure all the tasks are registered via `/metadata/taskdefs` APIs. Add any missing task definition (as reported in the error) and try again. ### Where does my worker run? How does conductor run my tasks? Conductor does not run the workers. When a task is scheduled, it is put into the queue maintained by Conductor. Workers are required to poll for tasks using `/tasks/poll` API at periodic interval, execute the business logic for the task and report back the results using `POST /tasks` API call. Conductor, however will run [system tasks](/configuration/systask.html) on the Conductor server. ### How can I schedule workflows to run at a specific time? Netflix Conductor itself does not provide any scheduling mechanism. But there is a community project [_Schedule Conductor Workflows_](https://github.com/jas34/scheduledwf) which provides workflow scheduling capability as a pluggable module as well as workflow server. Other way is you can use any of the available scheduling systems to make REST calls to Conductor to start a workflow. Alternatively, publish a message to a supported eventing system like SQS to trigger a workflow. More details about [eventing](/configuration/eventhandlers.html). ### How do I setup Dynomite cluster? Visit Dynomite's [Github page](https://github.com/Netflix/dynomite) to find details on setup and support mechanism. ### Can I use conductor with Ruby / Go / Python? Yes. Workers can be written any language as long as they can poll and update the task results via HTTP endpoints. Conductor provides frameworks for Java and Python to simplify the task of polling and updating the status back to Conductor server. **Note:** Python and Go clients have been contributed by the community. ### How can I get help with Dynomite? Visit Dynomite's [Github page](https://github.com/Netflix/dynomite) to find details on setup and support mechanism. ### My workflow is running and the task is SCHEDULED but it is not being processed. Make sure that the worker is actively polling for this task. Navigate to the `Task Queues` tab on the Conductor UI and select your task name in the search box. Ensure that `Last Poll Time` for this task is current. In Conductor 3.x, ```conductor.redis.availabilityZone``` defaults to ```us-east-1c```. Ensure that this matches where your workers are, and that it also matches```conductor.redis.hosts```. ### How do I configure a notification when my workflow completes or fails? When a workflow fails, you can configure a "failure workflow" to run using the```failureWorkflow``` parameter. By default, three parameters are passed: * reason * workflowId: use this to pull the details of the failed workflow. * failureStatus You can also use the Workflow Status Listener: * Set the workflowStatusListenerEnabled field in your workflow definition to true which enables [notifications](/configuration/workflowdef.html#workflow-notifications). * Add a custom implementation of the Workflow Status Listener. Refer [this](/extend.html#workflow-status-listener). * This notification can be implemented in such a way as to either send a notification to an external system or to send an event on the conductor queue to complete/fail another task in another workflow as described [here](/configuration/eventhandlers.html). Refer to this [documentation](/configuration/workflowdef.html#workflow-notifications) to extend conductor to send out events/notifications upon workflow completion/failure. ### I want my worker to stop polling and executing tasks when the process is being terminated. (Java client) In a `PreDestroy` block within your application, call the `shutdown()` method on the `TaskRunnerConfigurer` instance that you have created to facilitate a graceful shutdown of your worker in case the process is being terminated. ### Can I exit early from a task without executing the configured automatic retries in the task definition? Set the status to `FAILED_WITH_TERMINAL_ERROR` in the TaskResult object within your worker. This would mark the task as FAILED and fail the workflow without retrying the task as a fail-fast mechanism. ================================================ FILE: docs/docs/gettingstarted/basicconcepts.md ================================================ # Basic Concepts ## Definitions (aka Metadata or Blueprints) Conductor definitions are like class definitions in OOP paradigm, or templates. You define this once, and use for each workflow execution. Definitions to Executions have 1:N relationship. ## Tasks Tasks are the building blocks of Workflow. There must be at least one task in a Workflow. Tasks can be categorized into two types: * [System tasks](/configuration/systask.html) - executed by Conductor server. * [Worker tasks](/configuration/workerdef.html) - executed by your own workers. ## Workflow A Workflow is the container of your process flow. It could include several different types of Tasks, Sub-Workflows, inputs and outputs connected to each other, to effectively achieve the desired result. The tasks are either control tasks (fork, conditional etc) or application tasks (e.g. encode a file) that are executed on a remote machine. [Detailed description](/configuration/workflowdef.html) ## Task Definition Task definitions help define Task level parameters like inputs and outputs, timeouts, retries etc. * All tasks need to be registered before they can be used by active workflows. * A task can be re-used within multiple workflows. [Detailed description](/configuration/taskdef.html) ## System Tasks System tasks are executed within the JVM of the Conductor server and managed by Conductor for its execution and scalability. See [Systems tasks](/configuration/systask.html) for list of available Task types, and instructions for using them. !!! Note Conductor provides an API to create user defined tasks that are executed in the same JVM as the engine. See [WorkflowSystemTask](https://github.com/Netflix/conductor/blob/main/core/src/main/java/com/netflix/conductor/core/execution/tasks/WorkflowSystemTask.java) interface for details. ## Worker Tasks Worker tasks are implemented by your application(s) and run in a separate environment from Conductor. The worker tasks can be implemented in any language. These tasks talk to Conductor server via REST/gRPC to poll for tasks and update its status after execution. Worker tasks are identified by task type __SIMPLE__ in the blueprint. ================================================ FILE: docs/docs/gettingstarted/client.md ================================================ # Using the Client Conductor tasks that are executed by remote workers communicate over HTTP endpoints/gRPC to poll for the task and update the status of the execution. ## Client APIs Conductor provides the following java clients to interact with the various APIs | Client | Usage | |-----------------|---------------------------------------------------------------------------| | Metadata Client | Register / Update workflow and task definitions | | Workflow Client | Start a new workflow / Get execution status of a workflow | | Task Client | Poll for task / Update task result after execution / Get status of a task | ## SDKs * [Clojure](/how-tos/clojure-sdk.html) * [C#](/how-tos/csharp-sdk.html) * [Go](/how-tos/go-sdk.html) * [Java](/how-tos/java-sdk.html) * [Python](/how-tos/python-sdk.html) The non-Java Conductor SDKs are hosted on a separate GitHub repository: [github.com/conductor-sdk](https://github.com/conductor-sdk). Contributions from the community are encouraged! ================================================ FILE: docs/docs/gettingstarted/docker.md ================================================ # Running Conductor using Docker In this article we will explore how you can set up Netflix Conductor on your local machine using Docker compose. The docker compose will bring up the following: 1. Conductor API Server 2. Conductor UI 3. Elasticsearch for searching workflows ## Prerequisites 1. Docker: [https://docs.docker.com/get-docker/](https://docs.docker.com/get-docker/) 2. Recommended host with CPU and RAM to be able to run multiple docker containers (at-least 16GB RAM) ## Steps ### 1. Clone the Conductor Code ```shell $ git clone https://github.com/Netflix/conductor.git ``` ### 2. Build the Docker Compose ```shell $ cd conductor conductor $ cd docker docker $ docker-compose build ``` ### 3. Run Docker Compose ```shell docker $ docker-compose up ``` Once up and running, you will see the following in your Docker dashboard: 1. Elasticsearch 2. Conductor UI 3. Conductor Server You can access the UI & Server on your browser to verify that they are running correctly: #### Conductor Server URL [http://localhost:8080](http://localhost:8080) #### Conductor UI URL [http://localhost:5000/](http://localhost:5000) ### 4. Exiting Compose `Ctrl+c` will exit docker compose. To ensure images are stopped execute: `docker-compose down`. ## Alternative Persistence Engines By default `docker-compose.yaml` uses `config-local.properties`. This configures the `memory` database, where data is lost when the server terminates. This configuration is useful for testing or demo only. A selection of `docker-compose-*.yaml` and `config-*.properties` files are provided demonstrating the use of alternative persistence engines. | File | Containers | |--------------------------------|-----------------------------------------------------------------------------------------| | docker-compose.yaml |

    1. In Memory Conductor Server
    2. Elasticsearch
    3. UI
    | | docker-compose-dynomite.yaml |
    1. Conductor Server
    2. Elasticsearch
    3. UI
    4. Dynomite Redis for persistence
    | | docker-compose-postgres.yaml |
    1. Conductor Server
    2. Elasticsearch
    3. UI
    4. Postgres persistence
    | | docker-compose-prometheus.yaml | Brings up Prometheus server | For example this will start the server instance backed by a PostgreSQL DB. ``` docker-compose -f docker-compose.yaml -f docker-compose-postgres.yaml up ``` ## Standalone Server Image To build and run the server image, without using `docker-compose`, from the `docker` directory execute: ``` docker build -t conductor:server -f server/Dockerfile ../ docker run -p 8080:8080 -d --name conductor_server conductor:server ``` This builds the image `conductor:server` and runs it in a container named `conductor_server`. The API should now be accessible at `localhost:8080`. To 'login' to the running container, use the command: ``` docker exec -it conductor_server /bin/sh ``` ## Standalone UI Image From the `docker` directory, ``` docker build -t conductor:ui -f ui/Dockerfile ../ docker run -p 5000:5000 -d --name conductor_ui conductor:ui ``` This builds the image `conductor:ui` and runs it in a container named `conductor_ui`. The UI should now be accessible at `localhost:5000`. ### Note * In order for the UI to do anything useful the Conductor Server must already be running on port 8080, either in a Docker container (see above), or running directly in the local JRE. * Additionally, significant parts of the UI will not be functional without Elastisearch being available. Using the `docker-compose` approach alleviates these considerations. ## Monitoring with Prometheus Start Prometheus with: `docker-compose -f docker-compose-prometheus.yaml up -d` Go to [http://127.0.0.1:9090](http://127.0.0.1:9090). ## Combined Server & UI Docker Image This image at `/docker/serverAndUI` is provided to illustrate starting both the server & UI within the same container. The UI is hosted using nginx. ### Building the combined image From the `docker` directory, ``` docker build -t conductor:serverAndUI -f serverAndUI/Dockerfile ../ ``` ### Running the combined image - With interal DB: `docker run -p 8080:8080 -p 80:5000 -d -t conductor:serverAndUI` - With external DB: `docker run -p 8080:8080 -p 80:5000 -d -t -e "CONFIG_PROP=config.properties" conductor:serverAndUI` ## Potential problem when using Docker Images #### Not enough memory 1. You will need at least 16 GB of memory to run everything. You can modify the docker compose to skip using Elasticsearch if you have no option to run this with your memory options. 2. To disable Elasticsearch using Docker Compose - follow the steps listed here: **TODO LINK** #### Elasticsearch fails to come up in arm64 based CPU machines 1. As of writing this article, Conductor relies on 6.8.x version of Elasticsearch. This version doesn't have an arm64 based Docker image. You will need to use Elasticsearch 7.x which requires a bit of customization to get up and running #### Elasticsearch remains in yellow health state When you run Elasticsearch, sometimes the health remains in the *yellow* state. Conductor server by default requires *green* state to run when indexing is enabled. To work around this, you can use the following property: `conductor.elasticsearch.clusterHealthColor=yellow`. Reference: [Issue 2262][issue2262] #### Elasticsearch timeout By default, a standalone (single node) Elasticsearch has a *yellow* status which will cause timeout (`java.net.SocketTimeoutException`) for Conductor server (required status is *green*). Spin up a cluster (more than one node) to prevent the timeout or use config option `conductor.elasticsearch.clusterHealthColor=yellow`. Reference: [Issue 2262][issue2262] #### Changes in config-*.properties do not take effect Config is copy into image during docker build. You have to rebuild the image or better, link a volume to it to reflect new changes. #### To troubleshoot a failed startup Check the log of the server, which is located at `/app/logs` (default directory in dockerfile) #### Unable to access to conductor:server API on port 8080 It may takes some time for conductor server to start. Please check server log for potential error. #### Elasticsearch Elasticsearch is optional, please be aware that disable it will make most of the conductor UI not functional. ##### How to enable Elasticsearch * Set `conductor.indexing.enabled=true` in your_config.properties * Add config related to elasticsearch E.g.: `conductor.elasticsearch.url=http://es:9200` ##### How to disable Elasticsearch * Set `conductor.indexing.enabled=false` in your_config.properties * Comment out all the config related to elasticsearch E.g.: `conductor.elasticsearch.url=http://es:9200` [issue2262]: https://github.com/Netflix/conductor/issues/2262 ================================================ FILE: docs/docs/gettingstarted/hosted.md ================================================ # Hosted Solutions ## Orkes [Orkes](https://orkes.io) is a commercial vendor that offers a cloud hosted version of Conductor requiring minimal operational investment to get started. ### Developer Playground Orkes provides a developer playground for Conductor at [https://play.orkes.io/](https://play.orkes.io/). The playground is self-service and is free to try. The playground allows you to create new workflow definitions, tasks and execute them using the UI or through APIs. Orkes also operates a Slack community featuring discussion and guidance for their product and Conductor in general. ### Cloud Hosted Conductor Orkes provides multiple options of hosted Conductor clusters in the cloud (AWS, Azure, and GCP, in addition to private clouds) with enterprise support provided by the Orkes team. Beyond full compatibility with the open source Netflix Conductor, the Orkes product provides additional features, such as in the area of security and analytics, not present in the open source release. ================================================ FILE: docs/docs/gettingstarted/intro.md ================================================ # Why Conductor? ## Conductor was built to help Netflix orchestrate microservices based process flows with the following features: * A distributed server ecosystem, which stores workflow state information efficiently. * Allow creation of process / business flows in which each individual task can be implemented by the same / different microservices. * A DAG (Directed Acyclic Graph) based workflow definition. * Workflow definitions are decoupled from the service implementations. * Provide visibility and traceability into these process flows. * Simple interface to connect workers, which execute the tasks in workflows. * Workers are language agnostic, allowing each microservice to be written in the language most suited for the service. * Full operational control over workflows with the ability to pause, resume, restart, retry and terminate. * Allow greater reuse of existing microservices providing an easier path for onboarding. * User interface to visualize, replay and search the process flows. * Ability to scale to millions of concurrently running process flows. * Backed by a queuing service abstracted from the clients. * Be able to operate on HTTP or other transports e.g. gRPC. * Event handlers to control workflows via external actions. * Client implementations in Java, Python and other languages. * Various configurable properties with sensible defaults to fine tune workflow and task executions like rate limiting, concurrent execution limits etc. ## Why not peer to peer choreography? With peer to peer task choreography, we found it was harder to scale with growing business needs and complexities. Pub/sub model worked for simplest of the flows, but quickly highlighted some of the issues associated with the approach: * Process flows are “embedded” within the code of multiple application. * Often, there is tight coupling and assumptions around input/output, SLAs etc, making it harder to adapt to changing needs. * Almost no way to systematically answer “How much are we done with process X”? ================================================ FILE: docs/docs/gettingstarted/source.md ================================================ # Building Conductor From Source ## Build and Run In this article we will explore how you can set up Netflix Conductor on your local machine for trying out some of its features. ### Prerequisites 1. JDK 17 or greater 2. (Optional) Docker if you want to run tests. You can install docker from [here](https://www.docker.com/get-started/). 3. Node for building and running UI. Instructions at [https://nodejs.org](https://nodejs.org). 4. Yarn for building and running UI. Instructions at [https://classic.yarnpkg.com/en/docs/install](https://classic.yarnpkg.com/en/docs/install). ### Steps to build Conductor Server #### 1. Checkout the code Clone conductor code from the repo: https://github.com/Netflix/conductor ```shell $ git clone https://github.com/Netflix/conductor.git ``` #### 2. Build and run Server > **NOTE for Mac users**: If you are using a new Mac with an Apple Silicon Chip, you must make a small change to ```conductor/grpc/build.gradle``` - adding "osx-x86_64" to two lines: ``` protobuf { protoc { artifact = "com.google.protobuf:protoc:${revProtoBuf}:osx-x86_64" } plugins { grpc { artifact = "io.grpc:protoc-gen-grpc-java:${revGrpc}:osx-x86_64" } } ... } ``` You may also need to install rosetta: ```shell softwareupdate --install-rosetta ``` ```shell $ cd conductor conductor $ cd server server $ ../gradlew bootRun ``` Navigate to the swagger API docs: [http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config) ## Download and Run As an alternative to building from source, you can download and run the pre-compiled JAR. ```shell export CONDUCTOR_VER=3.3.4 export REPO_URL=https://repo1.maven.org/maven2/com/netflix/conductor/conductor-server curl $REPO_URL/$CONDUCTOR_VER/conductor-server-$CONDUCTOR_VER-boot.jar \ --output conductor-server-$CONDUCTOR_VER-boot.jar; java -jar conductor-server-$CONDUCTOR_VER-boot.jar ``` Navigate to the swagger URL: [http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config) ## Build and Run UI ### Conductor UI from Source The UI is a standard `create-react-app` React Single Page Application (SPA). To get started, with Node 14 and `yarn` installed, first run `yarn install` from within the `/ui` directory to retrieve package dependencies. ```shell $ cd conductor/ui ui $ yarn install ``` There is no need to "build" the project unless you require compiled assets to host on a production web server. If the latter is true, the project can be built with the command `yarn build`. To run the UI on the bundled development server, run `yarn run start`. Navigate your browser to `http://localhost:5000`. The server must already be running on port 8080. ```shell ui $ yarn run start ``` Launch UI [http://localhost:5000](http://localhost:5000) ## Summary 1. By default in-memory persistence is used, so any workflows created or excuted will be wiped out once the server is terminated. 2. Without indexing configured, the search functionality in UI will not work and will result an empty set. 3. See how to install Conductor using [Docker](docker.md) with persistence and indexing. ================================================ FILE: docs/docs/gettingstarted/startworkflow.md ================================================ # Starting a Workflow ## Start Workflow Endpoint When starting a Workflow execution with a registered definition, `/workflow` accepts following parameters: | Field | Description | Notes | |:--------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| | name | Name of the Workflow. MUST be registered with Conductor before starting workflow | | | version | Workflow version | defaults to latest available version | | input | JSON object with key value params, that can be used by downstream tasks | See [Wiring Inputs and Outputs](/configuration/workflowdef.html#wiring-inputs-and-outputs) for details | | correlationId | Unique Id that correlates multiple Workflow executions | optional | | taskToDomain | See [Task Domains](/configuration/taskdomains.html) for more information. | optional | | workflowDef | An adhoc [Workflow Definition](/configuration/workflowdef.html) to run, without registering. See [Dynamic Workflows](#dynamic-workflows). | optional | | externalInputPayloadStoragePath | This is taken care of by Java client. See [External Payload Storage](/externalpayloadstorage.html) for more info. | optional | | priority | Priority level for the tasks within this workflow execution. Possible values are between 0 - 99. | optional | **Example:** Send a `POST` request to `/workflow` with payload like: ```json { "name": "encode_and_deploy", "version": 1, "correlationId": "my_unique_correlation_id", "input": { "param1": "value1", "param2": "value2" } } ``` ## Dynamic Workflows If the need arises to run a one-time workflow, and it doesn't make sense to register Task and Workflow definitions in Conductor Server, as it could change dynamically for each execution, dynamic workflow executions can be used. This enables you to provide a workflow definition embedded with the required task definitions to the Start Workflow Request in the `workflowDef` parameter, avoiding the need to register the blueprints before execution. **Example:** Send a `POST` request to `/workflow` with payload like: ```json { "name": "my_adhoc_unregistered_workflow", "workflowDef": { "ownerApp": "my_owner_app", "ownerEmail": "my_owner_email@test.com", "createdBy": "my_username", "name": "my_adhoc_unregistered_workflow", "description": "Test Workflow setup", "version": 1, "tasks": [ { "name": "fetch_data", "type": "HTTP", "taskReferenceName": "fetch_data", "inputParameters": { "http_request": { "connectionTimeOut": "3600", "readTimeOut": "3600", "uri": "${workflow.input.uri}", "method": "GET", "accept": "application/json", "content-Type": "application/json", "headers": { } } }, "taskDefinition": { "name": "fetch_data", "retryCount": 0, "timeoutSeconds": 3600, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 0, "responseTimeoutSeconds": 3000 } } ], "outputParameters": { } }, "input": { "uri": "http://www.google.com" } } ``` !!! Note If the `taskDefinition` is defined with Metadata API, it doesn't have to be added in above dynamic workflow definition. ================================================ FILE: docs/docs/gettingstarted/steps.md ================================================ # High Level Steps Steps required for a new workflow to be registered and get executed 1. Define task definitions used by the workflow. 2. Create the workflow definition 3. Create task worker(s) that polls for scheduled tasks at regular interval ### Trigger Workflow Execution ``` POST /workflow/{name} { ... //json payload as workflow input } ``` ### Polling for a task ``` GET /tasks/poll/batch/{taskType} ``` ### Update task status ``` POST /tasks { "outputData": { "encodeResult":"success", "location": "http://cdn.example.com/file/location.png" //any task specific output }, "status": "COMPLETED" } ``` ================================================ FILE: docs/docs/googleba55068fa3e0e553.html ================================================ google-site-verification: googleba55068fa3e0e553.html ================================================ FILE: docs/docs/how-tos/Monitoring/Conductor-LogLevel.md ================================================ # Conductor Log Level Conductor is based on Spring Boot, so the log levels are set via [Spring Boot properties](https://docs.spring.io/spring-boot/docs/2.1.13.RELEASE/reference/html/boot-features-logging.html): From the Spring Boot Docs: > All the supported logging systems can have the logger levels set in the Spring Environment (for example, in application.properties) by using ```logging.level.=``` where level is one of TRACE, DEBUG, INFO, WARN, ERROR, FATAL, or OFF. The ```root``` logger can be configured by using logging.level.root. > The following example shows potential logging settings in ```application.properties```: ``` logging.level.root=warn logging.level.org.springframework.web=debug logging.level.org.hibernate=error ``` It’s also possible to set logging levels using environment variables. For example, ```LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG``` will set ```org.springframework.web``` to `DEBUG`. ================================================ FILE: docs/docs/how-tos/Tasks/creating-tasks.md ================================================ # Creating Task Definitions Tasks can be created using the tasks metadata API `POST /api/metadata/taskdefs` This API takes an array of new task definitions. ### Example using curl ```shell curl 'http://localhost:8080/api/metadata/taskdefs' \ -H 'accept: */*' \ -H 'content-type: application/json' \ --data-raw '[{"createdBy":"user","name":"sample_task_name_1","description":"This is a sample task for demo","responseTimeoutSeconds":10,"timeoutSeconds":30,"inputKeys":[],"outputKeys":[],"timeoutPolicy":"TIME_OUT_WF","retryCount":3,"retryLogic":"FIXED","retryDelaySeconds":5,"inputTemplate":{},"rateLimitPerFrequency":0,"rateLimitFrequencyInSeconds":1}]' ``` ### Example using node fetch ```javascript fetch("http://localhost:8080/api/metadata/taskdefs", { "headers": { "accept": "*/*", "content-type": "application/json", }, "body": "[{\"createdBy\":\"user\",\"name\":\"sample_task_name_1\",\"description\":\"This is a sample task for demo\",\"responseTimeoutSeconds\":10,\"timeoutSeconds\":30,\"inputKeys\":[],\"outputKeys\":[],\"timeoutPolicy\":\"TIME_OUT_WF\",\"retryCount\":3,\"retryLogic\":\"FIXED\",\"retryDelaySeconds\":5,\"inputTemplate\":{},\"rateLimitPerFrequency\":0,\"rateLimitFrequencyInSeconds\":1}]", "method": "POST" }); ``` ## Best Practices 1. You can update a set of tasks together in this API 2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](/configuration/taskdef.html) for all the options and details' 3. You can also use the Conductor Swagger UI to update the tasks ================================================ FILE: docs/docs/how-tos/Tasks/dynamic-vs-switch-tasks.md ================================================ --- sidebar_position: 1 --- # Dynamic vs Switch Tasks Learn more about 1. [Dynamic Tasks](/reference-docs/dynamic-task.html) 2. [Switch Tasks](/reference-docs/switch-task.html) Dynamic Tasks are useful in situations when need to run a task of which the task type is determined at runtime instead of during the configuration. It is similar to the [SWITCH](/reference-docs/switch-task.html) use case but with `DYNAMIC` we won't need to preconfigure all case options in the workflow definition itself. Instead, we can mark the task as `DYNAMIC` and determine which underlying task does it run during the workflow execution itself. 1. Use DYNAMIC task as a replacement for SWITCH if you have too many case options 2. DYNAMIC task is an option when you want to programmatically determine the next task to run instead of using expressions 3. DYNAMIC task simplifies the workflow execution UI view which will now only show the selected task 4. SWITCH task visualization is helpful as a documentation - showing you all options that the workflow could have taken 5. SWITCH task comes with a default task option which can be useful in some use cases ================================================ FILE: docs/docs/how-tos/Tasks/extending-system-tasks.md ================================================ # Extending System Tasks [System tasks](/configuration/systask.html) allow Conductor to run simple tasks on the server - removing the need to build (and deploy) workers for basic tasks. This allows for automating more mundane tasks without building specific microservices for them. However, sometimes it might be necessary to add additional parameters to a System Task to gain the behavior that is desired. ## Example HTTP Task ```json { "name": "get_weather_90210", "version": 1, "tasks": [ { "name": "get_weather_90210", "taskReferenceName": "get_weather_90210", "inputParameters": { "http_request": { "uri": "https://weatherdbi.herokuapp.com/data/weather/90210", "method": "GET", "connectionTimeOut": 1300, "readTimeOut": 1300 } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "data": "${get_weather_ref.output.response.body.currentConditions.comment}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "conductor@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ``` This very simple workflow has a single HTTP Task inside. No parameters need to be passed, and when run, the HTTP task will return the weather in Beverly Hills, CA (Zip code = 90210). > This API has a very slow response time. In the HTTP task, the connection is set to time out after 1300ms, which is *too short* for this API, resulting in a timeout. This API *will* work if we allowed for a longer timeout, but in order to demonstrate adding retries to the HTTP Task, we will artificially force the API call to fail. When this workflow is run - it fails, as expected. Now, sometimes an API call might fail due to an issue on the remote server, and retrying the call will result in a response. With many Conductor tasks, ```retryCount```, ```retryDelaySeconds``` and ```retryLogic``` fields can be applied to retry the worker (with the desired parameters). By default, the [HTTP Task](/reference-docs/http-task.html) does not have ```retryCount```, ```retryDelaySeconds``` or ```retryLogic``` built in. Attempting to add these parameters to a HTTP Task results in an error. ## The Solution We can create a task with the same name with the desired parameters. Defining the following task (note that the ```name``` is identical to the one in the workflow): ```json { "createdBy": "", "name": "get_weather_90210", "description": "editing HTTP task", "retryCount": 3, "timeoutSeconds": 5, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 5, "responseTimeoutSeconds": 5, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 } ``` We've added the three parameters: ```retryCount: 3, retryDelaySeconds: 5, retryLogic: FIXED``` The ```get_weather_90210``` task will now run 4 times (it will fail once, and then retry 3 times), with a ```FIXED``` 5 second delay between attempts. Re-running the task (and looking at the timeline view) shows that this is what occurs. There are 4 attempts, with a 5 second delay between them. If we change the ```retryLogic``` to EXPONENTIAL_BACKOFF, the delay between attempts grows exponentially: 1. 5*2^0 = 5 seconds 2. 5*2^1 = 10 seconds 3. 5*2^2 = 20 seconds ================================================ FILE: docs/docs/how-tos/Tasks/monitoring-task-queues.md ================================================ --- sidebar_position: 1 --- # Monitoring Task Queues Conductor offers an API and UI interface to monitor the task queues. This is useful to see details of the number of workers polling and monitoring the queue backlog. ### Using the UI ```http request /taskQueue ``` Access this screen via - Home > Task Queues On this screen you can select and view the details of the task queue. The following information is shown: 1. Queue Size - The number of tasks waiting to be executed 2. Workers - The count and list of works and their instance reference who are polling for work for this task ### Using APIs To see the size of the task queue via API: ```shell curl 'http://localhost:8080/api/tasks/queue/sizes?taskType=' \ -H 'accept: */*' ``` To see the worker poll information of the task queue via API: ```shell curl 'http://localhost:8080/api/tasks/queue/polldata?taskType=' \ -H 'accept: */*' ``` > Replace `` with your task name ================================================ FILE: docs/docs/how-tos/Tasks/reusing-tasks.md ================================================ --- sidebar_position: 1 --- # Reusing Tasks A powerful feature of Conductor is that it supports and enables re-usability out of the box. Task workers typically perform a unit of work and is usually a part of a larger workflow. Such workers are often re-usable in multiple workflows. Once a task is defined, you can use it across as any workflow. When re-using tasks, it's important to think of situations that a multi-tenant system faces. All the work assigned to this worker by default goes to the same task scheduling queue. This could result in your worker not being polled quickly if there is a noisy neighbour in the ecosystem. One way you can tackle this situation is by re-using the worker code, but having different task names registered for different use cases. And for each task name, you can run an appropriate number of workers based on expected load. ================================================ FILE: docs/docs/how-tos/Tasks/task-configurations.md ================================================ --- sidebar_position: 1 --- # Task Configurations Refer to [Task Definitions](/configuration/taskdef.html) for details on how to configure task definitions ### Example Here is a task template payload with commonly used fields: ```json { "createdBy": "user", "name": "sample_task_name_1", "description": "This is a sample task for demo", "responseTimeoutSeconds": 10, "timeoutSeconds": 30, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 5, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 } ``` ### Best Practices 1. Refer to [Task Timeouts](/how-tos/Tasks/task-timeouts.html) for additional information on how the various timeout settings work 2. Refer to [Monitoring Task Queues](/how-tos/Tasks/monitoring-task-queues.html) on how to monitor task queues ================================================ FILE: docs/docs/how-tos/Tasks/task-inputs.md ================================================ --- sidebar_position: 1 --- # Task Inputs Task inputs can be provided in multiple ways. This is configured in the workflow definition when a task is participating in the workflow. ### Inputs referred from Workflow inputs When we start a workflow, we can provide inputs to the workflow in a json format. For example: ```json { "worfklowInputNumberExample": 1, "worfklowInputTextExample": "SAMPLE", "worfklowInputJsonExample": { "nestedKey": "nestedValue" } } ``` These values can be referred as inputs into your task using the following expression: ```json { "taskInput1Key": "${workflow.input.worfklowInputNumberExample}", "taskInput2Key": "${workflow.input.worfklowInputJsonExample.nestedKey}" } ``` In this example, the tasks will receive the following inputs after they are evaluated: ```json { "taskInput1Key": 1, "taskInput2Key": "nestedValue" } ``` ### Inputs referred from other Task outputs Similar to how we can refer to workflow inputs, we can also refer to an output field that was generated by a task that executed before. Let's assume a task with the task reference name `previousTaskReference` executed and produced the following output: ```json { "taskOutputKey1": "outputValue", "taskOutputKey2": { "nestedKey1": "outputValue-1" } } ``` We can refer to these as the new task's input by using the following expression: ```json { "taskInput1Key": "${previousTaskReference.output.taskOutputKey1}", "taskInput2Key": "${previousTaskReference.output.taskOutputKey2.nestedKey1}" } ``` The expression format is based on [Json Path](https://goessner.net/articles/JsonPath/) and you can construct complex input params based on the syntax. ### Hard coded inputs Task inputs can also be hard coded in the workflow definitions. This is useful when you have a re-usable task which has configurable options that can be applied in different workflow contexts. ```json { "taskInput1": "OPTION_A", "taskInput2": 100 } ``` ================================================ FILE: docs/docs/how-tos/Tasks/task-timeouts.md ================================================ --- sidebar_position: 1 --- # Task Timeouts Tasks can be configured to handle various scenarios of timeouts. Here are some scenarios and the relevance configuration fields. | Scenario | Configuration | |-----------------------------------------------------------------------------------------------------------|--------------------------| | A task worker picked up the task, but fails to respond back with an update | `responseTimeoutSeconds` | | A task worker picked up the task and updates progress, but fails to complete within an expected timeframe | `timeoutSeconds` | | A task is stuck in a retry loop with repeated failures beyond an expected timeframe | `timeoutSeconds` | | Task doesn't get picked by any workers for a specific amount of time | `pollTimeoutSeconds` | | Task isn't completed within a specified amount of time despite being picked up by task workers | `timeoutSeconds` | > `timeoutSeconds` should always be greater than `responseTimeoutSeconds` ### Timeout Seconds ```json "timeoutSeconds" : 30 ``` When configured with a value > `0`, the system will wait for this task to complete successfully up until this number of seconds from when the task is first polled. We can use this to fail a workflow when a task breaches the overall SLA for completion. ### Response Timeout Seconds ```json "responseTimeoutSeconds" : 10 ``` When configured with a value > `0`, the system will wait for this number of seconds from when the task is polled before the worker updates back with a status. The worker can keep the task in `IN_PROGRESS` state if it requires more time to complete. ### Poll Timeout Seconds ```json "pollTimeoutSeconds" : 10 ``` When configured with a value > `0`, the system will wait for this number of seconds for the task to be picked up by a task worker. Useful when you want to detect a backlogged task queue with not enough workers. ================================================ FILE: docs/docs/how-tos/Tasks/updating-tasks.md ================================================ --- sidebar_position: 1 --- # Updating Task Definitions Updates to the task definitions can be made using the following API ```http PUT /api/metadata/taskdefs ``` This API takes a single task definition and updates itself. ### Example using curl ```shell curl 'http://localhost:8080/api/metadata/taskdefs' \ -X 'PUT' \ -H 'accept: */*' \ -H 'content-type: application/json' \ --data-raw '{"createdBy":"user","name":"sample_task_name_1","description":"This is a sample task for demo","responseTimeoutSeconds":10,"timeoutSeconds":30,"inputKeys":[],"outputKeys":[],"timeoutPolicy":"TIME_OUT_WF","retryCount":3,"retryLogic":"FIXED","retryDelaySeconds":5,"inputTemplate":{},"rateLimitPerFrequency":0,"rateLimitFrequencyInSeconds":1}' ``` ### Example using node fetch ```javascript fetch("http://localhost:8080/api/metadata/taskdefs", { "headers": { "accept": "*/*", "content-type": "application/json", }, "body": "{\"createdBy\":\"user\",\"name\":\"sample_task_name_1\",\"description\":\"This is a sample task for demo\",\"responseTimeoutSeconds\":10,\"timeoutSeconds\":30,\"inputKeys\":[],\"outputKeys\":[],\"timeoutPolicy\":\"TIME_OUT_WF\",\"retryCount\":3,\"retryLogic\":\"FIXED\",\"retryDelaySeconds\":5,\"inputTemplate\":{},\"rateLimitPerFrequency\":0,\"rateLimitFrequencyInSeconds\":1}", "method": "PUT" }); ``` ## Best Practices 1. You can also use the Conductor Swagger UI to update the tasks 2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](/how-tos/Tasks/task-configurations.html) for all the options and details' ================================================ FILE: docs/docs/how-tos/Test/testing-workflows.md ================================================ # Conductor Workflow Testing Guide ## Unit and Regression testing workflows ### Unit Tests Conductor workflows can be unit tested using `POST /workflow/test` endpoint. The approach is similar to how you unit test using Mock objects in Java or similar languages. #### Why Unit Test Workflows? Unit tests allows you to test for the correctness of the workflow definition ensuring: 1. Given a specific input workflow reaches the terminal state in COMPLETED or FAILED state 2. Given a specific input, the workflow executes specific set of tasks. This is useful for testing branching and dynamic forks 3. Task inputs are wired correctly - e.g. if a task receives its input from the output of another task, this can be verified using the unit test. ### Unit Testing Workflows Conductor SDKs provides the following method that allows testing a workflow definition against mock inputs: ```java public abstract Workflow testWorkflow(WorkflowTestRequest testRequest); ``` The actual workflow is executed on a real Conductor server ensuring you are testing the behavior that will match the ACTUAL executon of the server. ### Setting up Conductor server for testing Tests can be run against a remote server (useful when running integration tests) or local containerized instance. Recommended approach is to use `testcontainers`. ### Examples #### Unit Test * [LoanWorkflowTest.java](/client/src/test/java/com/netflix/conductor/client/testing/LoanWorkflowTest.java) * Testing workflows that contain sub-workflows : [SubWorkflowTest.java](/client/src/test/java/com/netflix/conductor/client/testing/SubWorkflowTest.java) #### Regression Test Workflows can be regression tested with golden inputs and outputs. This approach is useful when modifying workflows that are running in production to ensure the behavior remains correct. See [RegressionTest.java](/client/src/test/java/com/netflix/conductor/client/testing/RegressionTest.java) for an example, which uses previously captured workflow execution as golden input/output to verify the workflow execution. ================================================ FILE: docs/docs/how-tos/Workers/build-a-golang-task-worker.md ================================================ --- sidebar_position: 1 --- # Build a Go Task Worker ## Install ```shell go get github.com/netflix/conductor/client/go ``` This will create a Go project under $GOPATH/src and download any dependencies. ## Implementing a Task a Worker `task`package provides the types used to implement the worker. Here is a reference worker implementation: ```go package task import ( "fmt" ) // Implementation for "task_1" func Task_1_Execution_Function(t *task.Task) (taskResult *task.TaskResult, err error) { log.Println("Executing Task_1_Execution_Function for", t.TaskType) //Do some logic taskResult = task.NewTaskResult(t) output := map[string]interface{}{"task":"task_1", "key2":"value2", "key3":3, "key4":false} taskResult.OutputData = output taskResult.Status = "COMPLETED" err = nil return taskResult, err } ``` ## Worker Polling Here is an example that shows how to start polling for tasks after defining the tasks. ```go package main import ( "github.com/netflix/conductor/client/go" "github.com/netflix/conductor/client/go/task/sample" ) func main() { c := conductor.NewConductorWorker("http://localhost:8080", 1, 10000) c.Start("task_1", "", sample.Task_1_Execution_Function, false) c.Start("task_2", "mydomain", sample.Task_2_Execution_Function, true) } ``` ### `NewConductorWoker` parameters 1. baseUrl: Server address. 2. threadCount: No. of threads. Number of threads should be at-least same as the number of workers 3. pollingInterval: Time in millisecond between subsequent polls ================================================ FILE: docs/docs/how-tos/Workers/build-a-java-task-worker.md ================================================ --- sidebar_position: 1 --- # Build a Java Task Worker This guide provides introduction to building Task Workers in Java. ## Dependencies Conductor provides Java client libraries, which we will use to build a simple task worker. ### Maven Dependency ```xml com.netflix.conductor conductor-client 3.13.2 com.netflix.conductor conductor-common 3.13.2 ``` ### Gradle ```groovy implementation group: 'com.netflix.conductor', name: 'conductor-client', version: '3.13.2' implementation group: 'com.netflix.conductor', name: 'conductor-common', version: '3.13.2' ``` ## Implementing a Task Worker To create a worker, implement the `Worker` interface. ```java public class SampleWorker implements Worker { private final String taskDefName; public SampleWorker(String taskDefName) { this.taskDefName = taskDefName; } @Override public String getTaskDefName() { return taskDefName; } @Override public TaskResult execute(Task task) { TaskResult result = new TaskResult(task); result.setStatus(Status.COMPLETED); //Register the output of the task result.getOutputData().put("outputKey1", "value"); result.getOutputData().put("oddEven", 1); result.getOutputData().put("mod", 4); return result; } } ``` ### Implementing worker's logic Worker's core implementation logic goes in the `execute` method. Upon completion, set the `TaskResult` with status as one of the following: 1. **COMPLETED**: If the task has completed successfully. 2. **FAILED**: If there are failures - business or system failures. Based on the task's configuration, when a task fails, it may be retried. The `getTaskDefName()` method returns the name of the task for which this worker provides the execution logic. See [SampleWorker.java](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/SampleWorker.java) for the complete example. ## Configuring polling using TaskRunnerConfigurer The `TaskRunnerConfigurer` can be used to register the worker(s) and initialize the polling loop. It manages the task workers thread pool and server communication (poll and task update). Use the [Builder](https://github.com/Netflix/conductor/blob/main/client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java#L64) to create an instance of the `TaskRunnerConfigurer`. The builder accepts the following parameters: ```java TaskClient taskClient = new TaskClient(); taskClient.setRootURI("http://localhost:8080/api/"); //Point this to the server API int threadCount = 2; //number of threads used to execute workers. To avoid starvation, should be same or more than number of workers Worker worker1 = new SampleWorker("task_1"); Worker worker2 = new SampleWorker("task_5"); // Create TaskRunnerConfigurer TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(taskClient, Arrays.asList(worker1, worker2)) .withThreadCount(threadCount) .build(); // Start the polling and execution of tasks configurer.init(); ``` See [Sample](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/Main.java) for full example. ### Configuration Details Initialize the `Builder` with the following: | Parameter | Description | | --- | --- | | TaskClient | TaskClient used to communicate with the Conductor server | | Workers | Workers that will be used for polling work and task execution. | | Parameter | Description | Default | | --- | --- | --- | | withEurekaClient | EurekaClient is used to identify if the server is in discovery or not. When the server goes out of discovery, the polling is stopped unless `pollOutOfDiscovery` is set to true. If passed null, discovery check is not done. | provided by platform | | withThreadCount | Number of threads assigned to the workers. Should be at-least the size of taskWorkers to avoid starvation in a busy system. | Number of registered workers | | withSleepWhenRetry | Time in milliseconds, for which the thread should sleep when task update call fails, before retrying the operation. | 500 | | withUpdateRetryCount | Number of attempts to be made when updating task status when update status call fails. | 3 | | withWorkerNamePrefix | String prefix that will be used for all the workers. | workflow-worker- | Once an instance is created, call `init()` method to initialize the `TaskPollExecutor` and begin the polling and execution of tasks. !!! tip "Note" To ensure that the `TaskRunnerConfigurer` stops polling for tasks when the instance becomes unhealthy, call the provided `shutdown()` hook in a `PreDestroy` block. #### Properties The worker behavior can be further controlled by using these properties: | Property | Type | Description | Default | | --- | --- | --- | --- | | paused | boolean | If set to true, the worker stops polling.| false | | pollInterval | int | Interval in milliseconds at which the server should be polled for tasks. | 1000 | | pollOutOfDiscovery | boolean | If set to true, the instance will poll for tasks regardless of the discovery status. This is useful while running on a dev machine. | false | Further, these properties can be set either by a `Worker` implementation or by setting the following system properties in the JVM: | Name | Description | | --- | --- | | `conductor.worker.` | Applies to ALL the workers in the JVM. | | `conductor.worker..` | Applies to the specified worker. Overrides the global property. | ================================================ FILE: docs/docs/how-tos/Workers/build-a-python-task-worker.md ================================================ --- sidebar_position: 1 --- # Build a Python Task Worker ## Install the python client ```shell virtualenv conductorclient source conductorclient/bin/activate cd ../conductor/client/python python setup.py install ``` ## Implement a Task Worker [ConductorWorker](https://github.com/Netflix/conductor/blob/main/polyglot-clients/python/conductor/ConductorWorker.py#L36) class is used to implement task workers. The following script shows how to bring up two task workers named `book_flight` and `book_car`: ```python from __future__ import print_function from conductor.ConductorWorker import ConductorWorker def book_flight_task(task): return {'status': 'COMPLETED', 'output': {'booking_ref': 2341111, 'airline': 'delta'}, 'logs': ['trying delta', 'skipping aa']} def book_car_task(task): return {'status': 'COMPLETED', 'output': {'booking_ref': "84545fdfd", 'agency': 'hertz'}, 'logs': ['trying hertz']} def main(): print('Starting Travel Booking workflows') cc = ConductorWorker('http://localhost:8080/api', 1, 0.1) cc.start('book_flight', book_flight_task, False) cc.start('book_car', book_car_task, True) if __name__ == '__main__': main() ``` ### `ConductorWorker` parameters ```python server_url: str The url to the server hosting the conductor api. Ex: 'http://localhost:8080/api' thread_count: int The number of threads that will be polling for and executing tasks in case of using the start method. polling_interval: float The number of seconds that each worker thread will wait between polls to the conductor server. worker_id: str, optional The worker_id of the worker that is going to execute the task. For further details, refer to the documentation By default, it is set to hostname of the machine ``` ### `start` method parameters ```pythhon taskType: str The name of the task that the worker is looking to execute exec_function: function The function that the worker will execute. The function must return a dict with the `status`, `output` and `logs` keys present. If this is not present, an Exception will be raised wait: bool Whether the worker will block execution of further code. Since the workers are being run in daemon threads, when the program completes execution, all the threads are destroyed. Setting wait to True prevents the program from ending. If multiple workers are being called from the same program, all but the last start call but have wait set to False. The last start call must always set wait to True. If a single worker is being called, set wait to True. domain: str, optional The domain of the task under which the worker will run. For further details refer to the conductor server documentation By default, it is set to None ``` See [https://github.com/Netflix/conductor/tree/main/polyglot-clients/python](https://github.com/Netflix/conductor/tree/main/polyglot-clients/python) for the source code. ================================================ FILE: docs/docs/how-tos/Workflows/debugging-workflows.md ================================================ --- sidebar_position: 1 --- # Debugging Workflows Conductor UI is a tool that we can leverage for debugging issues. Refer to the following articles to search and view your workflow execution. 1. [Searching Workflows](/how-tos/Workflows/searching-workflows.html) 2. [View Workflow Executions](/how-tos/Workflows/view-workflow-executions.html) ## Debugging Executions Open the **Tasks > Diagram** tab to see the diagram of the overall workflow execution If there is a failure, you will them on the view marked as red. In most cases it should be clear what went wrong from the view itself. To see details of the failure, you can click on the failed task. The following fields are useful in debugging | Field Name | Description | |-------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| | Task Detail > Summary > Reason for Incompletion | If an exception was thrown by the worker, it will be captured and displayed here | | Task Detail > Summary > Worker | The worker instance id where this failure last occurred. Useful to dig for detailed logs if not already captured by Conductor | | Task Detail > Input | Verify if the task inputs were computed and provided correctly to the task | | Task Detail > Output | If output of a previous task is used as an input to your next task, refer here for what was produced | | Task Detail > Logs | If your task is supplying logs, we can look at that here | | Task Detail > Retried Task - Select an instance | If your task was retried, we can see all the attempts and correponding details here | Note: We can also access the task list from **Tasks > Task List** tab. Here is a screen grab of the fields referred above. ![Debugging Wowkflow Execution](/img/tutorial/workflow_debugging.png) ## Recovering From Failures Once we have resolved the underlying issue of workflow execution failure, we might want to replay or retry failed workflows. The UI has functions that would allow us to do this: The **Actions** button provides the following options: |Action Name|Description| |---|---| | Restart with Current Definitions | Restart this workflow from the beginning using the same version of the workflow definition that originally ran this workflow execution. This is useful if the workflow definition has changed and we want to retain this instance to the original version| | Restart with Latest Definitions | Restart this workflow from the beginning using the latest definition of the workflow. If we made changes to definition, we can use this option to re-run this flow with the latest version| | Retry - From failed task | Retry this workflow from the failed task|
    > **Note:** Conductor configurations allow your tasks to be retried automatically for transient failures. > Refer to the task configuration options on how to leverage this. ================================================ FILE: docs/docs/how-tos/Workflows/handling-errors.md ================================================ # Handling Errors When a workflow fails, there are 2 ways to handle the exception. ## Set ```failureWorkflow``` in Workflow Definition In your main workflow definition, you can configure a workflow to run upon failure, by adding the following parameter to the workflow: ```json "failureWorkflow": "", "method": "POST", "body": { "text": "workflow: ${workflow.input.workflowId} failed. ${workflow.input.reason}" }, "connectionTimeOut": 5000, "readTimeOut": 5000 } }, "type": "HTTP", "retryCount": 3 } ], "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "conductor@example.com", "timeoutPolicy": "ALERT_ONLY", } ``` ## Set ```workflowStatusListenerEnabled``` When this is enabled, notifications are now possible, and by building a custom implementation of the Workflow Status Listener, a notification can be sent to an external service. [More details.](https://github.com/Netflix/conductor/issues/1017#issuecomment-468869173) ================================================ FILE: docs/docs/how-tos/Workflows/searching-workflows.md ================================================ --- sidebar_position: 1 --- # Searching Workflows In this article we will learn how to search through workflow executions via the UI. ### Prerequisites 1. Conductor app and UI installed and running in an environment. If required we can look at the following options to get an environment up and running. 1. [Build and Run Conductor Locally](/gettingstarted/local.html) 2. [Running via Docker Compose](/gettingstarted/docker.html) ## UI Workflows View Open the home page of the UI installation. It will take you to the `Workflow Executions` view. This is where we can look at available workflow executions. ### Basic Search The following fields are available for searching for workflows. | Search Field Name | Description | |-------------------|---------------------------------------------------------------------------------------------------------| | Workflow Name | Use this field to filter workflows by the configured name | | Workflow ID | Use this field to filter to a specific workflow by its id | | Status | Use this field to filter by status - available options are presented as a multi-select option | | Start Time - From | Use this field to filter workflows that started on or after the time specified | | Start Time - To | Use this field to filter workflows that started on or before the time specified | | Lookback (days) | Use this field to filter workflows that ran in the last given number of days | | Free Text Query | If you have indexing enabled, you can query by values that was part of your workflow inputs and outputs | The table listing has options to 1. Select columns for display 2. Sort by column value At the bottom of the table, there are options to 1. Select number of rows per page 2. Navigating through pages ### Find by Tasks In addition to the options listed in **Basic Search** view, we have the following options in the **Find by Tasks** view. | Search Field Name | Description | |--------------------|--------------------------------------------------------------------------------------------------------------| | Include Task ID | Use this field to filter workflows that contains a task with this id | | Include Task Name | Use this field to filter workflows that contains a task with name | | Free Text in Tasks | If you have indexing enabled, you can query by values that was part of your workflow task inputs and outputs | ================================================ FILE: docs/docs/how-tos/Workflows/starting-workflows.md ================================================ # Starting Workflows Workflow executions can be started by using the following API: ```http POST /api/workflow/{name} ``` `{name}` is the placeholder for workflow name. The POST API body is your workflow input parameters which can be empty if there are none. ### Using Client SDKs Conductor offers client SDKs for popular languages which has library methods that can be used to make this API call. Refer to the SDK documentation to configure a client in your selected language to invoke workflow executions. ### Example using curl ```bash curl 'https://localhost:8080/api/workflow/sample_workflow' \ -H 'accept: text/plain' \ -H 'content-type: application/json' \ --data-raw '{"service":"fedex"}' ``` In this example we are specifying one input param called `service` with a value of `fedex` and the name of the workflow is `sample_workflow` ### Example using node fetch ```javascript fetch("https://localhost:8080/api/workflow/sample_workflow", { "headers": { "accept": "text/plain", "content-type": "application/json", }, "body": "{\"service\":\"fedex\"}", "method": "POST", }); ``` ================================================ FILE: docs/docs/how-tos/Workflows/updating-workflows.md ================================================ # Updating Workflows Workflows can be created or updated using the workflow metadata API ```html PUT /api/metadata/workflow ``` ### Example using curl ```shell curl 'http://localhost:8080/api/metadata/workflow' \ -X 'PUT' \ -H 'accept: */*' \ -H 'content-type: application/json' \ --data-raw '[{"name":"sample_workflow","version":1,"tasks":[{"name":"ship_via_fedex","taskReferenceName":"ship_via_fedex","type":"SIMPLE"}],"schemaVersion":2}]' ``` ### Example using node fetch ```javascript fetch("http://localhost:8080/api/metadata/workflow", { "headers": { "accept": "*/*", "content-type": "application/json" }, "body": "[{\"name\":\"sample_workflow\",\"version\":1,\"tasks\":[{\"name\":\"ship_via_fedex\",\"taskReferenceName\":\"ship_via_fedex\",\"type\":\"SIMPLE\"}],\"schemaVersion\":2}]", "method": "PUT" }); ``` ## Best Practices 1. If you are updating the workflow with new tasks, remember to register the task definitions first 2. You can also use the Conductor Swagger UI to update the workflows ================================================ FILE: docs/docs/how-tos/Workflows/versioning-workflows.md ================================================ --- sidebar_position: 1 --- # Versioning Workflows Every workflow has a version number (this number **must** be an integer.) Versioning allows you to run different versions of the same workflow simultaneously. ## Summary > Use Case: A new version of your core workflow will add a capability that is required for *veryImportantCustomer*. However, *otherVeryImportantCustomer* will not be ready to implement this code for another 6 months. ## Version 1 ```json { "name": "Core_workflow", "description": "Very_important_business", "version": 1, "tasks": [ { } ], "outputParameters": { } } ``` ## Version 2 ```json { "name": "Core_workflow", "description": "Very_important_business", "version": 2, "tasks": [ { } ], "outputParameters": { } } ``` ### Version 2 launch Initially, both customers are on version 1 of the workflow. * **veryImportantCustomer* may begin transitioning traffic onto version 2. Any tasks that remain unfinished on version 1 *stay* on version 1. * *otherVeryImportantCustomer* remains on version 1. ### 6 months later * All *veryImportantCustomer* workflows are on version 2. * *otherVeryImportantCustomer* may begin transitioning traffic onto version 2. Any tasks that remain unfinished on version 1 *stay* on version 1. ================================================ FILE: docs/docs/how-tos/Workflows/view-workflow-executions.md ================================================ --- sidebar_position: 1 --- # View Workflow Executions In this article we will learn how to view workflow executions via the UI. ### Prerequisites 1. Conductor app and UI installed and running in an environment. If required we can look at the following options to get an environment up and running. 1. [Build and Run Conductor Locally](/gettingstarted/local.html) 2. [Running via Docker Compose](/gettingstarted/docker.html) ### Viewing a Workflow Execution Refer to [Searching Workflows](/how-tos/Workflows/searching-workflows.html) to filter and find an execution you want to view. Click on the workflow id hyperlink to open the Workflow Execution Details page. The following tabs are available to view the details of the Workflow Execution | Tab Name | Description | |-----------------------|-------------------------------------------------------------------------------------------------------------------| | Tasks | Shows a view with the sub tabs **Diagram**, **Task List** and **Timeline** | | Tasks > Diagram | Visual view of the workflow and its tasks. | | Tasks > Task List | Tabular view of the task executions under this workflow. If there were failures, we will be able to see that here | | Tasks > Timeline | Shows the time each of the tasks took for execution in a timeline view | | Summary | Summary view of the workflow execution | | Workflow Input/Output | Shows the input and output payloads of the workflow. Enable copy mode to copy all or parts of the payload | | JSON | Full JSON payload of the workflow including all tasks, inputs and outputs. Useful for detailed debugging. | ### Viewing a Workflow Task Detail From both the **Tasks > Diagram** and **Tasks > Task List** views, we can click to see a task execution detail. This opens a flyout panel from the side and contains the following tabs. | Tab Name | Description | |------------|------------------------------------------------------------------------------------------------------------------------------------------| | Summary | Summary info of the task execution | | Input | Task input payload - refer to this tab to see computed inputs passed into the task. Enable copy mode to copy all or parts of the payload | | Output | Shows the output payload produced by the executed task. Enable copy mode to copy all or parts of the payload | | Log | Any log messages logged by the task worked will show up here | | JSON | Complete JSON payload for debugging issues | | Definition | Task definition used when executing this task | ### Execution Path An exciting feature of conductor is the ability to see the exact execution path of a workflow. The executed paths are shown in green and is easy to follow like the example below. The alternative paths are greyed out for reference ![Conductor UI - Workflow Run](/img/tutorial/workflow_execution_view.png) Errors will be visible on the UI in ref such as the example below ![Conductor UI - Failed Task](/img/tutorial/workflow_task_fail.png) ================================================ FILE: docs/docs/how-tos/clojure-sdk.md ================================================ # Conductor Clojure Software Development Kit for Netflix Conductor, written on and providing support for Clojure. The code for the Clojure SDk is available on [Github](https://github.com/conductor-sdk/conductor-clojure). Please feel free to file PRs, issues, etc. there. ## Get the SDK https://clojars.org/io.orkes/conductor-clojure ## Quick Guide 1. Create connection options ```clojure (def options { :url "http://localhost:8080/api/" ;; Conductor Server Path :app-key "THIS-IS-SOME-APP-KEY" ;; Optional if using Orkes Conductor :app-secret "THIS-IS-SOME-APP-SECRET" ;; Optional if using Orkes Conductor } ) ``` 1. Creating a task using above options ``` clojure (ns some.namespace (:require [io.orkes.metadata :as metadata]) ;; Will Create a task. returns nil (metadata/register-tasks options [{ :name "cool_clj_task" :description "some description" :ownerEmail "somemail@mail.com" :retryCount 3 :timeoutSeconds 300 :responseTimeoutSeconds 180 }]) ) ``` 2. Creating a Workflow that uses the task ``` clojure (ns some.namespace (:require [io.orkes.metadata :as metadata]) ;; Will Register a workflow that uses the above task returns nil (metadata/register-workflow-def options { :name "cool_clj_workflow" :description "created programmatically from clj" :version 1 :tasks [ { :name "cool_clj_task" :taskReferenceName "cool_clj_task_ref" :inputParameters {} :type "SIMPLE" } ] :inputParameters [] :outputParameters {:message "${clj_prog_task_ref.output.:message}"} :schemaVersion 2 :restartable true :ownerEmail "owner@yahoo.com" :timeoutSeconds 0 })) ``` 3. Create and run a list of workers ``` clojure ;; Creates a worker and starts polling for work. will return an instance of Runner which can then be used to shutdown (def instance (runner-executor-for-workers (list { :name "cool_clj_task" :execute (fn [someData] [:completed {:message "Hi From Clj i was created programmatically"}]) }) options )) ;; Shutsdown the polling for the workers defined above (.shutdown instance) ``` ## Options Options are a map with optional paremeters ``` (def options { :url "http://localhost:8080/api/" ;; Api url (Optional will default to "http://localhost:8080") :app-key "THIS-IS-SOME-APP-KEY" ;; Application Key (This is only relevant if you are using Orkes Conductor) :app-secret "THIS-IS-SOME-APP-SECRET" ;; Application Secret (This is only relevant if you are using Orkes Conductor) } ) ``` ## Metadata namespace Holds the functions to register workflows and tasks. `(:require [conductor.metadata :as metadata])` ### Registering tasks Takes the option map and a list/vector of tasks to register. on success it will return nil ```clojure (metadata/register-tasks options [{ :name "cool_clj_task_b" :description "some description" :ownerEmail "mail@gmail.com" :retryCount 3 :timeoutSeconds 300 :responseTimeoutSeconds 180 }, { :name "cool_clj_task_z" :description "some description" :ownerEmail "mail@gmail.com" :retryCount 3 :timeoutSeconds 300 :responseTimeoutSeconds 180 } { :name "cool_clj_task_x" :description "some description" :ownerEmail "mail@gmail.com" :retryCount 3 :timeoutSeconds 300 :responseTimeoutSeconds 180 } ]) ``` ### Registering a workspace ```clojure (metadata/register-workflow-def options { :name "cool_clj_workflow_2" :description "created programmatically from clj" :version 1 :tasks [ { :name "cool_clj_task_b" :taskReferenceName "cool_clj_task_ref" :inputParameters {} :type "SIMPLE" }, { :name "someting", :taskReferenceName "other" :inputParameters {} :type "FORK_JOIN" :forkTasks [[ { :name "cool_clj_task_z" :taskReferenceName "cool_clj_task_z_ref" :inputParameters {} :type "SIMPLE" } ] [ { :name "cool_clj_task_x" :taskReferenceName "cool_clj_task_x_ref" :inputParameters {} :type "SIMPLE" } ] ] } { :name "join" :type "JOIN" :taskReferenceName "join_ref" :joinOn [ "cool_clj_task_z", "cool_clj_task_x"] } ] :inputParameters [] :outputParameters {"message" "${clj_prog_task_ref.output.:message}"} :schemaVersion 2 :restartable true :ownerEmail "mail@yahoo.com" :timeoutSeconds 0 :timeoutPolicy "ALERT_ONLY" }) ``` ## Client namespace The client namespace holds the function to start a workflow and running a worker `[io.orkes.client :as conductor]` ``` clojure ;; Creates a worker and starts polling for work. will return an instance of Runner which can then be used to shutdown (def instance (runner-executor-for-workers (list { :name "cool_clj_task" :execute (fn [someData] [:completed {:message "Hi From Clj i was created programmatically"}]) }) options )) ;; Shutsdown the polling for the workers defined above (.shutdown instance) ``` The (runner-executor-for-workers) function will take a list of worker implementations map, and options and start pooling for work it will return a TaskRunnerConfigurer instance, which you can shutdown by calling the .shutdown() java method ## Mapper-Utils namespace The `[io.orkes.mapper-utils :as mapper-utils]` namespace holds the functions to map to java object which are mostly not necesary. ### The mapper-utils/java-map->clj-map protocol Will map a java map to a clojure map which may come in handy for workers implementation. for example consider a worker that sums two input parameters. For a workflow defined like this : ``` clojure (metadata/register-workflow-def options {:name "simple_wf" :description "created programmatically from clj" :version 1 :tasks [{:name "simplest_task" :taskReferenceName "repl_task_ref" :inputParameters {"firstNumber" "${workflow.input.firstNumber}" "secondNumber" "${workflow.input.secondNumber}"} :type "SIMPLE"}] :inputParameters ["firstNumber" "secondNumber"] :outputParameters {"result" "${repl_task_ref.output.:result}"} :schema-version 2 :restartable true :ownerEmail "mail@yahoo.com" :timeoutSeconds 0 :timeoutPolicy "ALERT_ONLY"}) ``` To be able to use the input params you would need to use the string names like this: ``` clojure (def instance (conductor/runner-executor-for-workers (list {:name "simplest_task" :execute (fn [someData] [:completed {"result" (+ (get someData "firstNumber") (get someData "secondNumber"))}])}) options)) ``` A more clojure friendly way would be to convert to clojure our map : ``` clojure (def instance (conductor/runner-executor-for-workers (list {:name "simplest_task" :execute (fn [someData] (let [convertedToClj (-> someData mapper-utils/java-map->clj-map)] [:completed {"result" (+ (:firstNumber convertedToClj) (:secondNumber convertedToClj))}] ))}) options)) ``` ================================================ FILE: docs/docs/how-tos/csharp-sdk.md ================================================ # Netflix Conductor Client C# SDK `conductor-csharp` repository provides the client SDKs to build Task Workers and Clients in C# The code for the C# SDk is available on [Github](https://github.com/conductor-sdk/conductor-csharp). Please feel free to file PRs, issues, etc. there. ## Quick Start 1. [Get Secrets](#Get-Secrets) 2. [Write workers](#Write-workers) 3. [Run workers](#Run-workers) 4. [Worker Configurations](#Worker-Configurations) ### Dependencies `conductor-csharp` packages are published to nugget package manager. You can find the latest releases [here](https://www.nuget.org/packages/conductor-csharp/). ### Write workers ``` internal class MyWorkflowTask : IWorkflowTask { public MyWorkflowTask(){} public string TaskType => "test_ctask"; public int? Priority => null; public async Task Execute(Conductor.Client.Models.Task task, CancellationToken token) { Dictionary newOutput = new Dictionary(); newOutput.Add("output", "1"); return task.Completed(task.OutputData.MergeValues(newOutput)); } } internal class MyWorkflowTask2 : IWorkflowTask { public MyWorkflowTask2(){} public string TaskType => "test_ctask2"; public int? Priority => null; public async Task Execute(Conductor.Client.Models.Task task, CancellationToken token) { Dictionary newOutput = new Dictionary(); //Reuse the existing code written in C# newOutput.Add("output", "success"); return task.Completed(task.OutputData.MergeValues(newOutput)); } } ``` ### Run workers ``` using System; using System.Threading; using System.Threading.Tasks; using System.Collections.Generic; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Conductor.Client.Models; using Conductor.Client.Extensions; using Conductor.Client.Interfaces; using Task = System.Threading.Tasks.Task; using Conductor.Client; using System.Collections.Concurrent; namespace TestOrkesSDK { class Program { static void Main(string[] args) { new HostBuilder() .ConfigureServices((ctx, services) => { // First argument is optional headers which client wasnt to pass. Configuration configuration = new Configuration(new ConcurrentDictionary(), "KEY", "SECRET"); services.AddConductorWorker(configuration); services.AddConductorWorkflowTask(); services.AddHostedService(); }) .ConfigureLogging(logging => { logging.SetMinimumLevel(LogLevel.Debug); logging.AddConsole(); }) .RunConsoleAsync(); Console.ReadLine(); } } internal class MyWorkflowTask : IWorkflowTask { public MyWorkflowTask() { } public string TaskType => "my_ctask"; public int? Priority => null; public async Task Execute(Conductor.Client.Models.Task task, CancellationToken token) { Dictionary newOutput = new Dictionary(); newOutput.Add("output", 1); return task.Completed(task.OutputData.MergeValues(newOutput)); } } } ``` ####Note: Replace KEY and SECRET by obtaining a new key and secret from [Orkes Playground](https://play.orkes.io/) See [Generating Access Keys for Programmatic Access](https://orkes.io/content/docs/getting-started/concepts/access-control#access-keys) for details./ ``` internal class WorkflowsWorkerService : BackgroundService { private readonly IWorkflowTaskCoordinator workflowTaskCoordinator; private readonly IEnumerable workflowTasks; public WorkflowsWorkerService( IWorkflowTaskCoordinator workflowTaskCoordinator, IEnumerable workflowTasks ) { this.workflowTaskCoordinator = workflowTaskCoordinator; this.workflowTasks = workflowTasks; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { foreach (var worker in workflowTasks) { workflowTaskCoordinator.RegisterWorker(worker); } // start all the workers so that it can poll for the tasks await workflowTaskCoordinator.Start(); } } ``` ### Worker Configurations Worker configuration is handled via Configuration object passed when initializing TaskHandler. ``` Configuration configuration = new Configuration(new ConcurrentDictionary(), "KEY", "SECRET", "https://play.orkes.io/"); ``` ### Registering and starting the workflow using SDK. Below is the code snippet that shows how to register a simple workflow and start execution: ``` IDictionary optionalHeaders = new ConcurrentDictionary(); Configuration configuration = new Configuration(optionalHeaders, "keyId", "keySecret"); //Create task definition MetadataResourceApi metadataResourceApi = new MetadataResourceApi(configuration); TaskDef taskDef = new TaskDef(name: "test_task"); taskDef.OwnerEmail = "test@test.com"; metadataResourceApi.RegisterTaskDef(new List() { taskDef}); //Create workflow definition WorkflowDef workflowDef = new WorkflowDef(); workflowDef.Name = "test_workflow"; workflowDef.OwnerEmail = "test@test.com"; workflowDef.SchemaVersion = 2; WorkflowTask workflowTask = new WorkflowTask(); workflowTask.Type = "HTTP"; workflowTask.Name = "test_"; //Same as registered task definition. IDictionary requestParams = new Dictionary(); requestParams.Add("uri", "https://www.google.com"); //adding a key/value using the Add() method requestParams.Add("method", "GET"); Dictionary request = new Dictionary(); request.Add("http_request", requestParams); workflowTask.InputParameters = request; workflowDef.Tasks = new List() { workflowTask }; //Run a workflow WorkflowResourceApi workflowResourceApi = new WorkflowResourceApi(configuration); Dictionary input = new Dictionary(); //Fill the input map which workflow consumes. workflowResourceApi.StartWorkflow("test_workflow", input, 1); Console.ReadLine(); ``` Please see [Conductor.Api](https://github.com/conductor-sdk/conductor-csharp/tree/main/Api) for the APIs. ================================================ FILE: docs/docs/how-tos/go-sdk.md ================================================ # Netflix Conductor Go SDK The code for the Golang SDk is available on [Github](https://github.com/conductor-sdk/conductor-go). Please feel free to file PRs, issues, etc. there. ## Quick Start 1. [Setup conductor-go package](#Setup-conductor-go-package) 2. [Create and run Task Workers](https://github.com/conductor-sdk/conductor-go/blob/main/workers_sdk.md) 3. [Create workflows using Code](https://github.com/conductor-sdk/conductor-go/blob/main/workflow_sdk.md) 4. [API Documentation](https://github.com/conductor-sdk/conductor-go/blob/main/docs/) ### Setup conductor go package Create a folder to build your package: ```shell mkdir quickstart/ cd quickstart/ go mod init quickstart ``` Get Conductor Go SDK ```shell go get github.com/conductor-sdk/conductor-go ``` ## Configuration ### Authentication settings (optional) Use if your conductor server requires authentication * keyId: Key * keySecret: Secret for the Key ```go authenticationSettings := settings.NewAuthenticationSettings( "keyId", "keySecret", ) ``` ### Access Control Setup See [Access Control](https://orkes.io/content/docs/getting-started/concepts/access-control) for more details on role based access control with Conductor and generating API keys for your environment. ### Configure API Client ```go apiClient := client.NewAPIClient( settings.NewAuthenticationSettings( KEY, SECRET, ), settings.NewHttpSettings( "https://play.orkes.io", ), ) ``` ### Setup Logging SDK uses [logrus](https://github.com/sirupsen/logrus) for the logging. ```go func init() { log.SetFormatter(&log.TextFormatter{}) log.SetOutput(os.Stdout) log.SetLevel(log.DebugLevel) } ``` ### Next: [Create and run Task Workers](https://github.com/conductor-sdk/conductor-go/blob/main/workers_sdk.md) ================================================ FILE: docs/docs/how-tos/java-sdk.md ================================================ # Conductor Java SDK Conductor provides the following java clients to interact with the various APIs | Client | Usage | |-----------------|---------------------------------------------------------------------------| | Metadata Client | Register / Update workflow and task definitions | | Workflow Client | Start a new workflow / Get execution status of a workflow | | Task Client | Poll for task / Update task result after execution / Get status of a task | ## Java #### Worker Conductor provides an automated framework to poll for tasks, manage the execution thread and update the status of the execution back to the server. Implement the [Worker](https://github.com/Netflix/conductor/blob/main/client/src/main/java/com/netflix/conductor/client/worker/Worker.java) interface to execute the task. #### TaskRunnerConfigurer The TaskRunnerConfigurer can be used to register the worker(s) and initialize the polling loop. Manages the task workers thread pool and server communication (poll and task update). Use the [Builder](https://github.com/Netflix/conductor/blob/master/client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java#L62) to create an instance of the TaskRunnerConfigurer. The builder accepts the following parameters: Initialize the Builder with the following: TaskClient | TaskClient used to communicate to the Conductor server | | Workers | Workers that will be used for polling work and task execution. | | Parameter | Description | Default | |--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------| | withEurekaClient | EurekaClient is used to identify if the server is in discovery or not. When the server goes out of discovery, the polling is stopped unless `pollOutOfDiscovery` is set to true. If passed null, discovery check is not done. | provided by platform | | withThreadCount | Number of threads assigned to the workers. Should be at-least the size of taskWorkers to avoid starvation in a busy system. | Number of registered workers | | withSleepWhenRetry | Time in milliseconds, for which the thread should sleep when task update call fails, before retrying the operation. | 500 | | withUpdateRetryCount | Number of attempts to be made when updating task status when update status call fails. | 3 | | withWorkerNamePrefix | String prefix that will be used for all the workers. | workflow-worker- | | withShutdownGracePeriodSeconds | Waiting seconds before forcing shutdown of your worker | 10 | Once an instance is created, call `init()` method to initialize the TaskPollExecutor and begin the polling and execution of tasks. !!! tip "Note" To ensure that the TaskRunnerConfigurer stops polling for tasks when the instance becomes unhealthy, call the provided `shutdown()` hook in a `PreDestroy` block. **Properties** The worker behavior can be further controlled by using these properties: | Property | Type | Description | Default | |--------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------|---------| | paused | boolean | If set to true, the worker stops polling. | false | | pollInterval | int | Interval in milliseconds at which the server should be polled for tasks. | 1000 | | pollOutOfDiscovery | boolean | If set to true, the instance will poll for tasks regardless of the discovery
    status. This is useful while running on a dev machine. | false | Further, these properties can be set either by Worker implementation or by setting the following system properties in the JVM: | Name | Description | |---------------------------------------------|------------------------------------------------------------------| | `conductor.worker.` | Applies to ALL the workers in the JVM. | | `conductor.worker..` | Applies to the specified worker. Overrides the global property. | **Examples** * [Sample Worker Implementation](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/SampleWorker.java) * [Example](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/Main.java) ================================================ FILE: docs/docs/how-tos/python-sdk.md ================================================ --- sidebar_position: 1 --- # Python SDK Software Development Kit for Netflix Conductor, written on and providing support for Python. The code for the Python SDk is available on [Github](https://github.com/conductor-sdk/conductor-python). Please feel free to file PRs, issues, etc. there. ## Quick Guide 1. Create a virtual environment $ virtualenv conductor $ source conductor/bin/activate $ python3 -m pip list Package Version ---------- ------- pip 22.0.3 setuptools 60.6.0 wheel 0.37.1 2. Install latest version of `conductor-python` from pypi $ python3 -m pip install conductor-python Collecting conductor-python Collecting certifi>=14.05.14 Collecting urllib3>=1.15.1 Requirement already satisfied: setuptools>=21.0.0 in ./conductor/lib/python3.8/site-packages (from conductor-python) (60.6.0) Collecting six>=1.10 Installing collected packages: certifi, urllib3, six, conductor-python Successfully installed certifi-2021.10.8 conductor-python-1.0.7 six-1.16.0 urllib3-1.26.8 3. Create a worker capable of executing a `Task`. Example: from conductor.client.worker.worker_interface import WorkerInterface class SimplePythonWorker(WorkerInterface): def execute(self, task): task_result = self.get_task_result_from_task(task) task_result.add_output_data('key', 'value') task_result.status = 'COMPLETED' return task_result * The `add_output_data` is the most relevant part, since you can store information in a dictionary, which will be sent within `TaskResult` as your execution response to Conductor 4. Create a main method to start polling tasks to execute with your worker. Example: from conductor.client.automator.task_handler import TaskHandler from conductor.client.configuration.configuration import Configuration from conductor.client.worker.sample.faulty_execution_worker import FaultyExecutionWorker from conductor.client.worker.sample.simple_python_worker import SimplePythonWorker def main(): configuration = Configuration(debug=True) task_definition_name = 'python_example_task' workers = [ SimplePythonWorker(task_definition_name), FaultyExecutionWorker(task_definition_name) ] with TaskHandler(workers, configuration) as task_handler: task_handler.start() if __name__ == '__main__': main() * This example contains two workers, each with a different execution method, capable of running the same `task_definition_name` 5. Now that you have implemented the example, you can start the Conductor server locally: 1. Clone [Netflix Conductor repository](https://github.com/Netflix/conductor): $ git clone https://github.com/Netflix/conductor.git $ cd conductor/ 2. Start the Conductor server: /conductor$ ./gradlew bootRun 3. Start Conductor UI: /conductor$ cd ui/ /conductor/ui$ yarn install /conductor/ui$ yarn run start You should be able to access: * Conductor API: * http://localhost:8080/swagger-ui/index.html * Conductor UI: * http://localhost:5000 6. Create a `Task` within `Conductor`. Example: $ curl -X 'POST' \ 'http://localhost:8080/api/metadata/taskdefs' \ -H 'accept: */*' \ -H 'Content-Type: application/json' \ -d '[ { "name": "python_task_example", "description": "Python task example", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF", "responseTimeoutSeconds": 180, "ownerEmail": "example@example.com" } ]' 7. Create a `Workflow` within `Conductor`. Example: $ curl -X 'POST' \ 'http://localhost:8080/api/metadata/workflow' \ -H 'accept: */*' \ -H 'Content-Type: application/json' \ -d '{ "createTime": 1634021619147, "updateTime": 1630694890267, "name": "workflow_with_python_task_example", "description": "Workflow with Python Task example", "version": 1, "tasks": [ { "name": "python_task_example", "taskReferenceName": "python_task_example_ref_1", "inputParameters": {}, "type": "SIMPLE" } ], "inputParameters": [], "outputParameters": { "workerOutput": "${python_task_example_ref_1.output}" }, "schemaVersion": 2, "restartable": true, "ownerEmail": "example@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0 }' 8. Start a new workflow: $ curl -X 'POST' \ 'http://localhost:8080/api/workflow/workflow_with_python_task_example' \ -H 'accept: text/plain' \ -H 'Content-Type: application/json' \ -d '{}' You should receive a *Workflow ID* at the *Response body* * *Workflow ID* example: `8ff0bc06-4413-4c94-b27a-b3210412a914` Now you must be able to see its execution through the UI. * Example: `http://localhost:5000/execution/8ff0bc06-4413-4c94-b27a-b3210412a914` 9. Run your Python file with the `main` method ### Unit Tests #### Simple validation ```shell /conductor-python/src$ python3 -m unittest -v test_execute_task (tst.automator.test_task_runner.TestTaskRunner) ... ok test_execute_task_with_faulty_execution_worker (tst.automator.test_task_runner.TestTaskRunner) ... ok test_execute_task_with_invalid_task (tst.automator.test_task_runner.TestTaskRunner) ... ok ---------------------------------------------------------------------- Ran 3 tests in 0.001s OK ``` #### Run with code coverage ```shell /conductor-python/src$ python3 -m coverage run --source=conductor/ -m unittest ``` Report: ```shell /conductor-python/src$ python3 -m coverage report ``` Visual coverage results: ```shell /conductor-python/src$ python3 -m coverage html ``` ================================================ FILE: docs/docs/index.md ================================================
    Scalable Workflow Orchestration
    Conductor is a platform created by Netflix to orchestrate workflows that span across microservices.
    Get Started
    Open Source
    Apache-2.0 license for commercial and non-commerical use. Freedom to deploy, modify and contribute back.
    Modular
    A fully abstracted backend enables you choose your own database persistence layer and queueing service.
    Proven
    Enterprise ready, Java Spring based platform that has been battle tested in production systems at Netflix and elsewhere.
    Control
    Powerful flow control constructs including Decisions, Dynamic Fork-Joins and Subworkflows. Variables and templates are supported.
    Polyglot
    Client libraries in multiple languages allows workers to be implemented in Java, Node JS, Python and C#.
    Scalable
    Distributed architecture for both orchestrator and workers scalable from a single workflow to millions of concurrent processes.
    Developer Experience
    • Discover and visualize the process flows from the bundled UI
    • Integrated interface to create, refine and validate workflows
    • JSON based workflow definition DSL
    • Full featured API for custom automation
    Observability
    • Understand, debug and iterate on task and workflow executions.
    • Fine grain operational control over workflows with the ability to pause, resume, restart, retry and terminate

    Why Conductor?

    Service Orchestration

    Workflow definitions are decoupled from task implementations. This allows the creation of process flows in which each individual task can be implemented by an encapsulated microservice.

    Designing a workflow orchestrator that is resilient and horizontally scalable is not a simple problem. At Netflix we have developed a solution in Conductor.

    Service Choreography

    Process flows are implicitly defined across multiple service implementations, often with tight peer-to-peer coupling between services. Multiple event buses and complex pub/sub models limit observability around process progress and capacity.
    ================================================ FILE: docs/docs/labs/beginner.md ================================================ # Beginner Lab ## Hands on mode Please feel free to follow along using any of these resources: - Using cURL - Postman or similar REST client ## Creating a Workflow Let's create a simple workflow that adds Netflix Idents to videos. We'll be mocking the adding Idents part and focusing on actually executing this process flow. !!!info "What are Netflix Idents?" Netflix Idents are those 4 second videos with Netflix logo, which appears at the beginning and end of shows. You might have also noticed they're different for Animation and several other genres. !!!warning "Disclaimer" Obviously, this is not how Netflix adds Idents. Those Workflows are indeed very complex. But, it should give you an idea about how Conductor can be used to implement similar features. The workflow in this lab will look like this: ![img](img/bgnr_complete_workflow.png) This workflow contains the following: * Worker Task `verify_if_idents_are_added` to verify if Idents are already added. * [Switch Task](/reference-docs/switch-task.html) that takes output from the previous task, and decides whether to schedule the `add_idents` task. * `add_idents` task which is another worker Task. ### Creating Task definitions Let's create the [task definition](/configuration/taskdef.html) for `verify_if_idents_are_added` in JSON. This task will be a *SIMPLE* task which is supposed to be executed by an Idents microservice. We'll be mocking the Idents microservice part. **Note** that at this point, we don't have to specify whether it is a System task or Worker task. We are only specifying the required configurations for the task, like number of times it should be retried, timeouts etc. We shall start by using `name` parameter for task name. ```json { "name": "verify_if_idents_are_added" } ``` We'd like this task to be retried 3 times on failure. ```json { "name": "verify_if_idents_are_added", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10 } ``` And to timeout after 300 seconds. i.e. if the task doesn't finish execution within this time limit after transitioning to `IN_PROGRESS` state, the Conductor server cancels this task and schedules a new execution of this task in the queue. ```json { "name": "verify_if_idents_are_added", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF" } ``` And a [responseTimeout](/architecture/tasklifecycle.html#response-timeout-seconds) of 180 seconds. ```json { "name": "verify_if_idents_are_added", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF", "responseTimeoutSeconds": 180 } ``` We can define several other fields defined [here](/configuration/taskdef.html), but this is a good place to start with. Similarly, create another task definition: `add_idents`. ```json { "name": "add_idents", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF", "responseTimeoutSeconds": 180 } ``` Send a `POST` request to `/metadata/taskdefs` endpoint to register these tasks. You can use Swagger, Postman, CURL or similar tools. !!!info "Why is the Switch Task not registered?" System Tasks that are part of control flow do not need to be registered. However, some system tasks where the retries, rate limiting and other mechanisms are required, like `HTTP` Task, are to be registered though. !!! Important Task and Workflow Definition names are unique. The names we use below might have already been registered. For this lab, add a prefix with your username, `{my_username}_verify_if_idents_are_added` for example. This is definitely not recommended for Production usage though. **Example** ``` curl -X POST \ http://localhost:8080/api/metadata/taskdefs \ -H 'Content-Type: application/json' \ -d '[ { "name": "verify_if_idents_are_added", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF", "responseTimeoutSeconds": 180, "ownerEmail": "type your email here" }, { "name": "add_idents", "retryCount": 3, "retryLogic": "FIXED", "retryDelaySeconds": 10, "timeoutSeconds": 300, "timeoutPolicy": "TIME_OUT_WF", "responseTimeoutSeconds": 180, "ownerEmail": "type your email here" } ]' ``` ### Creating Workflow Definition Creating Workflow definition is almost similar. We shall use the Task definitions created above. Note that same Task definitions can be used in multiple workflows, or for multiple times in same Workflow (that's where `taskReferenceName` is useful). A workflow without any tasks looks like this: ```json { "name": "add_netflix_identation", "description": "Adds Netflix Identation to video files.", "version": 1, "schemaVersion": 2, "tasks": [] } ``` Add the first task that this workflow has to execute. All the tasks must be added to the `tasks` array. ```json { "name": "add_netflix_identation", "description": "Adds Netflix Identation to video files.", "version": 1, "schemaVersion": 2, "tasks": [ { "name": "verify_if_idents_are_added", "taskReferenceName": "ident_verification", "inputParameters": { "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" } ] } ``` **Wiring Input/Outputs** Notice how we were using `${workflow.input.contentId}` to pass inputs to this task. Conductor can wire inputs between workflow and tasks, and between tasks. i.e The task `verify_if_idents_are_added` is wired to accept inputs from the workflow input using JSONPath expression `${workflow.input.param}`. Learn more about wiring inputs and outputs [here](/configuration/workflowdef.html#wiring-inputs-and-outputs). Let's define `decisionCases` now. >Note: in earlier versions of this tutorial, the "decision" task was used. This has been deprecated. Checkout the Switch task structure [here](/reference-docs/switch-task.html). A Switch task is specified by the `evaulatorType`, `expression` (the expression that defines the Switch) and `decisionCases` which lists all the branches of Switch task. In this case, we'll use `"evaluatorType": "value-param"`, meaning that we'll just use the value inputted to make the decision. Alternatively, there is a `"evaluatorType": "JavaScript"` that can be used for more complicated evaluations. Adding the switch task (without any decision cases): ```json { "name": "add_netflix_identation", "description": "Adds Netflix Identation to video files.", "version": 2, "schemaVersion": 2, "tasks": [ { "name": "verify_if_idents_are_added", "taskReferenceName": "ident_verification", "inputParameters": { "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" }, { "name": "switch_task", "taskReferenceName": "is_idents_added", "inputParameters": { "case_value_param": "${ident_verification.output.is_idents_added}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case_value_param", "decisionCases": { } } ] } ``` Each switch task can have multiple tasks, so it has to be defined as an array. ```json { "name": "add_netflix_identation", "description": "Adds Netflix Identation to video files.", "version": 2, "schemaVersion": 2, "tasks": [ { "name": "verify_if_idents_are_added", "taskReferenceName": "ident_verification", "inputParameters": { "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" }, { "name": "switch_task", "taskReferenceName": "is_idents_added", "inputParameters": { "case_value_param": "${ident_verification.output.is_idents_added}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case_value_param", "decisionCases": { "false": [ { "name": "add_idents", "taskReferenceName": "add_idents_by_type", "inputParameters": { "identType": "${workflow.input.identType}", "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" } ] } } ] } ``` Just like the task definitions, register this workflow definition by sending a POST request to `/workflow` endpoint. **Example** ``` curl -X POST \ http://localhost:8080/api/metadata/workflow \ -H 'Content-Type: application/json' \ -d '{ "name": "add_netflix_identation", "description": "Adds Netflix Identation to video files.", "version": 2, "schemaVersion": 2, "tasks": [ { "name": "verify_if_idents_are_added", "taskReferenceName": "ident_verification", "inputParameters": { "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" }, { "name": "switch_task", "taskReferenceName": "is_idents_added", "inputParameters": { "case_value_param": "${ident_verification.output.is_idents_added}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case_value_param", "decisionCases": { "false": [ { "name": "add_idents", "taskReferenceName": "add_idents_by_type", "inputParameters": { "identType": "${workflow.input.identType}", "contentId": "${workflow.input.contentId}" }, "type": "SIMPLE" } ] } } ] }' ``` ### Starting the Workflow Send a `POST` request to `/workflow` with: ```json { "name": "add_netflix_identation", "version": 2, "correlationId": "my_netflix_identation_workflows", "input": { "identType": "animation", "contentId": "my_unique_content_id" } } ``` Example: ``` curl -X POST \ http://localhost:8080/api/workflow/add_netflix_identation \ -H 'Content-Type: application/json' \ -d '{ "identType": "animation", "contentId": "my_unique_content_id" }' ``` Successful POST request should return a workflow Id, which you can use to find the execution in the UI. ### Conductor User Interface Open the UI and navigate to the RUNNING tab, the Workflow should be in the state as below: ![img](img/bgnr_state_scheduled.png) Feel free to explore the various functionalities that the UI exposes. To elaborate on a few: * Workflow Task modals (Opens on clicking any of the tasks in the workflow), which includes task I/O, logs and task JSON. * Task Details tab, which shows the sequence of task execution, status, start/end time, and link to worker details which executed the task. * Input/Output tab shows workflow input and output. ### Poll for Worker task Now that `verify_if_idents_are_added` task is in `SCHEDULED` state, it is the worker's turn to fetch the task, execute it and update Conductor with final status of the task. Ideally, the workers implementing the [Client](/gettingstarted/client.html#worker) interface would do this process, executing the tasks on real microservices. But, let's mock this part. Send a `GET` request to `/poll` endpoint with your task type. For example: ``` curl -X GET \ http://localhost:8080/api/tasks/poll/verify_if_idents_are_added ``` ### Return response, add logs We can respond to Conductor with any of the following states: * Task has COMPLETED. * Task has FAILED. * Call back after seconds [Process the task at a later time]. Considering our Ident Service has verified that the Ident's are not yet added to given Content Id, let's return the task status by sending the below `POST` request to `/tasks` endpoint, with payload: ```json { "workflowInstanceId": "{workflowId}", "taskId": "{taskId}", "reasonForIncompletion": "", "callbackAfterSeconds": 0, "workerId": "localhost", "status": "COMPLETED", "outputData": { "is_idents_added": false } } ``` Example: ``` curl -X POST \ http://localhost:8080/api/tasks \ -H 'Content-Type: application/json' \ -d '{ "workflowInstanceId": "cb7c5041-aa85-4940-acb4-3bdcfa9f5c5c", "taskId": "741f362b-ee9a-47b6-81b5-9bbbd5c4c992", "reasonForIncompletion": "", "callbackAfterSeconds": 0, "workerId": "string", "status": "COMPLETED", "outputData": { "is_idents_added": false }, "logs": [ { "log": "Ident verification successful for title: {some_title_name}, with Id: {some_id}", "createdTime": 1550178825 } ] }' ``` !!! Info "Check logs in UI" You can find the logs we just sent by clicking the `verify_if_idents_are_added`, upon which a modal should open with `Logs` tab. ### Why is System task executed, but Worker task is Scheduled. You will notice that Workflow is in the state as below after sending the POST request: ![img](img/bgnr_systask_state.png) Conductor has executed `is_idents_added` all through it's lifecycle, without us polling, or returning the status of Task. If it is still unclear, `is_idents_added` is a System task, and System tasks are executed by Conductor Server. But, `add_idents` is a SIMPLE task. So, the complete lifecyle of this task (Poll, Update) should be handled by a worker to continue with W\workflow execution. When Conductor has finished executing all the tasks in given flow, the workflow will reach Terminal state (COMPLETED, FAILED, TIMED_OUT etc.) ## Next steps You can play around this workflow by failing one of the Tasks, restarting or retrying the Workflow, or by tuning the number of retries, timeoutSeconds etc. ================================================ FILE: docs/docs/labs/eventhandlers.md ================================================ # Events and Event Handlers ## About In this Lab, we shall: * Publish an Event to Conductor using `Event` task. * Subscribe to Events, and perform actions: * Start a Workflow * Complete Task Conductor Supports Eventing with two Interfaces: * [Event Task](/configuration/systask.html#event) * [Event Handlers](/configuration/eventhandlers.html#event-handler) We shall create a simple cyclic workflow similar to this: ![img](img/EventHandlerCycle.png) ## Create Workflow Definitions Let's create two workflows: * `test_workflow_for_eventHandler` which will have an `Event` task to start another workflow, and a `WAIT` System task that will be completed by an event. * `test_workflow_startedBy_eventHandler` which will have an `Event` task to generate an event to complete `WAIT` task in the above workflow. Send `POST` requests to `/metadata/workflow` endpoint with below payloads: ```json { "name": "test_workflow_for_eventHandler", "description": "A test workflow to start another workflow with EventHandler", "version": 1, "tasks": [ { "name": "test_start_workflow_event", "taskReferenceName": "start_workflow_with_event", "type": "EVENT", "sink": "conductor" }, { "name": "test_task_tobe_completed_by_eventHandler", "taskReferenceName": "test_task_tobe_completed_by_eventHandler", "type": "WAIT" } ], "ownerEmail": "example@email.com" } ``` ```json { "name": "test_workflow_startedBy_eventHandler", "description": "A test workflow which is started by EventHandler, and then goes on to complete task in another workflow.", "version": 1, "tasks": [ { "name": "test_complete_task_event", "taskReferenceName": "complete_task_with_event", "inputParameters": { "sourceWorkflowId": "${workflow.input.sourceWorkflowId}" }, "type": "EVENT", "sink": "conductor" } ], "ownerEmail": "example@email.com" } ``` ### Event Tasks in Workflow `EVENT` task is a System task, and we shall define it just like other Tasks in Workflow, with `sink` parameter. Also, `EVENT` task doesn't have to be registered before using in Workflow. This is also true for the `WAIT` task. Hence, we will not be registering any tasks for these workflows. ## Events are sent, but they're not handled (yet) Once you try to start `test_workflow_for_eventHandler` workflow, you would notice that the event is sent successfully, but the second worflow `test_workflow_startedBy_eventHandler` is not started. We have sent the Events, but we also need to define `Event Handlers` for Conductor to take any `actions` based on the Event. Let's create `Event Handlers`. ## Create Event Handlers Event Handler definitions are pretty much like Task or Workflow definitions. We start by name: ```json { "name": "test_start_workflow" } ``` Event Handler should know the Queue it has to listen to. This should be defined in `event` parameter. When using Conductor queues, define `event` with format: ```conductor:{workflow_name}:{taskReferenceName}``` And when using SQS, define with format: ```sqs:{my_sqs_queue_name}``` ```json { "name": "test_start_workflow", "event": "conductor:test_workflow_for_eventHandler:start_workflow_with_event" } ``` Event Handler can perform a list of actions defined in `actions` array parameter, for this particular `event` queue. ```json { "name": "test_start_workflow", "event": "conductor:test_workflow_for_eventHandler:start_workflow_with_event", "actions": [ "" ], "active": true } ``` Let's define `start_workflow` action. We shall pass the name of workflow we would like to start. The `start_workflow` parameter can use any of the values from the general [Start Workflow Request](/gettingstarted/startworkflow.html). Here we are passing in the workflowId, so that the Complete Task Event Handler can use it. ```json { "action": "start_workflow", "start_workflow": { "name": "test_workflow_startedBy_eventHandler", "input": { "sourceWorkflowId": "${workflowInstanceId}" } } } ``` Send a `POST` request to `/event` endpoint: ```json { "name": "test_start_workflow", "event": "conductor:test_workflow_for_eventHandler:start_workflow_with_event", "actions": [ { "action": "start_workflow", "start_workflow": { "name": "test_workflow_startedBy_eventHandler", "input": { "sourceWorkflowId": "${workflowInstanceId}" } } } ], "active": true } ``` Similarly, create another Event Handler to complete task. ```json { "name": "test_complete_task_event", "event": "conductor:test_workflow_startedBy_eventHandler:complete_task_with_event", "actions": [ { "action": "complete_task", "complete_task": { "workflowId": "${sourceWorkflowId}", "taskRefName": "test_task_tobe_completed_by_eventHandler" } } ], "active": true } ``` ## Final flow of Workflow After wiring all of the above, starting the `test_workflow_for_eventHandler` should: 1. Start `test_workflow_startedBy_eventHandler` workflow. 2. Sets `test_task_tobe_completed_by_eventHandler` WAIT task `IN_PROGRESS`. 3. `test_workflow_startedBy_eventHandler` event task would publish an Event to complete the WAIT task above. 4. Both the workflows would move to `COMPLETED` state. ================================================ FILE: docs/docs/labs/kitchensink.md ================================================ # Kitchen Sink An example kitchensink workflow that demonstrates the usage of all the schema constructs. ### Definition ```json { "name": "kitchensink", "description": "kitchensink workflow", "version": 1, "tasks": [ { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE" }, { "name": "event_task", "taskReferenceName": "event_0", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "EVENT", "sink": "conductor" }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${task_2.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_4", "taskReferenceName": "task_4", "inputParameters": { "mod": "${task_2.output.mod}", "oddEven": "${task_2.output.oddEven}" }, "type": "SIMPLE" }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${task_4.output.dynamicTasks}", "input": "${task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input" }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN" } ], "1": [ { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_10", "taskReferenceName": "task_10", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], [ { "name": "task_11", "taskReferenceName": "task_11", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ] ] }, { "name": "join", "taskReferenceName": "join2", "type": "JOIN", "joinOn": [ "wf3", "wf4" ] } ] } }, { "name": "search_elasticsearch", "taskReferenceName": "get_es_1", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_30", "taskReferenceName": "task_30", "inputParameters": { "statuses": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "type": "SIMPLE" } ], "outputParameters": { "statues": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "ownerEmail": "example@email.com", "schemaVersion": 2 } ``` ### Visual Flow ![img](/img/kitchensink.png) ### Running Kitchensink Workflow 1. Start the server as documented [here](/gettingstarted/docker.html). Use ```-DloadSample=true``` java system property when launching the server. This will create a kitchensink workflow, related task definitions and kick off an instance of kitchensink workflow. 2. Once the workflow has started, the first task remains in the ```SCHEDULED``` state. This is because no workers are currently polling for the task. 3. We will use the REST endpoints directly to poll for tasks and updating the status. #### Start workflow execution Start the execution of the kitchensink workflow by posting the following: ```shell curl -X POST --header 'Content-Type: application/json' --header 'Accept: text/plain' 'http://localhost:8080/api/workflow/kitchensink' -d ' { "task2Name": "task_5" } ' ``` The response is a text string identifying the workflow instance id. #### Poll for the first task: ```shell curl http://localhost:8080/api/tasks/poll/task_1 ``` The response should look something like: ```json { "taskType": "task_1", "status": "IN_PROGRESS", "inputData": { "mod": null, "oddEven": null }, "referenceTaskName": "task_1", "retryCount": 0, "seq": 1, "pollCount": 1, "taskDefName": "task_1", "scheduledTime": 1486580932471, "startTime": 1486580933869, "endTime": 0, "updateTime": 1486580933902, "startDelayInSeconds": 0, "retried": false, "callbackFromWorker": true, "responseTimeoutSeconds": 3600, "workflowInstanceId": "b0d1a935-3d74-46fd-92b2-0ca1e388659f", "taskId": "b9eea7dd-3fbd-46b9-a9ff-b00279459476", "callbackAfterSeconds": 0, "polledTime": 1486580933902, "queueWaitTime": 1398 } ``` #### Update the task status * Note the values for ```taskId``` and ```workflowInstanceId``` fields from the poll response * Update the status of the task as ```COMPLETED``` as below: ```json curl -H 'Content-Type:application/json' -H 'Accept:application/json' -X POST http://localhost:8080/api/tasks/ -d ' { "taskId": "b9eea7dd-3fbd-46b9-a9ff-b00279459476", "workflowInstanceId": "b0d1a935-3d74-46fd-92b2-0ca1e388659f", "status": "COMPLETED", "outputData": { "mod": 5, "taskToExecute": "task_1", "oddEven": 0, "dynamicTasks": [ { "name": "task_1", "taskReferenceName": "task_1_1", "type": "SIMPLE" }, { "name": "sub_workflow_4", "taskReferenceName": "wf_dyn", "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1" } } ], "inputs": { "task_1_1": {}, "wf_dyn": {} } } }' ``` This will mark the task_1 as completed and schedule ```task_5``` as the next task. Repeat the same process for the subsequently scheduled tasks until the completion. ================================================ FILE: docs/docs/labs/running-first-workflow.md ================================================ # A First Workflow In this article we will explore how we can run a really simple workflow that runs without deploying any new microservice. Conductor can orchestrate HTTP services out of the box without implementing any code. We will use that to create and run the first workflow. See [System Task](/configuration/systask.html) for the list of such built-in tasks. Using system tasks is a great way to run a lot of our code in production. To bring up a local instance of Conductor follow one of the recommended steps: 1. [Running Locally - From Code](/gettingstarted/local.html) 2. [Running Locally - Docker Compose](/gettingstarted/docker.html) --- ## Configuring our First Workflow This is a sample workflow that we can leverage for our test. ```json { "name": "first_sample_workflow", "description": "First Sample Workflow", "version": 1, "tasks": [ { "name": "get_population_data", "taskReferenceName": "get_population_data", "inputParameters": { "http_request": { "uri": "https://datausa.io/api/data?drilldowns=Nation&measures=Population", "method": "GET" } }, "type": "HTTP" } ], "inputParameters": [], "outputParameters": { "data": "${get_population_data.output.response.body.data}", "source": "${get_population_data.output.response.body.source}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "example@email.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0 } ``` This is an example workflow that queries a publicly available JSON API to retrieve some data. This workflow doesn’t require any worker implementation as the tasks in this workflow are managed by the system itself. This is an awesome feature of Conductor. For a lot of typical work, we won’t have to write any code at all. Let's talk about this workflow a little more so that we can gain some context. ```json "name" : "first_sample_workflow" ``` This line here is how we name our workflow. In this case our workflow name is `first_sample_workflow` This workflow contains just one worker. The workers are defined under the key `tasks`. Here is the worker definition with the most important values: ```json { "name": "get_population_data", "taskReferenceName": "get_population_data", "inputParameters": { "http_request": { "uri": "https://datausa.io/api/data?drilldowns=Nation&measures=Population", "method": "GET" } }, "type": "HTTP" } ``` Here is a list of fields and what it does: 1. `"name"` : Name of our worker 2. `"taskReferenceName"` : This is a reference to this worker in this specific workflow implementation. We can have multiple workers of the same name in our workflow, but we will need a unique task reference name for each of them. Task reference name should be unique across our entire workflow. 3. `"inputParameters"` : These are the inputs into our worker. We can hard code inputs as we have done here. We can also provide dynamic inputs such as from the workflow input or based on the output of another worker. We can find examples of this in our documentation. 4. `"type"` : This is what defines what the type of worker is. In our example - this is `HTTP`. There are more task types which we can find in the Conductor documentation. 5. `"http_request"` : This is an input that is required for tasks of type `HTTP`. In our example we have provided a well known internet JSON API url and the type of HTTP method to invoke - `GET` We haven't talked about the other fields that we can use in our definitions as these are either just metadata or more advanced concepts which we can learn more in the detailed documentation. Ok, now that we have walked through our workflow details, let's run this and see how it works. To configure the workflow, head over to the swagger API of conductor server and access the metadata workflow create API: [http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create) If the link doesn’t open the right Swagger section, we can navigate to Metadata-Resource → `POST /api/metadata/workflow` ![Swagger UI - Metadata - Workflow](/img/tutorial/metadataWorkflowPost.png) Paste the workflow payload into the Swagger API and hit Execute. Now if we head over to the UI, we can see this workflow definition created: ![Conductor UI - Workflow Definition](/img/tutorial/uiWorkflowDefinition.png) If we click through we can see a visual representation of the workflow: ![Conductor UI - Workflow Definition - Visual Flow](/img/tutorial/uiWorkflowDefinitionVisual.png) ## 2. Running our First Workflow Let’s run this workflow. To do that we can use the swagger API under the workflow-resources [http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1) ![Swagger UI - Metadata - Workflow - Run](/img/tutorial/metadataWorkflowRun.png) Hit **Execute**! Conductor will return a workflow id. We will need to use this id to load this up on the UI. If our UI installation has search enabled we wouldn't need to copy this. If we don't have search enabled (using Elasticsearch) copy it from the Swagger UI. ![Swagger UI - Metadata - Workflow - Run](/img/tutorial/workflowRunIdCopy.png) Ok, we should see this running and get completed soon. Let’s go to the UI to see what happened. To load the workflow directly, use this URL format: ``` http://localhost:5000/execution/ ``` Replace `` with our workflow id from the previous step. We should see a screen like below. Click on the different tabs to see all inputs and outputs and task list etc. Explore away! ![Conductor UI - Workflow Run](/img/tutorial/workflowLoaded.png) ## Summary In this blog post — we learned how to run a sample workflow in our Conductor installation. Concepts we touched on: 1. Workflow creation 2. System tasks such as HTTP 3. Running a workflow via API Thank you for reading, and we hope you found this helpful. Please feel free to reach out to us for any questions and we are happy to help in any way we can. ================================================ FILE: docs/docs/metrics/client.md ================================================ # Client Metrics When using the Java client, the following metrics are published: | Name | Purpose | Tags | | ------------- |:-------------| -----| | task_execution_queue_full | Counter to record execution queue has saturated | taskType| | task_poll_error | Client error when polling for a task queue | taskType, includeRetries, status | | task_paused | Counter for number of times the task has been polled, when the worker has been paused | taskType | | task_execute_error | Execution error | taskType| | task_ack_failed | Task ack failed | taskType | | task_ack_error | Task ack has encountered an exception | taskType | | task_update_error | Task status cannot be updated back to server | taskType | | task_poll_counter | Incremented each time polling is done | taskType | | task_poll_time | Time to poll for a batch of tasks | taskType | | task_execute_time | Time to execute a task | taskType | | task_result_size | Records output payload size of a task | taskType | | workflow_input_size | Records input payload size of a workflow | workflowType, workflowVersion | | external_payload_used | Incremented each time external payload storage is used | name, operation, payloadType | Metrics on client side supplements the one collected from server in identifying the network as well as client side issues. [1]: https://github.com/Netflix/spectator ================================================ FILE: docs/docs/metrics/server.md ================================================ # Server Metrics Conductor uses [spectator](https://github.com/Netflix/spectator) to collect the metrics. - To enable conductor serve to publish metrics, add this [dependency](http://netflix.github.io/spectator/en/latest/registry/metrics3/) to your build.gradle. - Conductor Server enables you to load additional modules dynamically, this feature can be controlled using this [configuration](https://github.com/Netflix/conductor/blob/master/server/README.md#additional-modules-optional). - Create your own AbstractModule that overides configure function and registers the Spectator metrics registry. - Initialize the Registry and add it to the global registry via ```((CompositeRegistry)Spectator.globalRegistry()).add(...)```. The following metrics are published by the server. You can use these metrics to configure alerts for your workflows and tasks. | Name | Purpose | Tags | | ------------- |:-------------| -----| | workflow_server_error | Rate at which server side error is happening | methodName| | workflow_failure | Counter for failing workflows|workflowName, status| | workflow_start_error | Counter for failing to start a workflow|workflowName| | workflow_running | Counter for no. of running workflows | workflowName, version| | workflow_execution | Timer for Workflow completion | workflowName, ownerApp | | task_queue_wait | Time spent by a task in queue | taskType| | task_execution | Time taken to execute a task | taskType, includeRetries, status | | task_poll | Time taken to poll for a task | taskType| | task_poll_count | Counter for number of times the task is being polled | taskType, domain | | task_queue_depth | Pending tasks queue depth | taskType, ownerApp | | task_rate_limited | Current number of tasks being rate limited | taskType | | task_concurrent_execution_limited | Current number of tasks being limited by concurrent execution limit | taskType | | task_timeout | Counter for timed out tasks | taskType | | task_response_timeout | Counter for tasks timedout due to responseTimeout | taskType | | task_update_conflict | Counter for task update conflicts. Eg: when the workflow is in terminal state | workflowName, taskType, taskStatus, workflowStatus | | event_queue_messages_processed | Counter for number of messages fetched from an event queue | queueType, queueName | | observable_queue_error | Counter for number of errors encountered when fetching messages from an event queue | queueType | | event_queue_messages_handled | Counter for number of messages executed from an event queue | queueType, queueName | | external_payload_storage_usage | Counter for number of times external payload storage was used | name, operation, payloadType | [1]: https://github.com/Netflix/spectator ## Collecting metrics with Log4j One way of collecting metrics is to push them into the logging framework (log4j). Log4j supports various appenders that can print metrics into a console/file or even send them to remote metrics collectors over e.g. syslog channel. Conductor provides optional modules that connect metrics registry with the logging framework. To enable these modules, configure following additional modules property in config.properties: conductor.metrics-logger.enabled = true conductor.metrics-logger.reportPeriodSeconds = 15 This will push all available metrics into log4j every 15 seconds. By default, the metrics will be handled as a regular log message (just printed to console with default log4j.properties). In order to change that, you can use following log4j configuration that prints metrics into a dedicated file: log4j.rootLogger=INFO,console,file log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.appender.file=org.apache.log4j.RollingFileAppender log4j.appender.file.File=/app/logs/conductor.log log4j.appender.file.MaxFileSize=10MB log4j.appender.file.MaxBackupIndex=10 log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n # Dedicated file appender for metrics log4j.appender.fileMetrics=org.apache.log4j.RollingFileAppender log4j.appender.fileMetrics.File=/app/logs/metrics.log log4j.appender.fileMetrics.MaxFileSize=10MB log4j.appender.fileMetrics.MaxBackupIndex=10 log4j.appender.fileMetrics.layout=org.apache.log4j.PatternLayout log4j.appender.fileMetrics.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.logger.ConductorMetrics=INFO,console,fileMetrics log4j.additivity.ConductorMetrics=false This configuration is bundled with conductor-server in file: log4j-file-appender.properties and can be utilized by setting env var: LOG4J_PROP=log4j-file-appender.properties This variable is used by _startup.sh_ script. ### Integration with logstash using a log file The metrics collected by log4j can be further processed and pushed into a central collector such as ElasticSearch. One way of achieving this is to use: log4j file appender -> logstash -> ElasticSearch. Considering the above setup, you can deploy logstash to consume the contents of /app/logs/metrics.log file, process it and send further to elasticsearch. Following configuration needs to be used in logstash to achieve it: pipeline.yml: - pipeline.id: conductor_metrics path.config: "/usr/share/logstash/pipeline/logstash_metrics.conf" pipeline.workers: 2 logstash_metrics.conf input { file { path => ["/conductor-server-logs/metrics.log"] codec => multiline { pattern => "^%{TIMESTAMP_ISO8601} " negate => true what => previous } } } filter { kv { field_split => ", " include_keys => [ "name", "type", "count", "value" ] } mutate { convert => { "count" => "integer" "value" => "float" } } } output { elasticsearch { hosts => ["elasticsearch:9200"] } } Note: In addition to forwarding the metrics into ElasticSearch, logstash will extract following fields from each metric: name, type, count, value and set proper types ### Integration with fluentd using a syslog channel Another example of metrics collection uses: log4j syslog appender -> fluentd -> prometheus. In this case, a specific log4j properties file needs to be used so that metrics are pushed into a syslog channel: ``` log4j.rootLogger=INFO,console,file log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.appender.file=org.apache.log4j.RollingFileAppender log4j.appender.file.File=/app/logs/conductor.log log4j.appender.file.MaxFileSize=10MB log4j.appender.file.MaxBackupIndex=10 log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n # Syslog based appender streaming metrics into fluentd log4j.appender.server=org.apache.log4j.net.SyslogAppender log4j.appender.server.syslogHost=fluentd:5170 log4j.appender.server.facility=LOCAL1 log4j.appender.server.layout=org.apache.log4j.PatternLayout log4j.appender.server.layout.ConversionPattern=%d{ISO8601} %5p [%t] (%C) - %m%n log4j.logger.ConductorMetrics=INFO,console,server log4j.additivity.ConductorMetrics=false ``` And on the fluentd side you need following configuration: ``` @type prometheus @type syslog port 5170 bind 0.0.0.0 tag conductor ; only allow TIMER metrics of workflow execution and extract tenant ID @type regexp expression /^.*type=TIMER, name=workflow_execution.class-WorkflowMonitor.+workflowName-(?.*)_(?.+), count=(?\d+), min=(?[\d.]+), max=(?[\d.]+), mean=(?[\d.]+).*$/ types count:integer,min:float,max:float,mean:float @type prometheus name conductor_workflow_count type gauge desc The total number of executed workflows key count workflow ${workflow} tenant ${tenant} user ${email} name conductor_workflow_max_duration type gauge desc Max duration in millis for a workflow key max workflow ${workflow} tenant ${tenant} user ${email} name conductor_workflow_mean_duration type gauge desc Mean duration in millis for a workflow key mean workflow ${workflow} tenant ${tenant} user ${email} @type stdout ``` With above configuration, fluentd will: - Listen to raw metrics on 0.0.0.0:5170 - Collect only workflow_execution TIMER metrics - Process the raw metrics and expose 3 prometheus specific metrics - Expose prometheus metrics on http://fluentd:24231/metrics ## Collecting metrics with Prometheus Another way to collect metrics is using Prometheus client to push them to Prometheus server. Conductor provides optional modules that connect metrics registry with Prometheus. To enable these modules, configure following additional module property in config.properties: conductor.metrics-prometheus.enabled = true This will simply push these metrics via Prometheus collector. However, you need to configure your own Prometheus collector and expose the metrics via an endpoint. ================================================ FILE: docs/docs/reference-docs/annotation-processor.md ================================================ # Annotation Processor - Original Author: Vicent Martí - https://github.com/vmg - Original Repo: https://github.com/vmg/protogen This module is strictly for code generation tasks during builds based on annotations. Currently supports `protogen` ### Usage See example below ### Example This is an actual example of this module which is implemented in common/build.gradle ```groovy task protogen(dependsOn: jar, type: JavaExec) { classpath configurations.annotationsProcessorCodegen main = 'com.netflix.conductor.annotationsprocessor.protogen.ProtoGenTask' args( "conductor.proto", "com.netflix.conductor.proto", "github.com/netflix/conductor/client/gogrpc/conductor/model", "${rootDir}/grpc/src/main/proto", "${rootDir}/grpc/src/main/java/com/netflix/conductor/grpc", "com.netflix.conductor.grpc", jar.archivePath, "com.netflix.conductor.common", ) } ``` ================================================ FILE: docs/docs/reference-docs/archival-of-workflows.md ================================================ # Archival Of Workflows Conductor has support for archiving workflow upon termination or completion. Enabling this will delete the workflow from the configured database, but leave the associated data in Elasticsearch so it is still searchable. To enable, set the `conductor.workflow-status-listener.type` property to `archive`. A number of additional properties are available to control archival. | Property | Default Value | Description | | -- | -- | -- | | conductor.workflow-status-listener.archival.ttlDuration | 0s | The time to live in seconds for workflow archiving module. Currently, only RedisExecutionDAO supports this | | conductor.workflow-status-listener.archival.delayQueueWorkerThreadCount | 5 | The number of threads to process the delay queue in workflow archival | | conductor.workflow-status-listener.archival.delaySeconds | 60 | The time to delay the archival of workflow | ================================================ FILE: docs/docs/reference-docs/azureblob-storage.md ================================================ # Azure Blob Storage The [AzureBlob storage](https://github.com/Netflix/conductor/tree/main/azureblob-storage) module uses azure blob to store and retrieve workflows/tasks input/output payload that went over the thresholds defined in properties named `conductor.[workflow|task].[input|output].payload.threshold.kb`. **Warning** Azure Java SDK use libs already present inside `conductor` like `jackson` and `netty`. You may encounter deprecated issues, or conflicts and need to adapt the code if the module is not maintained along with `conductor`. It has only been tested with **v12.2.0**. ## Configuration ### Usage Cf. Documentation [External Payload Storage](https://netflix.github.io/conductor/externalpayloadstorage/#azure-blob-storage) ### Example ```properties conductor.additional.modules=com.netflix.conductor.azureblob.AzureBlobModule es.set.netty.runtime.available.processors=false workflow.external.payload.storage=AZURE_BLOB workflow.external.payload.storage.azure_blob.connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;EndpointSuffix=localhost workflow.external.payload.storage.azure_blob.signedurlexpirationseconds=360 ``` ## Testing You can use [Azurite](https://github.com/Azure/Azurite) to simulate an Azure Storage. ### Troubleshoots * When using **es5 persistence** you will receive an `java.lang.IllegalStateException` because the Netty lib will call `setAvailableProcessors` two times. To resolve this issue you need to set the following system property ``` es.set.netty.runtime.available.processors=false ``` If you want to change the default HTTP client of azure sdk, you can use `okhttp` instead of `netty`. For that you need to add the following [dependency](https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/storage/azure-storage-blob#default-http-client). ``` com.azure:azure-core-http-okhttp:${compatible version} ``` ================================================ FILE: docs/docs/reference-docs/directed-acyclic-graph.md ================================================ # Directed Acyclic Graph (DAG) ## What is a Directed Acyclic Graph (DAG)? Conductor workflows are directed acyclic graphs (DAGs). But, what exactly is a DAG? To understand a DAG, we'll walk through each term (but not in order): ### Graph A graph is "a collection of vertices (or point) and edges (or lines) that indicate connections between the vertices." By this definition, this is a graph - just not exactly correct in the context of DAGs:

    pirate vs global warming graph

    But in the context of workflows, we're thinking of a graph more like this:

    a regular graph (source: wikipedia)

    Imagine each vertex as a microservice, and the lines are how the microservices are connected together. However, this graph is not a directed graph - as there is no direction given to each connection. ### Directed A directed graph means that there is a direction to each connection. For example, this graph is directed:

    directed graph

    Each arrow has a direction, Point "N" can proceed directly to "B", but "B" cannot proceed to "N" in the opposite direction. ### Acyclic Acyclic means without circular or cyclic paths. In the directed example above, A -> B -> D -> A is a cyclic loop. So a Directed Acyclic Graph is a set of vertices where the connections are directed without any looping. DAG charts can only "move forward" and cannot redo a step (or series of steps.) Since a Conductor workflow is a series of vertices that can connect in only a specific direction and cannot loop, a Conductor workflow is thus a directed acyclic graph:

    Conductor Dag

    ### Can a workflow have loops and still be a DAG? Yes. For example, Conductor workflows have Do-While loops:

    Conductor Dag

    This is still a DAG, because the loop is just shorthand for running the tasks inside the loop over and over again. For example, if the 2nd loop in the above image is run 3 times, the workflow path will be: 1. zero_offset_fix_1 2. post_to_orbit_ref_1 3. zero_offset_fix_2 4. post_to_orbit_ref_2 5. zero_offset_fix_3 6. post_to_orbit_ref_3 The path is directed forward, and the loop just makes it easier to define the workflow. ================================================ FILE: docs/docs/reference-docs/do-while-task.md ================================================ --- sidebar_position: 1 --- # Do-While ```json "type" : "DO_WHILE" ``` ## Introduction Sequentially execute a list of task as long as a condition is true. The list of tasks is executed first, before the condition is checked (even for the first iteration). When scheduled, each task of this loop will see its `taskReferenceName` concatenated with __i, with i being the iteration number, starting at 1. Warning: taskReferenceName containing arithmetic operators must not be used. Each task output is stored as part of the DO_WHILE task, indexed by the iteration value (see example below), allowing the condition to reference the output of a task for a specific iteration (eg. $.LoopTask['iteration]['first_task']) The DO_WHILE task is set to `FAILED` as soon as one of the loopOver fails. In such case retry, iteration starts from 1. ### Limitations - Domain or isolation group execution is unsupported; - Nested DO_WHILE is unsupported, however, DO_WHILE task supports SUB_WORKFLOW as loopOver task, so we can achieve similar functionality. - Since loopover tasks will be executed in loop inside scope of parent do while task, crossing branching outside of DO_WHILE task is not respected. Branching inside loopOver task is supported. ## Configuration ### Input Parameters: | name | type | description | |---------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | loopCondition | String | Condition to be evaluated after every iteration. This is a Javascript expression, evaluated using the Nashorn engine. If an exception occurs during evaluation, the DO_WHILE task is set to FAILED_WITH_TERMINAL_ERROR. | | loopOver | List[Task] | List of tasks that needs to be executed as long as the condition is true. | ### Output Parameters | name | type | description | |-----------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | iteration | Integer | Iteration number: the current one while executing; the final one once the loop is finished | | `i` | Map[String, Any] | Iteration number as a string, mapped to the task references names and their output. | | * | Any | Any state can be stored here if the `loopCondition` does so. For example `storage` will exist if `loopCondition` is `if ($.LoopTask['iteration'] <= 10) {$.LoopTask.storage = 3; true } else {false}` | ## Examples The following definition: ```json { "name": "Loop Task", "taskReferenceName": "LoopTask", "type": "DO_WHILE", "inputParameters": { "value": "${workflow.input.value}" }, "loopCondition": "if ( ($.LoopTask['iteration'] < $.value ) || ( $.first_task['response']['body'] > 10)) { false; } else { true; }", "loopOver": [ { "name": "first task", "taskReferenceName": "first_task", "inputParameters": { "http_request": { "uri": "http://localhost:8082", "method": "POST" } }, "type": "HTTP" },{ "name": "second task", "taskReferenceName": "second_task", "inputParameters": { "http_request": { "uri": "http://localhost:8082", "method": "POST" } }, "type": "HTTP" } ], "startDelay": 0, "optional": false } ``` will produce the following execution, assuming 3 executions occurred (alongside `first_task__1`, `first_task__2`, `first_task__3`, `second_task__1`, `second_task__2` and `second_task__3`): ```json { "taskType": "DO_WHILE", "outputData": { "iteration": 3, "1": { "first_task": { "response": {}, "headers": { "Content-Type": "application/json" } }, "second_task": { "response": {}, "headers": { "Content-Type": "application/json" } } }, "2": { "first_task": { "response": {}, "headers": { "Content-Type": "application/json" } }, "second_task": { "response": {}, "headers": { "Content-Type": "application/json" } } }, "3": { "first_task": { "response": {}, "headers": { "Content-Type": "application/json" } }, "second_task": { "response": {}, "headers": { "Content-Type": "application/json" } } } } } ``` ## Example using iteration key Sometimes, you may want to use the iteration value/counter in the tasks used in the loop. In this example, an API call is made to GitHub (to the Netflix Conductor repository), but each loop increases the pagination. The Loop ```taskReferenceName``` is "get_all_stars_loop_ref". In the ```loopCondition``` the term ```$.get_all_stars_loop_ref['iteration']``` is used. In tasks embedded in the loop, ```${get_all_stars_loop_ref.output.iteration}``` is used. In this case, it is used to define which page of results the API should return. ```json { "name": "get_all_stars", "taskReferenceName": "get_all_stars_loop_ref", "inputParameters": { "stargazers": "4000" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.get_all_stars_loop_ref['iteration'] < Math.ceil($.stargazers/100)) { true; } else { false; }", "loopOver": [ { "name": "100_stargazers", "taskReferenceName": "hundred_stargazers_ref", "inputParameters": { "counter": "${get_all_stars_loop_ref.output.iteration}", "http_request": { "uri": "https://api.github.com/repos/ntflix/conductor/stargazers?page=${get_all_stars_loop_ref.output.iteration}&per_page=100", "method": "GET", "headers": { "Authorization": "token ${workflow.input.gh_token}", "Accept": "application/vnd.github.v3.star+json" } } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "retryCount": 3 } ] } ``` ================================================ FILE: docs/docs/reference-docs/dynamic-fork-task.md ================================================ # Dynamic Fork ```json "type" : "FORK_JOIN_DYNAMIC" ``` ## Introduction A Fork operation in conductor, lets you run a specified list of other tasks or sub workflows in parallel after the fork task. A fork task is followed by a join operation that waits on the forked tasks or sub workflows to finish. The `JOIN` task also collects outputs from each of the forked tasks or sub workflows. In a regular fork operation (`FORK_JOIN` task), the list of tasks or sub workflows that need to be forked and run in parallel are already known at the time of workflow definition creation time. However, there are cases when that list can only be determined at run-time and that is when the dynamic fork operation (FORK_JOIN_DYNAMIC task) is needed. There are three things that are needed to configure a `FORK_JOIN_DYNAMIC` task. 1. A list of tasks or sub-workflows that needs to be forked and run in parallel. 2. A list of inputs to each of these forked tasks or sub-workflows 3. A task prior to the `FORK_JOIN_DYNAMIC` tasks outputs 1 and 2 above that can be wired in as in input to the `FORK_JOIN_DYNAMIC` tasks ## Use Cases A `FORK_JOIN_DYNAMIC` is useful, when a set of tasks or sub-workflows needs to be executed and the number of tasks or sub-workflows are determined at run time. E.g. Let's say we have a task that resizes an image, and we need to create a workflow that will resize an image into multiple sizes. In this case, a task can be created prior to the `FORK_JOIN_DYNAMIC` task that will prepare the input that needs to be passed into the `FORK_JOIN_DYNAMIC` task. The single image resize task does one job. The `FORK_JOIN_DYNAMIC` and the following `JOIN` will manage the multiple invokes of the single image resize task. Here, the responsibilities are clearly broken out, where the single image resize task does the core job and `FORK_JOIN_DYNAMIC` manages the orchestration and fault tolerance aspects. ## Configuration Here is an example of a `FORK_JOIN_DYNAMIC` task followed by a `JOIN` task ```json { "inputParameters": { "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput" }, { "name": "image_multiple_convert_resize_join", "taskReferenceName": "image_multiple_convert_resize_join_ref", "type": "JOIN" } ``` Dissecting into this example above, let's look at the three things that are needed to configured for the `FORK_JOIN_DYNAMIC` task `dynamicForkTasksParam` This is a JSON array of task or sub-workflow objects that specifies the list of tasks or sub-workflows that needs to be forked and run in parallel `dynamicForkTasksInputParamName` This is a JSON map of task or sub-workflow objects that specifies the list of tasks or sub-workflows that needs to be forked and run in parallel fooBarTask This is a task that is defined prior to the FORK_JOIN_DYNAMIC in the workflow definition. This task will need to output (outputParameters) 1 and 2 above so that it can be wired into inputParameters of the FORK_JOIN_DYNAMIC tasks. (dynamicTasks and dynamicTasksInput) ## Input Configuration | Attribute | Description | | ----------- | ----------- | | name | Task Name. A unique name that is descriptive of the task function | | taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | | type | Task Type. In this case, `FORK_JOIN_DYNAMIC` | | inputParameters | The input parameters that will be supplied to this task. | | dynamicForkTasksParam | This is a JSON array of tasks or sub-workflow objects that needs to be forked and run in parallel (Note: This has a different format for ```SUB_WORKFLOW``` compared to ```SIMPLE``` tasks.) | | dynamicForkTasksInputParamName | A JSON map, where the keys are task or sub-workflow names, and the values are its corresponding inputParameters | ## Example Let's say we have a task that resizes an image, and we need to create a workflow that will resize an image into multiple sizes. In this case, a task can be created prior to the `FORK_JOIN_DYNAMIC` task that will prepare the input that needs to be passed into the `FORK_JOIN_DYNAMIC` task. These will be: * ```dynamicForkTasksParam``` the JSON array of tasks/subworkflows to be run in parallel. Each JSON object will have: * A unique ```taskReferenceName```. * The name of the Task/Subworkflow to be called (note - the location of this key:value is different for a subworkflow). * The type of the task (This is optional for SIMPLE tasks). * ```dynamicForkTasksInputParamName``` a JSON map of input parameters for each task. The keys will be the unique ```taskReferenceName``` defined in the first JSON array, and the values will be the specific input parameters for the task/subworkflow. The ```image_resize``` task works to resize just one image. The `FORK_JOIN_DYNAMIC` and the following `JOIN` will manage the multiple invocations of the single ```image_resize``` task. The responsibilities are clearly broken out, where the individual ```image_resize``` tasks do the core job and `FORK_JOIN_DYNAMIC` manages the orchestration and fault tolerance aspects of handling multiple invocations of the task. ## The workflow Here is an example of a `FORK_JOIN_DYNAMIC` task followed by a `JOIN` task. The fork is named and given a taskReferenceName, but all of the input parameters are JSON variables that we will discuss next: ```json { "name": "image_multiple_convert_resize_fork", "taskReferenceName": "image_multiple_convert_resize_fork_ref", "inputParameters": { "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput" }, { "name": "image_multiple_convert_resize_join", "taskReferenceName": "image_multiple_convert_resize_join_ref", "type": "JOIN" } ``` This appears in the UI as follows: ![diagram of dynamic fork](/img/dynamic-task-diagram.png) Let's assume this data is sent to the workflow: ``` { "fileLocation": "https://pbs.twimg.com/media/FJY7ud0XEAYVCS8?format=png&name=900x900", "outputFormats": ["png","jpg"], "outputSizes": [ {"width":300, "height":300}, {"width":200, "height":200} ], "maintainAspectRatio": "true" } ``` With 2 file formats and 2 sizes in the input, we'll be creating 4 images total. The first task will generate the tasks and the parameters for these tasks: * `dynamicForkTasksParam` This is a JSON array of task or sub-workflow objects that specifies the list of tasks or sub-workflows that needs to be forked and run in parallel. This JSON varies depeding oon the type of task. ### ```dynamicForkTasksParam``` Simple task In this case, our fork is running a SIMPLE task: ```image_convert_resize```: ``` { "dynamicTasks": [ { "name": :"image_convert_resize", "taskReferenceName": "image_convert_resize_png_300x300_0", ... }, { "name": :"image_convert_resize", "taskReferenceName": "image_convert_resize_png_200x200_1", ... }, { "name": :"image_convert_resize", "taskReferenceName": "image_convert_resize_jpg_300x300_2", ... }, { "name": :"image_convert_resize", "taskReferenceName": "image_convert_resize_jpg_200x200_3", ... } ]} ``` ### ```dynamicForkTasksParam``` SubWorkflow task In this case, our Dynamic fork is running a SUB_WORKFLOW task: ```image_convert_resize_subworkflow``` ``` { "dynamicTasks": [ { "subWorkflowParam" : { "name": :"image_convert_resize_subworkflow", "version": "1" }, "type" : "SUB_WORKFLOW", "taskReferenceName": "image_convert_resize_subworkflow_png_300x300_0", ... }, { "subWorkflowParam" : { "name": :"image_convert_resize_subworkflow", "version": "1" }, "type" : "SUB_WORKFLOW", "taskReferenceName": "image_convert_resize_subworkflow_png_200x200_1", ... }, { "subWorkflowParam" : { "name": :"image_convert_resize_subworkflow", "version": "1" }, "type" : "SUB_WORKFLOW", "taskReferenceName": "image_convert_resize_subworkflow_jpg_300x300_2", ... }, { "subWorkflowParam" : { "name": :"image_convert_resize_subworkflow", "version": "1" }, "type" : "SUB_WORKFLOW", "taskReferenceName": "image_convert_resize_subworkflow_jpg_200x200_3", ... } ]} ``` * `dynamicForkTasksInputParamName` This is a JSON map of task or sub-workflow objects and all the input parameters that these tasks will need to run. ``` "dynamicTasksInput":{ "image_convert_resize_jpg_300x300_2":{ "outputWidth":300 "outputHeight":300 "fileLocation":"https://pbs.twimg.com/media/FJY7ud0XEAYVCS8?format=png&name=900x900" "outputFormat":"jpg" "maintainAspectRatio":true } "image_convert_resize_jpg_200x200_3":{ "outputWidth":200 "outputHeight":200 "fileLocation":"https://pbs.twimg.com/media/FJY7ud0XEAYVCS8?format=png&name=900x900" "outputFormat":"jpg" "maintainAspectRatio":true } "image_convert_resize_png_200x200_1":{ "outputWidth":200 "outputHeight":200 "fileLocation":"https://pbs.twimg.com/media/FJY7ud0XEAYVCS8?format=png&name=900x900" "outputFormat":"png" "maintainAspectRatio":true } "image_convert_resize_png_300x300_0":{ "outputWidth":300 "outputHeight":300 "fileLocation":"https://pbs.twimg.com/media/FJY7ud0XEAYVCS8?format=png&name=900x900" "outputFormat":"png" "maintainAspectRatio":true } ``` ### The Join The [JOIN](/reference-docs/join-task.html) task will run after all of the dynamic tasks, collecting the output for all of the tasks. ================================================ FILE: docs/docs/reference-docs/dynamic-task.md ================================================ # Dynamic ```json "type" : "DYNAMIC" ``` ### Introduction Dynamic Task allows to execute one of the registered Tasks dynamically at run-time. It accepts the task name to execute as `taskToExecute` in `inputParameters`. ### Use Cases Consider a scenario, when we have to make decision of executing a task dynamically i.e. while the workflow is still running. In such cases, Dynamic Task would be useful. ### Configuration Dynamic task is defined directly inside the workflow with type `DYNAMIC`. #### Inputs Following are the input parameters : | name | description | |----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | dynamicTaskNameParam | Name of the parameter from the task input whose value is used to schedule the task. e.g. if the value of the parameter is ABC, the next task scheduled is of type 'ABC'. | #### Output TODO: Talk about output of the task, what to expect ### Examples Suppose in a workflow, we have to take decision to ship the courier with the shipping service providers on the basis of Post Code. Following task `shipping_info` generates an output on the basis of which decision would be taken to run the next task. ```json { "name": "shipping_info", "retryCount": 3, "timeoutSeconds": 600, "pollTimeoutSeconds": 1200, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "ownerEmail":"abc@example.com", "rateLimitPerFrequency": 1 } ``` Following are the two worker tasks, one among them would execute on the basis of output generated by the `shipping_info` task : ```json { "name": "ship_via_fedex", "retryCount": 3, "timeoutSeconds": 600, "pollTimeoutSeconds": 1200, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "ownerEmail":"abc@example.com", "rateLimitPerFrequency": 2 }, { "name": "ship_via_ups", "retryCount": 3, "timeoutSeconds": 600, "pollTimeoutSeconds": 1200, "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "rateLimitFrequencyInSeconds": 60, "ownerEmail":"abc@example.com", "rateLimitPerFrequency": 2 } ``` We will create the Workflow with the following definition : ```json { "name": "Shipping_Flow", "description": "Ships smartly on the basis of Shipping info", "version": 1, "tasks": [ { "name": "shipping_info", "taskReferenceName": "shipping_info", "inputParameters": { }, "type": "SIMPLE" }, { "name": "shipping_task", "taskReferenceName": "shipping_task", "inputParameters": { "taskToExecute": "${shipping_info.output.shipping_service}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" } ], "restartable": true, "ownerEmail":"abc@example.com", "workflowStatusListenerEnabled": true, "schemaVersion": 2 } ``` Workflow is the created as shown in the below diagram. ![Conductor UI - Workflow Diagram](/img/tutorial/ShippingWorkflow.png) Note : `shipping_task` is a `DYNAMIC` task and the `taskToExecute` parameter can be set with input value provided while running the workflow or with the output of previous tasks. Here, it is set to the output provided by the previous task i.e. `${shipping_info.output.shipping_service}`. If the input value is provided while running the workflow it can be accessed by `${workflow.input.shipping_service}`. ```json { "shipping_service": "ship_via_fedex" } ``` We can see in the below example that on the basis of Post Code the shipping service is being decided. Based on given set of inputs i.e. Post Code starts with '9' hence, `ship_via_fedex` is executed - ![Conductor UI - Workflow Run](/img/tutorial/ShippingWorkflowRunning.png) If the Post Code started with anything other than 9 `ship_via_ups` is executed - ![Conductor UI - Workflow Run](/img/tutorial/ShippingWorkflowUPS.png) If the incorrect task name or the task that doesn't exist is provided then the workflow fails and we get the error `"Invalid task specified. Cannot find task by name in the task definitions."` If the null reference is provided in the task name then also the workflow fails and we get the error `"Cannot map a dynamic task based on the parameter and input. Parameter= taskToExecute, input= {taskToExecute=null}"` ================================================ FILE: docs/docs/reference-docs/event-task.md ================================================ --- sidebar_position: 4 --- # Event Task ```json "type" : "EVENT" ``` ### Introduction EVENT is a task used to publish an event into one of the supported eventing systems in Conductor. Conductor supports the following eventing models: 1. Conductor internal events (type: conductor) 2. SQS (type: sqs) ### Use Cases Consider a use case where at some point in the execution, an event is published to an external eventing system such as SQS. Event tasks are useful for creating event based dependencies for workflows and tasks. Consider an example where we want to publish an event into SQS to notify an external system. ```json { "type": "EVENT", "sink": "sqs:sqs_queue_name", "asyncComplete": false } ``` An example where we want to publish a message to conductor's internal queuing system. ```json { "type": "EVENT", "sink": "conductor:internal_event_name", "asyncComplete": false } ``` ### Configuration #### Input Configuration | Attribute | Description | |-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | name | Task Name. A unique name that is descriptive of the task function | | taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | | type | Task Type. In this case, `EVENT` | | sink | External event queue in the format of `prefix:location`. Prefix is either `sqs` or `conductor` and `location` specifies the actual queue name. e.g. "sqs:send_email_queue" | | asyncComplete | Boolean | #### asyncComplete * ```false``` to mark status COMPLETED upon execution * ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. #### Output Configuration Tasks's output are sent as a payload to the external event. In case of SQS the task's output is sent to the SQS message a payload. | name | type | description | |--------------------|---------|---------------------------------------| | workflowInstanceId | String | Workflow id | | workflowType | String | Workflow Name | | workflowVersion | Integer | Workflow Version | | correlationId | String | Workflow CorrelationId | | sink | String | Copy of the input data "sink" | | asyncComplete | Boolean | Copy of the input data "asyncComplete | | event_produced | String | Name of the event produced | The published event's payload is identical to the output of the task (except "event_produced"). When producing an event with Conductor as sink, the event name follows the structure: ```conductor::``` For SQS, use the **name** of the queue and NOT the URI. Conductor looks up the URI based on the name. !!!warning When using SQS add the [ContribsModule](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/contribs/ContribsModule.java) to the deployment. The module needs to be configured with AWSCredentialsProvider for Conductor to be able to use AWS APIs. !!!warning When using Conductor as sink, you have two options: defining the sink as `conductor` in which case the queue name will default to the taskReferenceName of the Event Task, or specifying the queue name in the sink, as `conductor:`. The queue name is in the `event` value of the event Handler, as `conductor::`. ### Supported Queuing Systems Conductor has support for the following external event queueing systems as part of the OSS build 1. SQS (prefix: sqs) 2. [NATS](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/nats) (prefix: nats) 3. [AMQP](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/amqp) (prefix: amqp_queue or amqp_exchange) 4. Internal Conductor (prefix: conductor) To add support for other ================================================ FILE: docs/docs/reference-docs/fork-task.md ================================================ # Fork ```json "type" : "FORK_JOIN" ``` ## Introduction A Fork operation lets you run a specified list of tasks or sub workflows in parallel. A fork task is followed by a join operation that waits on the forked tasks or sub workflows to finish. The `JOIN` task also collects outputs from each of the forked tasks or sub workflows. ## Use Cases `FORK_JOIN` tasks are typically used when a list of tasks can be run in parallel. E.g In a notification workflow, there could be multiple ways of sending notifications, i,e e-mail, SMS, HTTP etc.. These notifications are not dependent on each other, and so they can be run in parallel. In such cases, you can create 3 sub-lists of forked tasks for each of these operations. ## Configuration A `FORK_JOIN` task has a `forkTasks` attribute that expects an array. Each array is a sub-list of tasks. Each of these sub-lists are then invoked in parallel. The tasks defined within each sublist can be sequential or any other way as desired. A FORK_JOIN task has to be followed by a JOIN operation. The `JOIN` operator specifies which of the forked tasks to `joinOn` (wait for completion) before moving to the next stage in the workflow. #### Input Configuration | Attribute | Description | |-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| | name | Task Name. A unique name that is descriptive of the task function | | taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | | type | Task Type. In this case, `FORK_JOIN` | | inputParameters | The input parameters that will be supplied to this task | | forkTasks | A list of a list of tasks. Each of the outer list will be invoked in parallel. The inner list can be a graph of other tasks and sub-workflows | #### Output Configuration This is the output configuration of the `JOIN` task that is used in conjunction with the `FORK_JOIN` task. The output of the `JOIN` task is a map, where the keys are the names of the task reference names where were being `joinOn` and the keys are the corresponding outputs of those tasks. | Attribute | Description | |-----------------|-------------------------------------------------------------------------------------| | task_ref_name_1 | A task reference name that was being `joinOn`. The value is the output of that task | | task_ref_name_2 | A task reference name that was being `joinOn`. The value is the output of that task | | ... | ... | | task_ref_name_N | A task reference name that was being `joinOn`. The value is the output of that task | ### Example Imagine a workflow that sends 3 notifications: email, SMS and HTTP. Since none of these steps are dependent on the others, they can be run in parallel with a fork. The diagram will appear as: ![fork diagram](/img/fork-task-diagram.png) Here's the JSON definition for the workflow: ```json [ { "name": "fork_join", "taskReferenceName": "my_fork_join_ref", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "process_notification_payload", "taskReferenceName": "process_notification_payload_email", "type": "SIMPLE" }, { "name": "email_notification", "taskReferenceName": "email_notification_ref", "type": "SIMPLE" } ], [ { "name": "process_notification_payload", "taskReferenceName": "process_notification_payload_sms", "type": "SIMPLE" }, { "name": "sms_notification", "taskReferenceName": "sms_notification_ref", "type": "SIMPLE" } ], [ { "name": "process_notification_payload", "taskReferenceName": "process_notification_payload_http", "type": "SIMPLE" }, { "name": "http_notification", "taskReferenceName": "http_notification_ref", "type": "SIMPLE" } ] ] }, { "name": "notification_join", "taskReferenceName": "notification_join_ref", "type": "JOIN", "joinOn": [ "email_notification_ref", "sms_notification_ref" ] } ] ``` > Note: There are three parallel 'tines' to this fork, but only two of the outputs are required for the JOIN to continue. The diagram *does* draw an arrow from ```http_notification_ref``` to the ```notification_join```, but it is not required for the workflow to continue. Here is how the output of notification_join will look like. The output is a map, where the keys are the names of task references that were being `joinOn`. The corresponding values are the outputs of those tasks. ```json { "email_notification_ref": { "email_sent_at": "2021-11-06T07:37:17+0000", "email_sent_to": "test@example.com" }, "sms_notification_ref": { "smm_sent_at": "2021-11-06T07:37:17+0129", "sms_sen": "+1-425-555-0189" } } ``` See [JOIN](/reference-docs/join-task.html) for more details on the JOIN aspect of the FORK. ================================================ FILE: docs/docs/reference-docs/http-task.md ================================================ --- sidebar_position: 1 --- # HTTP Task ```json "type" : "HTTP" ``` ### Introduction An HTTP task is useful when you have a requirements such as: 1. Making calls to another service that exposes an API via HTTP 2. Fetch any resource or data present on an endpoint ### Use Cases If we have a scenario where we need to make an HTTP call into another service, we can make use of HTTP tasks. You can use the data returned from the HTTP call in your subsequent tasks as inputs. Using HTTP tasks you can avoid having to write the code that talks to these services and instead let Conductor manage it directly. This can reduce the code you have to maintain and allows for a lot of flexibility. ### Configuration HTTP task is defined directly inside the workflow with the task type `HTTP`. | name | type | description | |--------------|-------------|-------------------------| | http_request | HttpRequest | JSON object (see below) | #### Inputs | Name | Type | Description | |---------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | uri | String | URI for the service. Can be a partial when using vipAddress or includes the server address. | | method | String | HTTP method. GET, PUT, POST, DELETE, OPTIONS, HEAD | | accept | String | Accept header. Default: ```application/json``` | | contentType | String | Content Type - supported types are ```text/plain```, ```text/html```, and ```application/json``` (Default) | | headers | Map[String, Any] | A map of additional http headers to be sent along with the request. | | body | Map[] | Request body | | vipAddress | String | When using discovery based service URLs. | | asyncComplete | Boolean | ```false``` to mark status COMPLETED upon execution ; ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. | | oauthConsumerKey | String | [OAuth](https://oauth.net/core/1.0/) client consumer key | | oauthConsumerSecret | String | [OAuth](https://oauth.net/core/1.0/) client consumer secret | | connectionTimeOut | Integer | Connection Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 100. | | readTimeOut | Integer | Read Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 150. | #### Output | name | type | description | |--------------|------------------|-----------------------------------------------------------------------------| | response | Map | JSON body containing the response if one is present | | headers | Map[String, Any] | Response Headers | | statusCode | Integer | [Http Status Code](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) | | reasonPhrase | String | Http Status Code's reason phrase | ### Examples Following is the example of HTTP task with `GET` method. We can use variables in our URI as show in the example below. ```json { "name": "Get Example", "taskReferenceName": "get_example", "inputParameters": { "http_request": { "uri": "https://jsonplaceholder.typicode.com/posts/${workflow.input.queryid}", "method": "GET" } }, "type": "HTTP" } ``` Following is the example of HTTP task with `POST` method. > Here we are using variables for our POST body which happens to be data from a previous task. This is an example of how you can **chain** HTTP calls to make complex flows happen without writing any additional code. ```json { "name": "http_post_example", "taskReferenceName": "post_example", "inputParameters": { "http_request": { "uri": "https://jsonplaceholder.typicode.com/posts/", "method": "POST", "body": { "title": "${get_example.output.response.body.title}", "userId": "${get_example.output.response.body.userId}", "action": "doSomething" } } }, "type": "HTTP" } ``` Following is the example of HTTP task with `PUT` method. ```json { "name": "http_put_example", "taskReferenceName": "put_example", "inputParameters": { "http_request": { "uri": "https://jsonplaceholder.typicode.com/posts/1", "method": "PUT", "body": { "title": "${get_example.output.response.body.title}", "userId": "${get_example.output.response.body.userId}", "action": "doSomethingDifferent" } } }, "type": "HTTP" } ``` Following is the example of HTTP task with `DELETE` method. ```json { "name": "DELETE Example", "taskReferenceName": "delete_example", "inputParameters": { "http_request": { "uri": "https://jsonplaceholder.typicode.com/posts/1", "method": "DELETE" } }, "type": "HTTP" } ``` ### Best Practices 1. Why are my HTTP tasks not getting picked up? 1. We might have too many HTTP tasks in the queue. There is a concept called Isolation Groups that you can rely on for prioritizing certain HTTP tasks over others. Read more here: [Isolation Groups](/configuration/isolationgroups.html) ================================================ FILE: docs/docs/reference-docs/human-task.md ================================================ --- sidebar_position: 1 --- # Human ```json "type" : "HUMAN" ``` ### Introduction HUMAN is used when the workflow needs to be paused for an external signal by a human to continue. ### Use Cases HUMAN is used when the workflow needs to wait and pause for human intervention (like manual approval) or an event coming from external source such as Kafka, SQS or Conductor's internal queueing mechanism. Some use cases where HUMAN task is used: 1. To add a human approval task. When the task is approved/rejected by HUMAN task is updated using `POST /tasks` API to completion. ### Configuration * taskType: HUMAN * There are no other configurations required ================================================ FILE: docs/docs/reference-docs/inline-task.md ================================================ --- sidebar_position: 11 --- # Inline Task ```json "type": "INLINE" ``` ### Introduction Inline Task helps execute necessary logic at Workflow run-time, using an evaluator. There are two supported evaluators as of now: ### Configuration | name | description | |-------------|---------------------------------------------------| | value-param | Use a parameter directly as the value | | javascript | Evaluate Javascript expressions and compute value | ### Use Cases Consider a scenario, we have to run simple evaluations in Conductor server while creating Workers. Inline task can be used to run these evaluations using an evaluator engine. ### Example 1 ```json { "name": "inline_task_example", "taskReferenceName": "inline_task_example", "type": "INLINE", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "function e() { if ($.value == 1){return {\"result\": true}} else { return {\"result\": false}}} e();" } } ``` Following are the parameters in the above example : 1. `"evaluatorType"` - Type of the evaluator. Supported evaluators: value-param, javascript which evaluates javascript expression. 2. `"expression"` - Expression associated with the type of evaluator. For javascript evaluator, Javascript evaluation engine is used to evaluate expression defined as a string. Must return a value. Besides expression, any of the properties in the input values is accessible as `$.value` for the expression to evaluate. The task output can then be referenced in downstream tasks like: `"${inline_test.output.result}"` ### Example 2 Perhaps a weather API sometimes returns Celcius, and sometimes returns Fahrenheit temperature values. This task ensures that the downstream tasks ONLY receive Celcius values: ``` { "name": "INLINE_TASK", "taskReferenceName": "inline_test", "type": "INLINE", "inputParameters": { "scale": "${workflow.input.tempScale}", "temperature": "${workflow.input.temperature}", "evaluatorType": "javascript", "expression": "function SIvaluesOnly(){if ($.scale === "F"){ centigrade = ($.temperature -32)*5/9; return {temperature: centigrade} } else { return {temperature: $.temperature} }} SIvaluesOnly();" } } ``` ================================================ FILE: docs/docs/reference-docs/join-task.md ================================================ --- sidebar_position: 1 --- # Join ```json "type" : "JOIN" ``` ### Introduction A `JOIN` task is used in conjunction with a `FORK_JOIN` or `FORK_JOIN_DYNAMIC` task. When `JOIN` is used along with a `FORK_JOIN` task, tt waits for a list of zero or more of the forked tasks to be completed. However, when used with a `FORK_JOIN_DYNAMIC` task, it implicitly waits for all of the dynamically forked tasks to complete. ### Use Cases [FORK_JOIN](/reference-docs/fork-task.html) and [FORK_JOIN_DYNAMIC](/reference-docs/dynamic-fork-task.html) task are used to execute a collection of other tasks or sub workflows in parallel. In such cases, there is a need for these forked tasks to complete before moving to the next stage in the workflow. ### Configuration #### Input Configuration | Attribute | Description | |-------------------|--------------------------------------------------------------------------------------------------------------------------------------| | name | Task Name. A unique name that is descriptive of the task function | | taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | | type | Task Type. In this case, `JOIN` | | joinOn | A list of task reference names, that this `JOIN` task will wait for completion | #### Output Configuration | Attribute | Description | |-----------------|-------------------------------------------------------------------------------------| | task_ref_name_1 | A task reference name that was being `joinOn`. The value is the output of that task | | task_ref_name_2 | A task reference name that was being `joinOn`. The value is the output of that task | | ... | ... | | task_ref_name_N | A task reference name that was being `joinOn`. The value is the output of that task | ### Examples #### Simple Example Here is an example of a _`JOIN`_ task. This task will wait for the completion of tasks `my_task_ref_1` and `my_task_ref_2` as specified by the `joinOn` attribute. ```json { "name": "join_task", "taskReferenceName": "my_join_task_ref", "type": "JOIN", "joinOn": [ "my_task_ref_1", "my_task_ref_2" ] } ``` #### Example - ignoring one fork Here is an example of a `JOIN` task used in conjunction with a `FORK_JOIN` task. The 'FORK_JOIN' spawns 3 tasks. An `email_notification` task, a `sms_notification` task, and a `http_notification` task. Email and SMS are usually best effort delivery systems. However, in case of a http based notification you get a return code and you can retry until it succeeds or eventually give up. When you setup a notification workflow, you may decide to continue, if you kicked off an email and sms notification. In that case, you can decide to `joinOn` those specific tasks. However, the `http_notification` task will still continue to execute, but it will not block the rest of the workflow from proceeding. ```json [ { "name": "fork_join", "taskReferenceName": "my_fork_join_ref", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "email_notification", "taskReferenceName": "email_notification_ref", "type": "SIMPLE" } ], [ { "name": "sms_notification", "taskReferenceName": "sms_notification_ref", "type": "SIMPLE" } ], [ { "name": "http_notification", "taskReferenceName": "http_notification_ref", "type": "SIMPLE" } ] ] }, { "name": "notification_join", "taskReferenceName": "notification_join_ref", "type": "JOIN", "joinOn": [ "email_notification_ref", "sms_notification_ref" ] } ] ``` Here is how the output of notification_join will look like. The output is a map, where the keys are the names of task references that were being `joinOn`. The corresponding values are the outputs of those tasks. ```json { "email_notification_ref": { "email_sent_at": "2021-11-06T07:37:17+0000", "email_sent_to": "test@example.com" }, "sms_notification_ref": { "smm_sent_at": "2021-11-06T07:37:17+0129", "sms_sen": "+1-425-555-0189" } } ``` ================================================ FILE: docs/docs/reference-docs/json-jq-transform-task.md ================================================ --- sidebar_position: 1 --- # JSON JQ Transform Task ```json "type" : "JSON_JQ_TRANSFORM" ``` ### Introduction JSON_JQ_TRANSFORM_TASK is a System task that allows processing of JSON data that is supplied to the task, by using the popular JQ processing tool’s query expression language. Check the [JQ Manual](https://stedolan.github.io/jq/manual/v1.5/), and the [JQ Playground](https://jqplay.org/) for more information on JQ ### Use Cases JSON is a popular format of choice for data-interchange. It is widely used in web and server applications, document storage, API I/O etc. It’s also used within Conductor to define workflow and task definitions and passing data and state between tasks and workflows. This makes a tool like JQ a natural fit for processing task related data. Some common usages within Conductor includes, working with HTTP task, JOIN tasks or standalone tasks that try to transform data from the output of one task to the input of another. ### Configuration | Attribute | Description | |-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | name | Task Name. A unique name that is descriptive of the task function | | taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | | type | Task Type. In this case, JSON_JQ_TRANSFORM | | inputParameters | The input parameters that will be supplied to this task. The parameters will be a JSON object of atleast 2 attributes, one of which will be called queryExpression. The others are user named attributes. These attributes will be accessible by the JQ query processor | | inputParameters/user-defined-key(s) | User defined key(s) along with values. | | inputParameters/queryExpression | A JQ query expression | #### Output Configuration | Attribute | Description | |------------|---------------------------------------------------------------------------| | result | The first results returned by the JQ expression | | resultList | A List of results returned by the JQ expression | | error | An optional error message, indicating that the JQ query failed processing | ### Example Here is an example of a _`JSON_JQ_TRANSFORM`_ task. The `inputParameters` attribute is expected to have a value object that has the following 1. A list of key value pair objects denoted key1/value1, key2/value2 in the example below. Note the key1/value1 are arbitrary names used in this example. 2. A key with the name `queryExpression`, whose value is a JQ expression. The expression will operate on the value of the `inputParameters` attribute. In the example below, the `inputParameters` has 2 inner objects named by attributes `key1` and `key2`, each of which has an object that is named `value1` and `value2`. They have an associated array of strings as values, `"a", "b"` and `"c", "d"`. The expression `key3: (.key1.value1 + .key2.value2)` concat's the 2 string arrays into a single array against an attribute named `key3` ```json { "name": "jq_example_task", "taskReferenceName": "my_jq_example_task", "type": "JSON_JQ_TRANSFORM", "inputParameters": { "key1": { "value1": [ "a", "b" ] }, "key2": { "value2": [ "c", "d" ] }, "queryExpression": "{ key3: (.key1.value1 + .key2.value2) }" } } ``` The execution of this example task above will provide the following output. The `resultList` attribute stores the full list of the `queryExpression` result. The `result` attribute stores the first element of the resultList. An optional `error` attribute along with a string message will be returned if there was an error processing the query expression. ```json { "result": { "key3": [ "a", "b", "c", "d" ] }, "resultList": [ { "key3": [ "a", "b", "c", "d" ] } ] } ``` ## Example JQ transforms ### Cleaning up a JSON response A HTTP Task makes an API call to GitHub to request a list of "stargazers" (users who have starred a repository). The API response (for just one user) looks like: Snippet of ```${hundred_stargazers_ref.output}``` ``` JSON "body":[ { "starred_at":"2016-12-14T19:55:46Z", "user":{ "login":"lzehrung", "id":924226, "node_id":"MDQ6VXNlcjkyNDIyNg==", "avatar_url":"https://avatars.githubusercontent.com/u/924226?v=4", "gravatar_id":"", "url":"https://api.github.com/users/lzehrung", "html_url":"https://github.com/lzehrung", "followers_url":"https://api.github.com/users/lzehrung/followers", "following_url":"https://api.github.com/users/lzehrung/following{/other_user}", "gists_url":"https://api.github.com/users/lzehrung/gists{/gist_id}", "starred_url":"https://api.github.com/users/lzehrung/starred{/owner}{/repo}", "subscriptions_url":"https://api.github.com/users/lzehrung/subscriptions", "organizations_url":"https://api.github.com/users/lzehrung/orgs", "repos_url":"https://api.github.com/users/lzehrung/repos", "events_url":"https://api.github.com/users/lzehrung/events{/privacy}", "received_events_url":"https://api.github.com/users/lzehrung/received_events", "type":"User", "site_admin":false } } ] ``` We only need the ```starred_at``` and ```login``` parameters for users who starred the repository AFTER a given date (provided as an input to the workflow ```${workflow.input.cutoff_date}```). We'll use the JQ Transform to simplify the output: ```JSON { "name": "jq_cleanup_stars", "taskReferenceName": "jq_cleanup_stars_ref", "inputParameters": { "starlist": "${hundred_stargazers_ref.output.response.body}", "queryExpression": "[.starlist[] | select (.starred_at > \"${workflow.input.cutoff_date}\") |{occurred_at:.starred_at, member: {github: .user.login}}]" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ``` The JSON is stored in ```starlist```. The ```queryExpression``` reads in the JSON, selects only entries where the ```starred_at``` value meets the date criteria, and generates output JSON of the form: ```JSON { "occurred_at": "date from JSON", "member":{ "github" : "github Login from JSON" } } ``` The entire expression is wrapped in [] to indicate that the response should be an array. ================================================ FILE: docs/docs/reference-docs/kafka-publish-task.md ================================================ --- sidebar_position: 13 --- # Kafka Publish Task ```json "type" : "KAFKA_PUBLISH" ``` ### Introduction A Kafka Publish task is used to push messages to another microservice via Kafka. ### Configuration The task expects an input parameter named ```kafka_request``` as part of the task's input with the following details: | name | description | |------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | bootStrapServers | bootStrapServers for connecting to given kafka. | | key | Key to be published | | keySerializer | Serializer used for serializing the key published to kafka. One of the following can be set :
    1. org.apache.kafka.common.serialization.IntegerSerializer
    2. org.apache.kafka.common.serialization.LongSerializer
    3. org.apache.kafka.common.serialization.StringSerializer.
    Default is String serializer | | value | Value published to kafka | | requestTimeoutMs | Request timeout while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.request.timeout.ms`. If the property is not set the value defaults to 100 ms | | maxBlockMs | maxBlockMs while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.max.block.ms`. If the property is not set the value defaults to 500 ms | | headers | A map of additional kafka headers to be sent along with the request. | | topic | Topic to publish | ### Examples Sample Task ```json { "name": "call_kafka", "taskReferenceName": "call_kafka", "inputParameters": { "kafka_request": { "topic": "userTopic", "value": "Message to publish", "bootStrapServers": "localhost:9092", "headers": { "x-Auth":"Auth-key" }, "key": "123", "keySerializer": "org.apache.kafka.common.serialization.IntegerSerializer" } }, "type": "KAFKA_PUBLISH" } ``` The task expects an input parameter named `"kafka_request"` as part of the task's input with the following details: 1. `"bootStrapServers"` - bootStrapServers for connecting to given kafka. 2. `"key"` - Key to be published. 3. `"keySerializer"` - Serializer used for serializing the key published to kafka. One of the following can be set : a. org.apache.kafka.common.serialization.IntegerSerializer b. org.apache.kafka.common.serialization.LongSerializer c. org.apache.kafka.common.serialization.StringSerializer. Default is String serializer. 4. `"value"` - Value published to kafka 5. `"requestTimeoutMs"` - Request timeout while publishing to kafka. If this value is not given the value is read from the property kafka.publish.request.timeout.ms. If the property is not set the value defaults to 100 ms. 6. `"maxBlockMs"` - maxBlockMs while publishing to kafka. If this value is not given the value is read from the property kafka.publish.max.block.ms. If the property is not set the value defaults to 500 ms. 7. `"headers"` - A map of additional kafka headers to be sent along with the request. 8. `"topic"` - Topic to publish. The producer created in the kafka task is cached. By default the cache size is 10 and expiry time is 120000 ms. To change the defaults following can be modified kafka.publish.producer.cache.size, kafka.publish.producer.cache.time.ms respectively. #### Kafka Task Output Task status transitions to `COMPLETED`. The task is marked as `FAILED` if the message could not be published to the Kafka queue. ================================================ FILE: docs/docs/reference-docs/redis.md ================================================ # Redis By default conductor runs with an in-memory Redis mock. However, you can change the configuration by setting the properties `conductor.db.type` and `conductor.redis.hosts`. ## `conductor.db.type` | Value | Description | |--------------------------------|----------------------------------------------------------------------------------------| | dynomite | Dynomite Cluster. Dynomite is a proxy layer that provides sharding and replication. | | memory | Uses an in-memory Redis mock. Should be used only for development and testing purposes.| | redis_cluster | Redis Cluster configuration. | | redis_sentinel | Redis Sentinel configuration. | | redis_standalone | Redis Standalone configuration. | ## `conductor.redis.hosts` Expected format is `host:port:rack` separated by semicolon, e.g.: ```properties conductor.redis.hosts=host0:6379:us-east-1c;host1:6379:us-east-1c;host2:6379:us-east-1c ``` ### Auth Support Password authentication is supported. The password should be set as the 4th param of the first host `host:port:rack:password`, e.g.: ```properties conductor.redis.hosts=host0:6379:us-east-1c:my_str0ng_pazz;host1:6379:us-east-1c;host2:6379:us-east-1c ``` **Notes** - In a cluster, all nodes use the same password. - In a sentinel configuration, sentinels and redis nodes use the same password. ================================================ FILE: docs/docs/reference-docs/set-variable-task.md ================================================ --- sidebar_position: 1 --- # Set Variable ```json "type" : "SET_VARIABLE" ``` ### Introduction Set Variable allows us to set workflow variables by creating or updating them with new values. ### Use Cases Variables can be initialized in the workflow definition as well as during the workflow run. Once a variable was initialized it can be read or overwritten with a new value by any other task. ### Configuration Set Variable task is defined directly inside the workflow with type `SET_VARIABLE`. ## Examples Suppose in a workflow, we have to store a value in a variable and then later in workflow reuse the value stored in the variable just as we do in programming, in such scenarios `Set Variable` task can be used. Following is the workflow definition with `SET_VARIABLE` task. ```json { "name": "Set_Variable_Workflow", "description": "Set a value to a variable and then reuse it later in the workflow", "version": 1, "tasks": [ { "name": "Set_Name", "taskReferenceName": "Set_Name", "type": "SET_VARIABLE", "inputParameters": { "name": "Foo" } }, { "name": "Read_Name", "taskReferenceName": "Read_Name", "inputParameters": { "var_name" : "${workflow.variables.name}" }, "type": "SIMPLE" } ], "restartable": true, "ownerEmail":"abc@example.com", "workflowStatusListenerEnabled": true, "schemaVersion": 2 } ``` In the above example, it can be seen that the task `Set_Name` is a Set Variable Task and the variable `name` is set to `Foo` and later in the workflow it is referenced by `"${workflow.variables.name}"` in another task. ================================================ FILE: docs/docs/reference-docs/start-workflow-task.md ================================================ --- sidebar_position: 1 --- # Start Workflow ```json "type" : "START_WORKFLOW" ``` ### Introduction Start Workflow starts another workflow. Unlike `SUB_WORKFLOW`, `START_WORKFLOW` does not create a relationship between starter and the started workflow. It also does not wait for the started workflow to complete. A `START_WORKFLOW` is considered successful once the requested workflow is started successfully. In other words, `START_WORKFLOW` is marked as COMPLETED, once the started workflow is in RUNNING state. ### Use Cases When another workflow needs to be started from a workflow, `START_WORKFLOW` can be used. ### Configuration Start Workflow task is defined directly inside the workflow with type `START_WORKFLOW`. #### Input **Parameters:** | name | type | description | |---------------|------------------|---------------------------------------------------------------------------------------------------------------------| | startWorkflow | Map[String, Any] | The value of this parameter is [Start Workflow Request](/gettingstarted/startworkflow.html#start-workflow-request). | #### Output | name | type | description | |------------|--------|--------------------------------| | workflowId | String | The id of the started workflow | ================================================ FILE: docs/docs/reference-docs/sub-workflow-task.md ================================================ --- sidebar_position: 1 --- # Sub Workflow ```json "type" : "SUB_WORKFLOW" ``` ### Introduction Sub Workflow task allows for nesting a workflow within another workflow. Nested workflows contain a reference to their parent. ### Use Cases Suppose we want to include another workflow inside our current workflow. In that case, Sub Workflow Task would be used. ### Configuration Sub Workflow task is defined directly inside the workflow with type `SUB_WORKFLOW`. #### Input **Parameters:** | name | type | description | |------------------|------------------|-------------| | subWorkflowParam | Map[String, Any] | See below | **subWorkflowParam** | name | type | description | |--------------------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| | name | String | Name of the workflow to execute | | version | Integer | Version of the workflow to execute | | taskToDomain | Map[String, String] | Allows scheduling the sub workflow's tasks per given mappings.
    See [Task Domains](/configuration/taskdomains.html) for instructions to configure taskDomains. | | workflowDefinition | [WorkflowDefinition](/configuration/workflowdef.html) | Allows starting a subworkflow with a dynamic workflow definition. | #### Output | name | type | description | |---------------|--------|-------------------------------------------------------------------| | subWorkflowId | String | Sub-workflow execution Id generated when running the sub-workflow | ### Examples Imagine we have a workflow that has a fork in it. In the example below, we input one image, but using a fork to create 2 images simultaneously: ![workflow with fork](/img/workflow_fork.png) The left fork will create a JPG, and the right fork a WEBP image. Maintaining this workflow might be difficult, as changes made to one side of the fork do not automatically propagate the other. Rather than using 2 tasks, we can define a ```image_convert_resize``` workflow that we can call for both forks as a sub-workflow: ```json {{ "name": "image_convert_resize_subworkflow1", "description": "Image Processing Workflow", "version": 1, "tasks": [{ "name": "image_convert_resize_multipleformat_fork", "taskReferenceName": "image_convert_resize_multipleformat_ref", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [{ "name": "image_convert_resize_sub", "taskReferenceName": "subworkflow_jpg_ref", "inputParameters": { "fileLocation": "${workflow.input.fileLocation}", "recipeParameters": { "outputSize": { "width": "${workflow.input.recipeParameters.outputSize.width}", "height": "${workflow.input.recipeParameters.outputSize.height}" }, "outputFormat": "jpg" } }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "image_convert_resize", "version": 1 } }], [{ "name": "image_convert_resize_sub", "taskReferenceName": "subworkflow_webp_ref", "inputParameters": { "fileLocation": "${workflow.input.fileLocation}", "recipeParameters": { "outputSize": { "width": "${workflow.input.recipeParameters.outputSize.width}", "height": "${workflow.input.recipeParameters.outputSize.height}" }, "outputFormat": "webp" } }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "image_convert_resize", "version": 1 } } ] ] }, { "name": "image_convert_resize_multipleformat_join", "taskReferenceName": "image_convert_resize_multipleformat_join_ref", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "subworkflow_jpg_ref", "upload_toS3_webp_ref" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "fileLocationJpg": "${subworkflow_jpg_ref.output.fileLocation}", "fileLocationWebp": "${subworkflow_webp_ref.output.fileLocation}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "conductor@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ``` Now our diagram will appear as: ![workflow with 2 subworkflows](/img/subworkflow_diagram.png) The inputs to both sides of the workflow are identical before and after - but we've abstracted the tasks into the sub-workflow. Any change to the sub-workflow will automatically occur in bth sides of the fork. Looking at the subworkflow (the WEBP version): ``` { "name": "image_convert_resize_sub", "taskReferenceName": "subworkflow_webp_ref", "inputParameters": { "fileLocation": "${workflow.input.fileLocation}", "recipeParameters": { "outputSize": { "width": "${workflow.input.recipeParameters.outputSize.width}", "height": "${workflow.input.recipeParameters.outputSize.height}" }, "outputFormat": "webp" } }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "image_convert_resize", "version": 1 } } ``` The ```subWorkflowParam``` tells conductor which workflow to call. The task is marked as completed upon the completion of the spawned workflow. If the sub-workflow is terminated or fails the task is marked as failure and retried if configured. ### Optional Sub Workflow Task If the Sub Workflow task is defined as optional in the parent workflow task definition, the parent workflow task will not be retried if sub-workflow is terminated or failed. In addition, even if the sub-workflow is retried/rerun/restarted after reaching to a terminal status, the parent workflow task status will remain as it is. ================================================ FILE: docs/docs/reference-docs/switch-task.md ================================================ --- sidebar_position: 1 --- # Switch ```json "type" : "SWITCH" ``` ### Introduction A switch task is similar to `case...switch` statement in a programming language. The `switch` expression, is a configuration on the `SWITCH` task type. Currently, two evaluators are supported: ### Configuration Following are the task configuration parameters : | name | type | description | |---------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| | evaluatorType | String | [evaluatortType values](#evaluator-types) | | expression | String | Depends on the [evaluatortType value](#evaluator-types) | | decisionCases | Map[String, List[task]] | Map where key is possible values that can result from `expression` being evaluated by `evaluatorType` with value being list of tasks to be executed. | | defaultCase | List[task] | List of tasks to be executed when no matching value is found in decision case (default condition) | #### Evaluator Types | name | description | expression | |-------------|---------------------------------------------------|-----------------------| | value-param | Use a parameter directly as the value | input parameter | | javascript | Evaluate JavaScript expressions and compute value | JavaScript expression | ### Use Cases Useful in any situation where we have to execute one of many task options. ### Output Following is/are output generated by the `Switch` Task. | name | type | description | |------------------|--------------|---------------------------------------------------------------| | evaluationResult | List[String] | A List of string representing the list of cases that matched. | ### Examples In this example workflow, we have to ship a package with the shipping service providers on the basis of input provided while running the workflow. Let's create a Workflow with the following switch task definition that uses `value-param` evaluatorType: ```json { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${workflow.input.service}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "switchCaseValue", "defaultCase": [ { ... } ], "decisionCases": { "fedex": [ { ... } ], "ups": [ { ... } ] } } ``` In the definition above the value of the parameter `switch_case_value` is used to determine the switch-case. The evaluator type is `value-param` and the expression is a direct reference to the name of an input parameter. If the value of `switch_case_value` is `fedex` then the decision case `ship_via_fedex`is executed as shown below. ![Conductor UI - Workflow Run](/img/Switch_Fedex.png) In a similar way - if the input was `ups`, then `ship_via_ups` will be executed. If none of the cases match then the default option is executed. Here is an example using the `javascript` evaluator type: ```json { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "inputValue": "${workflow.input.service}" }, "type": "SWITCH", "evaluatorType": "javascript", "expression": "$.inputValue == 'fedex' ? 'fedex' : 'ups'", "defaultCase": [ { ... } ], "decisionCases": { "fedex": [ { ... } ], "ups": [ { ... } ] } } ``` ================================================ FILE: docs/docs/reference-docs/terminate-task.md ================================================ --- sidebar_position: 1 --- # Terminate ```json "type" : "TERMINATE" ``` ### Introduction Task that can terminate a workflow with a given status and modify the workflow's output with a given parameter, it can act as a `return` statement for conditions where you simply want to terminate your workflow. ### Use Cases Use it when you want to terminate the workflow without continuing the execution. For example, if you have a decision where the first condition is met, you want to execute some tasks, otherwise you want to finish your workflow. ### Configuration Terminate task is defined directly inside the workflow with type `TERMINATE`. ```json { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": "${task0.output}" }, "type": "TERMINATE", "startDelay": 0, "optional": false } ``` #### Inputs **Parameters:** | name | type | description | notes | |-------------------|--------|-----------------------------------------|-------------------------| | terminationStatus | String | can only accept "COMPLETED" or "FAILED" | task cannot be optional | | workflowOutput | Any | Expected workflow output || |terminationReason|String| For failed tasks, this reason is passed to a failureWorkflow| ### Output **Outputs:** | name | type | description | |--------|------|-----------------------------------------------------------------------------------------------------------| | output | Map | The content of `workflowOutput` from the inputParameters. An empty object if `workflowOutput` is not set. | ### Examples Let's consider the same example we had in [Switch Task](/reference-docs/switch-task.html). Suppose in a workflow, we have to take decision to ship the courier with the shipping service providers on the basis of input provided while running the workflow. If the input provided while running workflow does not match with the available shipping providers then the workflow will fail and return. If input provided matches then it goes ahead. Here is a snippet that shows the default switch case terminating the workflow: ```json { "name": "switch_task", "taskReferenceName": "switch_task", "type": "SWITCH", "defaultCase": [ { "name": "terminate", "taskReferenceName": "terminate", "type": "TERMINATE", "inputParameters": { "terminationStatus": "FAILED", "terminationReason":"Shipping provider not found." } } ] } ``` Workflow gets created as shown in the diagram. ![Conductor UI - Workflow Diagram](/img/Terminate_Task.png) ### Best Practices 1. Include termination reason when terminating the workflow with failure status to make it easy to understand the cause. 2. Include any additional details (e.g. output of the tasks, switch case etc) that helps understand the path taken to termination. ================================================ FILE: docs/docs/reference-docs/wait-task.md ================================================ --- sidebar_position: 1 --- # Wait ```json "name": "waiting_task", "taskReferenceName":"waiting_task_ref", "type" : "WAIT" ``` ## Introduction WAIT is used when the workflow needs to be paused for an external signal to continue. ## Use Cases WAIT is used when the workflow needs to wait and pause for an external signal such as a human intervention (like manual approval) or an event coming from external source such as Kafka, SQS or Conductor's internal queueing mechanism. Some use cases where WAIT task is used: 1. Wait for a certain amount of time (e.g. 2 minutes) or until a certain date time (e.g. 12/25/2022 00:00) 2. To wait for and external signal coming from an event queue mechanism supported by Conductor ## Configuration * taskType: WAIT * Wait for a specific amount of time format: short: **D**d**H**h**M**m or full: **D**days**H**hours**M**minutes * The following are the accepted units: *days*, *d*, *hrs*, *hours*, *h*, *minutes*, *mins*, *m*, *seconds*, *secs*, *s* ```json { "taskType": "WAIT", "inputParameters": { "duration": "2 days 3 hours" } } ``` * Wait until specific date/time * e.g. the following Wait task remains blocked until Dec 25, 2022 9am PST * The date/time can be supplied in one of the following formats: **yyyy-MM-dd HH:mm**, **yyyy-MM-dd HH:mm**, **yyyy-MM-dd** ```json { "name":"wait_until_date", "taskReferenceName":"wait_until_date_ref", "taskType": "WAIT", "inputParameters": { "until": "2022-12-25 09:00 PST" } } ``` ## Ending a WAIT when there is no time duration specified To conclude a WAIT task, there are three endpoints that can be used. You'll need the ```workflowId```, ```taskRefName``` or ```taskId``` and the task status (generally ```COMPLETED``` or ```FAILED```). 1. POST ```/api/tasks``` 2. POST ```api/queue/update/{workflowId}/{taskRefName}/{status}``` 3. POST ```api/queue/update/{workflowId}/task/{taskId}/{status}``` Any parameter that is sent in the body of the POST message will be repeated as the output of the task. For example, if we send a COMPLETED message as follows: ```bash curl -X "POST" "https://play.orkes.io/api/queue/update/{workflowId}/waiting_around_ref/COMPLETED" -H 'Content-Type: application/json' -d '{"data_key":"somedatatoWait1","data_key2":"somedatatoWAit2"}' ``` The output of the task will be: ```json { "data_key":"somedatatoWait1", "data_key2":"somedatatoWAit2" } ``` ================================================ FILE: docs/docs/resources/code-of-conduct.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at netflixoss@netflix.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: docs/docs/resources/contributing.md ================================================ # Contributing Thanks for your interest in Conductor! This guide helps to find the most efficient way to contribute, ask questions, and report issues. Code of conduct ----- Please review our [code of conduct](code-of-conduct.md). I have a question! ----- We have a dedicated [discussion forum](https://github.com/Netflix/conductor/discussions) for asking "how to" questions and to discuss ideas. The discussion forum is a great place to start if you're considering creating a feature request or work on a Pull Request. *Please do not create issues to ask questions.* I want to contribute! ------ We welcome Pull Requests and already had many outstanding community contributions! Creating and reviewing Pull Requests take considerable time. This section helps you to set up a smooth Pull Request experience. The stable branch is [main](https://github.com/Netflix/conductor/tree/main). Please create pull requests for your contributions against [main](https://github.com/Netflix/conductor/tree/main) only. It's a great idea to discuss the new feature you're considering on the [discussion forum](https://github.com/Netflix/conductor/discussions) before writing any code. There are often different ways you can implement a feature. Getting some discussion about different options helps shape the best solution. When starting directly with a Pull Request, there is the risk of having to make considerable changes. Sometimes that is the best approach, though! Showing an idea with code can be very helpful; be aware that it might be throw-away work. Some of our best Pull Requests came out of multiple competing implementations, which helped shape it to perfection. Also, consider that not every feature is a good fit for Conductor. A few things to consider are: * Is it increasing complexity for the user, or might it be confusing? * Does it, in any way, break backward compatibility (this is seldom acceptable) * Does it require new dependencies (this is rarely acceptable for core modules) * Should the feature be opt-in or enabled by default. For integration with a new Queuing recipe or persistence module, a separate module which can be optionally enabled is the right choice. * Should the feature be implemented in the main Conductor repository, or would it be better to set up a separate repository? Especially for integration with other systems, a separate repository is often the right choice because the life-cycle of it will be different. Of course, for more minor bug fixes and improvements, the process can be more light-weight. We'll try to be responsive to Pull Requests. Do keep in mind that because of the inherently distributed nature of open source projects, responses to a PR might take some time because of time zones, weekends, and other things we may be working on. I want to report an issue ----- If you found a bug, it is much appreciated if you create an issue. Please include clear instructions on how to reproduce the issue, or even better, include a test case on a branch. Make sure to come up with a descriptive title for the issue because this helps while organizing issues. I have a great idea for a new feature ---- Many features in Conductor have come from ideas from the community. If you think something is missing or certain use cases could be supported better, let us know! You can do so by opening a discussion on the [discussion forum](https://github.com/Netflix/conductor/discussions). Provide as much relevant context to why and when the feature would be helpful. Providing context is especially important for "Support XYZ" issues since we might not be familiar with what "XYZ" is and why it's useful. If you have an idea of how to implement the feature, include that as well. Once we have decided on a direction, it's time to summarize the idea by creating a new issue. ## Code Style We use [spotless](https://github.com/diffplug/spotless) to enforce consistent code style for the project, so make sure to run `gradlew spotlessApply` to fix any violations after code changes. ## License By contributing your code, you agree to license your contribution under the terms of the APLv2: https://github.com/Netflix/conductor/blob/master/LICENSE All files are released with the Apache 2.0 license, and the following license header will be automatically added to your new file if none present: ``` /** * Copyright $YEAR Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ ``` ================================================ FILE: docs/docs/resources/license.md ================================================ # License Copyright 2022 Netflix, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: docs/docs/resources/related.md ================================================ # Community projects related to Conductor ## Client SDKs Further, all of the (non-Java) SDKs have a new GitHub home: the Conductor SDK repository is your new source for Conductor SDKs: * [Golang](https://github.com/conductor-sdk/conductor-go) * [Python](https://github.com/conductor-sdk/conductor-python) * [C#](https://github.com/conductor-sdk/conductor-csharp) * [Clojure](https://github.com/conductor-sdk/conductor-clojure) All contributions on the above client sdks can be made on [Conductor SDK](https://github.com/conductor-sdk) repository. ## Microservices operations * https://github.com/flaviostutz/schellar - Schellar is a scheduler tool for instantiating Conductor workflows from time to time, mostly like a cron job, but with transport of input/output variables between calls. * https://github.com/flaviostutz/backtor - Backtor is a backup scheduler tool that uses Conductor workers to handle backup operations and decide when to expire backups (ex.: keep backup 3 days, 2 weeks, 2 months, 1 semester) * https://github.com/cquon/conductor-tools - Conductor CLI for launching workflows, polling tasks, listing running tasks etc ## Conductor deployment * https://github.com/flaviostutz/conductor-server - Docker container for running Conductor with Prometheus metrics plugin installed and some tweaks to ease provisioning of workflows from json files embedded to the container * https://github.com/flaviostutz/conductor-ui - Docker container for running Conductor UI so that you can easily scale UI independently * https://github.com/flaviostutz/elasticblast - "Elasticsearch to Bleve" bridge tailored for running Conductor on top of Bleve indexer. The footprint of Elasticsearch may cost too much for small deployments on Cloud environment. * https://github.com/mohelsaka/conductor-prometheus-metrics - Conductor plugin for exposing Prometheus metrics over path '/metrics' ## OAuth2.0 Security Configuration Forked Repository - [Conductor (Secure)](https://github.com/maheshyaddanapudi/conductor/tree/oauth2) [OAuth2.0 Role Based Security!](https://github.com/maheshyaddanapudi/conductor/blob/oauth2/SECURITY.md) - Spring Security with easy configuration to secure the Conductor server APIs. Docker image published to [Docker Hub](https://hub.docker.com/repository/docker/conductorboot/server) ## Conductor Worker utilities * https://github.com/ggrcha/conductor-go-client - Conductor Golang client for writing Workers in Golang * https://github.com/courosh12/conductor-dotnet-client - Conductor DOTNET client for writing Workers in DOTNET * https://github.com/TwoUnderscorez/serilog-sinks-conductor-task-log - Serilog sink for sending worker log events to Netflix Conductor * https://github.com/davidwadden/conductor-workers - Various ready made Conductor workers for common operations on some platforms (ex.: Jira, Github, Concourse) ## Conductor Web UI * https://github.com/maheshyaddanapudi/conductor-ng-ui - Angular based - Conductor Workflow Management UI ## Conductor Persistence ### Mongo Persistence * https://github.com/maheshyaddanapudi/conductor/tree/mongo_persistence - With option to use Mongo Database as persistence unit. * Mongo Persistence / Option to use Mongo Database as persistence unit. * Docker Compose example with MongoDB Container. ### Oracle Persistence * https://github.com/maheshyaddanapudi/conductor/tree/oracle_persistence - With option to use Oracle Database as persistence unit. * Oracle Persistence / Option to use Oracle Database as persistence unit : version > 12.2 - Tested well with 19C * Docker Compose example with Oracle Container. ## Schedule Conductor Workflow * https://github.com/jas34/scheduledwf - It solves the following problem statements: * At times there are use cases in which we need to run some tasks/jobs only at a scheduled time. * In microservice architecture maintaining schedulers in various microservices is a pain. * We should have a central dedicate service that can do scheduling for us and provide a trigger to a microservices at expected time. * It offers an additional module `io.github.jas34.scheduledwf.config.ScheduledWfServerModule` built on the existing core of conductor and does not require deployment of any additional service. For more details refer: [Schedule Conductor Workflows](https://jas34.github.io/scheduledwf) and [Capability In Conductor To Schedule Workflows](https://github.com/Netflix/conductor/discussions/2256) ================================================ FILE: docs/docs/technicaldetails.md ================================================ # Technical Details ### gRPC Framework As part of this addition, all of the modules and bootstrap code within them were refactored to leverage providers, which facilitated moving the Jetty server into a separate module and the conformance to Guice guidelines and best practices. This feature constitutes a server-side gRPC implementation along with protobuf RPC schemas for the workflow, metadata and task APIs that can be run concurrently with the Jersey-based HTTP/REST server. The protobuf models for all the types are exposed through the API. gRPC java clients for the workflow, metadata and task APIs are also available for use. Another valuable addition is an idiomatic Go gRPC client implementation for the worker API. The proto models are auto-generated at compile time using this ProtoGen library. This custom library adds messageInput and messageOutput fields to all proto tasks and task definitions. The goal of these fields is providing a type-safe way to pass input and input metadata through tasks that use the gRPC API. These fields use the Any protobuf type which can store any arbitrary message type in a type-safe way, without the server needing to know the exact serialization format of the message. In order to expose these Any objects in the REST API, a custom encoding is used that contains the raw data of the serialized message by converting it into a dictionary with '@type' and '@value' keys, where '@type' is identical to the canonical representation and '@value' contains a base64 encoded string with the binary data of the serialized message. The JsonMapperProvider provides the object mapper initialized with this module to enable serialization/deserialization of these JSON objects. ### Cassandra Persistence The Cassandra persistence layer currently provides a partial implementation of the ExecutionDAO that supports all the CRUD operations for tasks and workflow execution. The data modelling is done in a denormalized manner and stored in two tables. The “workflows” table houses all the information for a workflow execution including all its tasks and is the source of truth for all the information regarding a workflow and its tasks. The “task_lookup” table, as the name suggests stores a lookup of taskIds to workflowId. This table facilitates the fast retrieval of task data given a taskId. All the datastore operations that are used during the critical execution path of a workflow have been implemented currently. Few of the operational abilities of the ExecutionDAO are yet to be implemented. This module also does not provide implementations for QueueDAO, PollDataDAO and RateLimitingDAO. We envision using the Cassandra DAO with an external queue implementation, since implementing a queuing recipe on top of Cassandra is an anti-pattern that we want to stay away from. ### External Payload Storage The implementation of this feature is such that the externalization of payloads is fully transparent and automated to the user. Conductor operators can configure the usage of this feature and is completely abstracted and hidden from the user, thereby allowing the operators full control over the barrier limits. Currently, only AWS S3 is supported as a storage system, however, as with all other Conductor components, this is pluggable and can be extended to enable any other object store to be used as an external payload storage system. The externalization of payloads is enforced using two kinds of [barriers](/externalpayloadstorage.html). Soft barriers are used when the payload size is warranted enough to be stored as part of workflow execution. These payloads will be stored in external storage and used during execution. Hard barriers are enforced to safeguard against voluminous data, and such payloads are rejected and the workflow execution is failed. The payload size is evaluated in the client before being sent over the wire to the server. If the payload size exceeds the configured soft limit, the client makes a request to the server for the location at which the payload is to be stored. In this case where S3 is being used, the server returns a signed url for the location and the client uploads the payload using this signed url. The relative path to the payload object is then stored in the workflow/task metadata. The server can then download this payload from this path and use as needed during execution. This allows the server to control access to the S3 bucket, thereby making the user applications where the worker processes are run completely agnostic of the permissions needed to access this location. ### Dynamic Workflow Executions In the earlier version (v1.x), Conductor allowed the execution of workflows referencing the workflow and task definitions stored as metadata in the system. This meant that a workflow execution with 10 custom tasks to run entailed: - Registration of the 10 task definitions if they don't exist (assuming workflow task type SIMPLE for simplicity) - Registration of the workflow definition - Each time a definition needs to be retrieved, a call to the metadata store needed to be performed - In addition to that, the system allowed current metadata that is in use to be altered, leading to possible inconsistencies/race conditions To eliminate these pain points, the execution was changed such that the workflow definition is embedded within the workflow execution and the task definitions are themselves embedded within this workflow definition. This enables the concept of ephemeral/dynamic workflows and tasks. Instead of fetching metadata definitions throughout the execution, the definitions are fetched and embedded into the execution at the start of the workflow execution. This also enabled the StartWorkflowRequest to be extended to provide the complete workflow definition that will be used during execution, thus removing the need for pre-registration. The MetadataMapperService prefetches the workflow and task definitions and embeds these within the workflow data, if not provided in the StartWorkflowRequest. Following benefits are seen as a result of these changes: - Grants immutability of the definition stored within the execution data against modifications to the metadata store - Better testability of workflows with faster experimental changes to definitions - Reduced stress on the datastore due to prefetching the metadata only once at the start ### Decoupling Elasticsearch from Persistence In the earlier version (1.x), the indexing logic was imbibed within the persistence layer, thus creating a tight coupling between the primary datastore and the indexing engine. This meant that the primary datastore determines how we orchestrate between the storage (redis, mysql, etc) and the indexer(elastic search). The main disadvantage of this approach is the lack of flexibility, that is, we cannot run an in-memory database and external elastic search or vice-versa. We plan to improve this further by removing the indexing from the critical path of workflow execution, thus reducing possible points of failure during execution. ### Elasticsearch 5/6 Support Indexing workflow execution is one of the primary features of Conductor. This enables archival of terminal state workflows from the primary data store, along with providing a clean search capability from the UI. In Conductor 1.x, we supported both versions 2 and 5 of Elasticsearch by shadowing version 5 and all its dependencies. This proved to be rather tedious increasing build times by over 10 minutes. In Conductor 2.x, we have removed active support for ES 2.x, because of valuable community contributions for elasticsearch 5 and elasticsearch 6 modules. Unlike Conductor 1.x, Conductor 2.x supports elasticsearch 5 by default, which can easily be replaced with version 6 by following the simple instructions [here](https://github.com/Netflix/conductor/tree/master/es6-persistence#build). ### Maintaining workflow consistency with distributed locking and fencing tokens #### Problem Conductor’s Workflow decide is the core logic which recursively evaluates the state of the workflow, schedules tasks, persists workflow and task(s) state at several checkpoints, and progresses the workflow. In a multi-node Conductor server deployment, the decide on a workflow can be triggered concurrently. For example, the worker can update Conductor server with latest task state, which calls decide, while the sweeper service (which periodically evaluates the workflow state to progress from task timeouts) would also call the decide on a different instance. The decide can be run concurrently in two different jvm nodes with two different workflow states, and based on the workflow configuration and current state, the result could be inconsistent. #### A two-part solution to maintain Workflow Consistency **Preventing concurrent decides with distributed locking:** The goal is to allow only one decide to run on a workflow at any given time across the whole Conductor Server cluster. This can be achieved by plugging in distributed locking implementations like Zookeeper, Redlock etc. A Zookeeper module implementing Conductor’s Locking service is provided. **Preventing stale data updates with fencing tokens:** While the locking service helps to run one decide at a time, it might still be possible for nodes with timed out locks to reactivate and continue execution from where it left off (usually with stale data). This can be avoided with fencing tokens, which basically is an incrementing counter on workflow state with read-before-write support in a transaction or similar construct. *At Netflix, we use Cassandra. Considering the tradeoffs of Cassandra’s Lightweight Transactions (LWT) and the probability of this stale updates happening, and our testing results, we’ve decided to first only rollout distributed locking with Zookeeper. We'll monitor our system and add C* LWT if needed. #### Setting up desired level of consistency Based on your requirements, it is possible to use none, one or both of the distributed locking and fencing tokens implementations. #### Alternative solution to distributed "decide" evaluation As mentioned in the previous section, the "decide" logic is triggered from multiple places in a conductor instance. Either a direct trigger such as user starting a workflow or a timed trigger from the Sweeper service. > Sweeper service is responsible for continually checking state of all workflows executions and trigger the "decide" logic which in turn can time the workflow out. In a single node deployment (single dynomite rack and single conductor server) this shouldn't be a problem. But when running multiple replicated dynomite racks and a conductor server on top of each rack, this might trigger the race condition described in previous section. > Dynomite rack is a single or multiple instance dynomite setup that holds all the data. > More on dynomite HA setup: (https://netflixtechblog.com/introducing-dynomite-making-non-distributed-databases-distributed-c7bce3d89404) In a cluster deployment, the default behavior for Dyno Queues is such, that it distributes the workload (round-robin style) to all the conductor servers. This can create a situation where the first task to be executed is queued for conductor server #1 but the sweeper service is queued for conductor server #2. ##### More on dyno queues Dyno queues are the default queuing mechanism of conductor. Queues are allocated and used for: * Task execution - each task type gets a queue * Workflow execution - single queue with all currently executing workflows (deciderQueue) * This queue is used by SweeperService **Each conductor server instance gets its own set of queues**. Or more precisely a queue shard of its own. This means that if you have 2 task types, you end up with 6 queues altogether e.g. ``` conductor_queues.test.QUEUE._deciderQueue.c conductor_queues.test.QUEUE._deciderQueue.d conductor_queues.test.QUEUE.HTTP.c conductor_queues.test.QUEUE.HTTP.d conductor_queues.test.QUEUE.LAMBDA.c conductor_queues.test.QUEUE.LAMBDA.d ``` > The "c" and "d" suffixes are the shards identifying conductor server instace #1 and instance #2 respectively. > The shard names are extracted from dynomite rack name such as us-east-1c that is set in "LOCAL_RACK" or "EC2_AVAILABILTY_ZONE" Considering an execution of a simple workflow with just 2 tasks: [HTTP, LAMBDA], you should end up with queues being filled as follows: ``` Workflow execution -> conductor_queues.test.QUEUE._deciderQueue.c HTTP taks execution -> conductor_queues.test.QUEUE.HTTP.d LAMBDA task execution -> conductor_queues.test.QUEUE.LAMBDA.c ``` Which means that SweeperService in conductor instance #1 is responsible for sweeping the workflow, conductor #2 is responsible for executing HTTP task and conductor #1 again responsible for executing LAMBDA task. This illustrates the race condition: If the HTTP task completion in instance #2 happens at the same time as sweep in instance #1 ... you can end up with 2 different updates to a workflow execution: one update timing workflow out while the other completing the task and scheduling next. > The round-robin strategy responsible for work distribution is defined [here](https://github.com/Netflix/dyno-queues/blob/1cde55bbb69acd631c671a0cb2f9db2419163e33/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java) ##### Back to alternative solution The alternative solution here is **Switching round-robin queue allocation for a local-only strategy**. Meaning that a workflow and its task executions are queued only for the conductor instance which started the workflow. This completely avoids the race condition for the price of removing task execution distribution. Since all tasks and the sweeper service read/write only from/to "local" queues, it is impossible to run into a race condition between conductor instances. The downside here is that the workload is not distributed across all conductor servers. Which might be an advantage in active-standby deployments. Considering other downsides ... Considering a situation where a conductor instance goes down: * With local-only strategy, the workflow executions from failed conductor instance will not progress until: * The conductor instance is restarted or * The executions are manually terminated and restarted from a different node * With round-robin strategy, there is a chance the tasks will be rescheduled on a different conductor node * This is nondeterministic though **Enabling local only queue allocation strategy for dyno queues:** Just enable following setting the config.properties: ``` workflow.dyno.queue.sharding.strategy=localOnly ``` > The default is roundRobin ================================================ FILE: docs/kitchensink.json ================================================ { "name": "kitchensink", "description": "kitchensink workflow", "version": 1, "tasks": [ { "name": "search_elasticsearch", "taskReferenceName": "get_es_0", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/workflow/_search?q=status:COMPLETED&size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE" }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${task_2.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_4", "taskReferenceName": "task_4", "inputParameters": { "mod": "${task_2.output.mod}", "oddEven": "${task_2.output.oddEven}" }, "type": "SIMPLE" }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${task_4.output.dynamicTasks}", "input": "${task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input" }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN" } ], "1": [ { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_10", "taskReferenceName": "task_10", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], [ { "name": "task_11", "taskReferenceName": "task_11", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ] ] }, { "name": "join", "taskReferenceName": "join2", "type": "JOIN", "joinOn": [ "wf3", "wf4" ] } ] } }, { "name": "search_elasticsearch", "taskReferenceName": "get_es_1", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/workflow/_search?q=status:COMPLETED&size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_30", "taskReferenceName": "task_30", "inputParameters": { "statuses": "${get_es_1.output...status}", "fistWorkflowId": "${get_es_1.output.workflowId[0]}" }, "type": "SIMPLE" } ], "schemaVersion": 2 } ================================================ FILE: docs/mkdocs.yml ================================================ site_name: Conductor Documentation site_description: Conductor is a platform created by Netflix to orchestrate workflows that span across microservices. repo_url: https://github.com/Netflix/conductor edit_uri: '' strict: true use_directory_urls: false nav: - Getting Started: - Running Conductor: - From Source: gettingstarted/source.md - Using Docker: gettingstarted/docker.md - Hosted Solutions: gettingstarted/hosted.md - Basic Concepts: gettingstarted/basicconcepts.md - High Level Steps: gettingstarted/steps.md - Using the Client: gettingstarted/client.md - Start a Workflow: gettingstarted/startworkflow.md - Why Conductor?: gettingstarted/intro.md - How-Tos: - Workflows: - Debugging Workflows: how-tos/Workflows/debugging-workflows.md - Handling Errors: how-tos/Workflows/handling-errors.md - Searching Workflows: how-tos/Workflows/searching-workflows.md - Starting Workflows: how-tos/Workflows/starting-workflows.md - Updating Workflows: how-tos/Workflows/updating-workflows.md - View Workflow Execution: how-tos/Workflows/view-workflow-executions.md - Versioning Workflows: how-tos/Workflows/versioning-workflows.md - Tasks: - Creating Task Definitions: how-tos/Tasks/creating-tasks.md - Dynamic vs Switch Tasks: how-tos/Tasks/dynamic-vs-switch-tasks.md - Monitoring Task Queues: how-tos/Tasks/monitoring-task-queues.md - Reusing Tasks: how-tos/Tasks/reusing-tasks.md - Task Configurations: how-tos/Tasks/task-configurations.md - Task Inputs: how-tos/Tasks/task-inputs.md - Task Timeouts: how-tos/Tasks/task-timeouts.md - Updating Task Definitions: how-tos/Tasks/updating-tasks.md - Extending System Tasks: how-tos/Tasks/extending-system-tasks.md - Workers: - Build a Go Task Worker: how-tos/Workers/build-a-golang-task-worker.md - Build a Java Task Worker: how-tos/Workers/build-a-java-task-worker.md - Build a Python Task Worker: how-tos/Workers/build-a-python-task-worker.md - Monitoring: - Conductor Log Level: how-tos/Monitoring/Conductor-LogLevel.md - Developer Labs: - Beginner: labs/beginner.md - A First Workflow: labs/running-first-workflow.md - Events and Event Handlers: labs/eventhandlers.md - Kitchen Sink: labs/kitchensink.md - Documentation: - Architecture: - Overview: architecture/overview.md - Task Lifecycle: architecture/tasklifecycle.md - API Specification: apispec.md - Configuration: - Task Definition: configuration/taskdef.md - Worker Definition: configuration/workerdef.md - Workflow Definition: configuration/workflowdef.md - System Tasks: configuration/systask.md - System Operators: configuration/sysoperator.md - Event Handlers: configuration/eventhandlers.md - Task Domains: configuration/taskdomains.md - Isolation Groups: configuration/isolationgroups.md - Operators: - Do-While: reference-docs/do-while-task.md - Dynamic: reference-docs/dynamic-task.md - Dynamic Fork: reference-docs/dynamic-fork-task.md - Fork: reference-docs/fork-task.md - Join: reference-docs/join-task.md - Set Variable: reference-docs/set-variable-task.md - Start Workflow: reference-docs/start-workflow-task.md - Sub Workflow: reference-docs/sub-workflow-task.md - Switch: reference-docs/switch-task.md - Terminate: reference-docs/terminate-task.md - Wait: reference-docs/wait-task.md - System Tasks: - Event Task: reference-docs/event-task.md - HTTP Task: reference-docs/http-task.md - Human Task: reference-docs/human-task.md - Inline Task: reference-docs/inline-task.md - JSON JQ Transform Task: reference-docs/json-jq-transform-task.md - Kafka Publish Task: reference-docs/kafka-publish-task.md - Conductor Metrics: - Server Metrics: metrics/server.md - Client Metrics: metrics/client.md - Advanced Topics: - Extending Conductor: extend.md - Annotation Processor: reference-docs/annotation-processor.md - Archival of Workflows: reference-docs/archival-of-workflows.md - Azure Blob Storage: reference-docs/azureblob-storage.md - External Payload Storage: externalpayloadstorage.md - Redis: reference-docs/redis.md - SDKs: - CSharp SDK: how-tos/csharp-sdk.md - Clojure SDK: how-tos/clojure-sdk.md - Go SDK: how-tos/go-sdk.md - Python SDK: how-tos/python-sdk.md - Best Practices: bestpractices.md - FAQ: faq.md - Directed Acyclic Graph: reference-docs/directed-acyclic-graph.md - Technical Details: technicaldetails.md - Resources: - Contributing: resources/contributing.md - Code of Conduct: resources/code-of-conduct.md - Related Projects: resources/related.md - License: resources/license.md theme: name: mkdocs custom_dir: theme/ extra_css: - css/custom.css markdown_extensions: - admonition - codehilite ================================================ FILE: docs/theme/main.html ================================================ {% extends "base.html" %} {% block extrahead %} {% endblock %} {% block content %} {% if page and page.is_homepage %}
    {% include "content.html" %}
    {% else %}
    {% include "toc.html" %}
    {% include "content.html" %}
    {% endif %} {% endblock %} {%- block next_prev %} {%- endblock %} {% block footer %}
    Netflix Engineering
    {% endblock %} ================================================ FILE: docs/theme/toc-sub.html ================================================ {%- if not nav_item.children %}
  • {{ nav_item.title }}
    • {%- else %}
  • {{ nav_item.title }}
      {%- for nav_item in nav_item.children %} {% include "toc-sub.html" %} {%- endfor %}
    • {%- endif %} ================================================ FILE: docs/theme/toc.html ================================================
      {%- for nav_item in nav %} {% include "toc-sub.html" %} {%- endfor %}
    ================================================ FILE: es6-persistence/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.retry:spring-retry' implementation "commons-io:commons-io:${revCommonsIo}" implementation "org.apache.commons:commons-lang3" // SBMTODO: remove guava dep implementation "com.google.guava:guava:${revGuava}" implementation "org.elasticsearch.client:transport" implementation "org.elasticsearch.client:elasticsearch-rest-client" implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client" testImplementation 'org.springframework.retry:spring-retry' testImplementation "org.awaitility:awaitility:${revAwaitility}" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation project(':conductor-common').sourceSets.test.output } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchConditions.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.config; import org.springframework.boot.autoconfigure.condition.AllNestedConditions; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; public class ElasticSearchConditions { private ElasticSearchConditions() {} public static class ElasticSearchV6Enabled extends AllNestedConditions { ElasticSearchV6Enabled() { super(ConfigurationPhase.PARSE_CONFIGURATION); } @SuppressWarnings("unused") @ConditionalOnProperty( name = "conductor.indexing.enabled", havingValue = "true", matchIfMissing = true) static class enabledIndexing {} @SuppressWarnings("unused") @ConditionalOnProperty( name = "conductor.elasticsearch.version", havingValue = "6", matchIfMissing = true) static class enabledES6 {} } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchProperties.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.config; import java.net.MalformedURLException; import java.net.URL; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; @ConfigurationProperties("conductor.elasticsearch") public class ElasticSearchProperties { /** * The comma separated list of urls for the elasticsearch cluster. Format -- * host1:port1,host2:port2 */ private String url = "localhost:9300"; /** The index prefix to be used when creating indices */ private String indexPrefix = "conductor"; /** The color of the elasticserach cluster to wait for to confirm healthy status */ private String clusterHealthColor = "green"; /** The size of the batch to be used for bulk indexing in async mode */ private int indexBatchSize = 1; /** The size of the queue used for holding async indexing tasks */ private int asyncWorkerQueueSize = 100; /** The maximum number of threads allowed in the async pool */ private int asyncMaxPoolSize = 12; /** * The time in seconds after which the async buffers will be flushed (if no activity) to prevent * data loss */ @DurationUnit(ChronoUnit.SECONDS) private Duration asyncBufferFlushTimeout = Duration.ofSeconds(10); /** The number of shards that the index will be created with */ private int indexShardCount = 5; /** The number of replicas that the index will be configured to have */ private int indexReplicasCount = 1; /** The number of task log results that will be returned in the response */ private int taskLogResultLimit = 10; /** The timeout in milliseconds used when requesting a connection from the connection manager */ private int restClientConnectionRequestTimeout = -1; /** Used to control if index management is to be enabled or will be controlled externally */ private boolean autoIndexManagementEnabled = true; /** * Document types are deprecated in ES6 and removed from ES7. This property can be used to * disable the use of specific document types with an override. This property is currently used * in ES6 module. * *

    Note that this property will only take effect if {@link * ElasticSearchProperties#isAutoIndexManagementEnabled} is set to false and index management is * handled outside of this module. */ private String documentTypeOverride = ""; /** Elasticsearch basic auth username */ private String username; /** Elasticsearch basic auth password */ private String password; public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getIndexPrefix() { return indexPrefix; } public void setIndexPrefix(String indexPrefix) { this.indexPrefix = indexPrefix; } public String getClusterHealthColor() { return clusterHealthColor; } public void setClusterHealthColor(String clusterHealthColor) { this.clusterHealthColor = clusterHealthColor; } public int getIndexBatchSize() { return indexBatchSize; } public void setIndexBatchSize(int indexBatchSize) { this.indexBatchSize = indexBatchSize; } public int getAsyncWorkerQueueSize() { return asyncWorkerQueueSize; } public void setAsyncWorkerQueueSize(int asyncWorkerQueueSize) { this.asyncWorkerQueueSize = asyncWorkerQueueSize; } public int getAsyncMaxPoolSize() { return asyncMaxPoolSize; } public void setAsyncMaxPoolSize(int asyncMaxPoolSize) { this.asyncMaxPoolSize = asyncMaxPoolSize; } public Duration getAsyncBufferFlushTimeout() { return asyncBufferFlushTimeout; } public void setAsyncBufferFlushTimeout(Duration asyncBufferFlushTimeout) { this.asyncBufferFlushTimeout = asyncBufferFlushTimeout; } public int getIndexShardCount() { return indexShardCount; } public void setIndexShardCount(int indexShardCount) { this.indexShardCount = indexShardCount; } public int getIndexReplicasCount() { return indexReplicasCount; } public void setIndexReplicasCount(int indexReplicasCount) { this.indexReplicasCount = indexReplicasCount; } public int getTaskLogResultLimit() { return taskLogResultLimit; } public void setTaskLogResultLimit(int taskLogResultLimit) { this.taskLogResultLimit = taskLogResultLimit; } public int getRestClientConnectionRequestTimeout() { return restClientConnectionRequestTimeout; } public void setRestClientConnectionRequestTimeout(int restClientConnectionRequestTimeout) { this.restClientConnectionRequestTimeout = restClientConnectionRequestTimeout; } public boolean isAutoIndexManagementEnabled() { return autoIndexManagementEnabled; } public void setAutoIndexManagementEnabled(boolean autoIndexManagementEnabled) { this.autoIndexManagementEnabled = autoIndexManagementEnabled; } public String getDocumentTypeOverride() { return documentTypeOverride; } public void setDocumentTypeOverride(String documentTypeOverride) { this.documentTypeOverride = documentTypeOverride; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } public List toURLs() { String clusterAddress = getUrl(); String[] hosts = clusterAddress.split(","); return Arrays.stream(hosts) .map( host -> (host.startsWith("http://") || host.startsWith("https://") || host.startsWith("tcp://")) ? toURL(host) : toURL("tcp://" + host)) .collect(Collectors.toList()); } private URL toURL(String url) { try { return new URL(url); } catch (MalformedURLException e) { throw new IllegalArgumentException(url + "can not be converted to java.net.URL"); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchV6Configuration.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.config; import java.net.InetAddress; import java.net.URI; import java.net.URL; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.Client; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.retry.backoff.FixedBackOffPolicy; import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es6.dao.index.ElasticSearchDAOV6; import com.netflix.conductor.es6.dao.index.ElasticSearchRestDAOV6; import com.fasterxml.jackson.databind.ObjectMapper; @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(ElasticSearchProperties.class) @Conditional(ElasticSearchConditions.ElasticSearchV6Enabled.class) public class ElasticSearchV6Configuration { private static final Logger log = LoggerFactory.getLogger(ElasticSearchV6Configuration.class); @Bean @Conditional(IsTcpProtocol.class) public Client client(ElasticSearchProperties properties) { Settings settings = Settings.builder() .put("client.transport.ignore_cluster_name", true) .put("client.transport.sniff", true) .build(); TransportClient transportClient = new PreBuiltTransportClient(settings); List clusterAddresses = getURIs(properties); if (clusterAddresses.isEmpty()) { log.warn("workflow.elasticsearch.url is not set. Indexing will remain DISABLED."); } for (URI hostAddress : clusterAddresses) { int port = Optional.ofNullable(hostAddress.getPort()).orElse(9200); try { transportClient.addTransportAddress( new TransportAddress(InetAddress.getByName(hostAddress.getHost()), port)); } catch (Exception e) { throw new RuntimeException("Invalid host" + hostAddress.getHost(), e); } } return transportClient; } @Bean @Conditional(IsHttpProtocol.class) public RestClient restClient(ElasticSearchProperties properties) { RestClientBuilder restClientBuilder = RestClient.builder(convertToHttpHosts(properties.toURLs())); if (properties.getRestClientConnectionRequestTimeout() > 0) { restClientBuilder.setRequestConfigCallback( requestConfigBuilder -> requestConfigBuilder.setConnectionRequestTimeout( properties.getRestClientConnectionRequestTimeout())); } return restClientBuilder.build(); } @Bean @Conditional(IsHttpProtocol.class) public RestClientBuilder restClientBuilder(ElasticSearchProperties properties) { RestClientBuilder builder = RestClient.builder(convertToHttpHosts(properties.toURLs())); if (properties.getUsername() != null && properties.getPassword() != null) { log.info( "Configure ElasticSearch with BASIC authentication. User:{}", properties.getUsername()); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials( AuthScope.ANY, new UsernamePasswordCredentials( properties.getUsername(), properties.getPassword())); builder.setHttpClientConfigCallback( httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } else { log.info("Configure ElasticSearch with no authentication."); } return builder; } @Bean @Conditional(IsHttpProtocol.class) public IndexDAO es6IndexRestDAO( RestClientBuilder restClientBuilder, ElasticSearchProperties properties, @Qualifier("es6RetryTemplate") RetryTemplate retryTemplate, ObjectMapper objectMapper) { return new ElasticSearchRestDAOV6( restClientBuilder, retryTemplate, properties, objectMapper); } @Bean @Conditional(IsTcpProtocol.class) public IndexDAO es6IndexDAO( Client client, @Qualifier("es6RetryTemplate") RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { return new ElasticSearchDAOV6(client, retryTemplate, properties, objectMapper); } @Bean public RetryTemplate es6RetryTemplate() { RetryTemplate retryTemplate = new RetryTemplate(); FixedBackOffPolicy fixedBackOffPolicy = new FixedBackOffPolicy(); fixedBackOffPolicy.setBackOffPeriod(1000L); retryTemplate.setBackOffPolicy(fixedBackOffPolicy); return retryTemplate; } private HttpHost[] convertToHttpHosts(List hosts) { return hosts.stream() .map(host -> new HttpHost(host.getHost(), host.getPort(), host.getProtocol())) .toArray(HttpHost[]::new); } public List getURIs(ElasticSearchProperties properties) { String clusterAddress = properties.getUrl(); String[] hosts = clusterAddress.split(","); return Arrays.stream(hosts) .map( host -> (host.startsWith("http://") || host.startsWith("https://") || host.startsWith("tcp://")) ? URI.create(host) : URI.create("tcp://" + host)) .collect(Collectors.toList()); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/config/IsHttpProtocol.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.config; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; import org.springframework.context.annotation.Configuration; import org.springframework.core.type.AnnotatedTypeMetadata; @EnableConfigurationProperties(ElasticSearchProperties.class) @Configuration public class IsHttpProtocol implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { String url = context.getEnvironment().getProperty("conductor.elasticsearch.url"); if (url.startsWith("http") || url.startsWith("https")) { return true; } return false; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/config/IsTcpProtocol.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.config; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; import org.springframework.context.annotation.Configuration; import org.springframework.core.type.AnnotatedTypeMetadata; @EnableConfigurationProperties(ElasticSearchProperties.class) @Configuration public class IsTcpProtocol implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { String url = context.getEnvironment().getProperty("conductor.elasticsearch.url"); if (url.startsWith("http") || url.startsWith("https")) { return false; } return true; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/BulkRequestBuilderWrapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.util.Objects; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.springframework.lang.NonNull; /** Thread-safe wrapper for {@link BulkRequestBuilder}. */ public class BulkRequestBuilderWrapper { private final BulkRequestBuilder bulkRequestBuilder; public BulkRequestBuilderWrapper(@NonNull BulkRequestBuilder bulkRequestBuilder) { this.bulkRequestBuilder = Objects.requireNonNull(bulkRequestBuilder); } public void add(@NonNull UpdateRequest req) { synchronized (bulkRequestBuilder) { bulkRequestBuilder.add(Objects.requireNonNull(req)); } } public void add(@NonNull IndexRequest req) { synchronized (bulkRequestBuilder) { bulkRequestBuilder.add(Objects.requireNonNull(req)); } } public int numberOfActions() { synchronized (bulkRequestBuilder) { return bulkRequestBuilder.numberOfActions(); } } public ActionFuture execute() { synchronized (bulkRequestBuilder) { return bulkRequestBuilder.execute(); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/BulkRequestWrapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.util.Objects; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.springframework.lang.NonNull; /** Thread-safe wrapper for {@link BulkRequest}. */ class BulkRequestWrapper { private final BulkRequest bulkRequest; BulkRequestWrapper(@NonNull BulkRequest bulkRequest) { this.bulkRequest = Objects.requireNonNull(bulkRequest); } public void add(@NonNull UpdateRequest req) { synchronized (bulkRequest) { bulkRequest.add(Objects.requireNonNull(req)); } } public void add(@NonNull IndexRequest req) { synchronized (bulkRequest) { bulkRequest.add(Objects.requireNonNull(req)); } } BulkRequest get() { return bulkRequest; } int numberOfActions() { synchronized (bulkRequest) { return bulkRequest.numberOfActions(); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchBaseDAO.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es6.dao.query.parser.Expression; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; abstract class ElasticSearchBaseDAO implements IndexDAO { String indexPrefix; String loadTypeMappingSource(String path) throws IOException { return applyIndexPrefixToTemplate( IOUtils.toString(ElasticSearchBaseDAO.class.getResourceAsStream(path))); } private String applyIndexPrefixToTemplate(String text) { String pattern = "\"template\": \"\\*(.*)\\*\""; Pattern r = Pattern.compile(pattern); Matcher m = r.matcher(text); StringBuilder sb = new StringBuilder(); while (m.find()) { m.appendReplacement( sb, m.group(0) .replaceFirst( Pattern.quote(m.group(1)), indexPrefix + "_" + m.group(1))); } m.appendTail(sb); return sb.toString(); } BoolQueryBuilder boolQueryBuilder(String expression, String queryString) throws ParserException { QueryBuilder queryBuilder = QueryBuilders.matchAllQuery(); if (StringUtils.isNotEmpty(expression)) { Expression exp = Expression.fromString(expression); queryBuilder = exp.getFilterBuilder(); } BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder); QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery(queryString); return QueryBuilders.boolQuery().must(stringQuery).must(filterQuery); } protected String getIndexName(String documentType) { return indexPrefix + "_" + documentType; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchDAOV6.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.io.IOException; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.LocalDate; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es6.config.ElasticSearchProperties; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; import com.netflix.conductor.metrics.Monitors; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.MapType; import com.fasterxml.jackson.databind.type.TypeFactory; @Trace public class ElasticSearchDAOV6 extends ElasticSearchBaseDAO implements IndexDAO { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchDAOV6.class); private static final String WORKFLOW_DOC_TYPE = "workflow"; private static final String TASK_DOC_TYPE = "task"; private static final String LOG_DOC_TYPE = "task_log"; private static final String EVENT_DOC_TYPE = "event"; private static final String MSG_DOC_TYPE = "message"; private static final int CORE_POOL_SIZE = 6; private static final long KEEP_ALIVE_TIME = 1L; private static final int UPDATE_REQUEST_RETRY_COUNT = 5; private static final String CLASS_NAME = ElasticSearchDAOV6.class.getSimpleName(); private final String workflowIndexName; private final String taskIndexName; private final String eventIndexPrefix; private String eventIndexName; private final String messageIndexPrefix; private String messageIndexName; private String logIndexName; private final String logIndexPrefix; private final String docTypeOverride; private final ObjectMapper objectMapper; private final Client elasticSearchClient; private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); private static final SimpleDateFormat SIMPLE_DATE_FORMAT = new SimpleDateFormat("yyyyMMWW"); private final ExecutorService executorService; private final ExecutorService logExecutorService; private final ConcurrentHashMap bulkRequests; private final int indexBatchSize; private final long asyncBufferFlushTimeout; private final ElasticSearchProperties properties; private final RetryTemplate retryTemplate; static { SIMPLE_DATE_FORMAT.setTimeZone(GMT); } public ElasticSearchDAOV6( Client elasticSearchClient, RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { this.objectMapper = objectMapper; this.elasticSearchClient = elasticSearchClient; this.indexPrefix = properties.getIndexPrefix(); this.workflowIndexName = getIndexName(WORKFLOW_DOC_TYPE); this.taskIndexName = getIndexName(TASK_DOC_TYPE); this.logIndexPrefix = this.indexPrefix + "_" + LOG_DOC_TYPE; this.messageIndexPrefix = this.indexPrefix + "_" + MSG_DOC_TYPE; this.eventIndexPrefix = this.indexPrefix + "_" + EVENT_DOC_TYPE; int workerQueueSize = properties.getAsyncWorkerQueueSize(); int maximumPoolSize = properties.getAsyncMaxPoolSize(); this.bulkRequests = new ConcurrentHashMap<>(); this.indexBatchSize = properties.getIndexBatchSize(); this.asyncBufferFlushTimeout = properties.getAsyncBufferFlushTimeout().toMillis(); this.properties = properties; if (!properties.isAutoIndexManagementEnabled() && StringUtils.isNotBlank(properties.getDocumentTypeOverride())) { docTypeOverride = properties.getDocumentTypeOverride(); } else { docTypeOverride = ""; } this.executorService = new ThreadPoolExecutor( CORE_POOL_SIZE, maximumPoolSize, KEEP_ALIVE_TIME, TimeUnit.MINUTES, new LinkedBlockingQueue<>(workerQueueSize), (runnable, executor) -> { LOGGER.warn( "Request {} to async dao discarded in executor {}", runnable, executor); Monitors.recordDiscardedIndexingCount("indexQueue"); }); int corePoolSize = 1; maximumPoolSize = 2; long keepAliveTime = 30L; this.logExecutorService = new ThreadPoolExecutor( corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<>(workerQueueSize), (runnable, executor) -> { LOGGER.warn( "Request {} to async log dao discarded in executor {}", runnable, executor); Monitors.recordDiscardedIndexingCount("logQueue"); }); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS); this.retryTemplate = retryTemplate; } @PreDestroy private void shutdown() { LOGGER.info("Starting graceful shutdown of executor service"); shutdownExecutorService(logExecutorService); shutdownExecutorService(executorService); } private void shutdownExecutorService(ExecutorService execService) { try { execService.shutdown(); if (execService.awaitTermination(30, TimeUnit.SECONDS)) { LOGGER.debug("tasks completed, shutting down"); } else { LOGGER.warn("Forcing shutdown after waiting for 30 seconds"); execService.shutdownNow(); } } catch (InterruptedException ie) { LOGGER.warn( "Shutdown interrupted, invoking shutdownNow on scheduledThreadPoolExecutor for delay queue"); execService.shutdownNow(); Thread.currentThread().interrupt(); } } @Override @PostConstruct public void setup() throws Exception { waitForHealthyCluster(); if (properties.isAutoIndexManagementEnabled()) { createIndexesTemplates(); createWorkflowIndex(); createTaskIndex(); } } private void waitForHealthyCluster() throws Exception { elasticSearchClient .admin() .cluster() .prepareHealth() .setWaitForGreenStatus() .execute() .get(); } /** Initializes the indexes templates task_log, message and event, and mappings. */ private void createIndexesTemplates() { try { initIndexesTemplates(); updateIndexesNames(); Executors.newScheduledThreadPool(1) .scheduleAtFixedRate(this::updateIndexesNames, 0, 1, TimeUnit.HOURS); } catch (Exception e) { LOGGER.error("Error creating index templates", e); } } private void initIndexesTemplates() { initIndexTemplate(LOG_DOC_TYPE); initIndexTemplate(EVENT_DOC_TYPE); initIndexTemplate(MSG_DOC_TYPE); } private void initIndexTemplate(String type) { String template = "template_" + type; GetIndexTemplatesResponse result = elasticSearchClient .admin() .indices() .prepareGetTemplates(template) .execute() .actionGet(); if (result.getIndexTemplates().isEmpty()) { LOGGER.info("Creating the index template '{}'", template); try { String templateSource = loadTypeMappingSource("/" + template + ".json"); elasticSearchClient .admin() .indices() .preparePutTemplate(template) .setSource(templateSource.getBytes(), XContentType.JSON) .execute() .actionGet(); } catch (Exception e) { LOGGER.error("Failed to init " + template, e); } } } private void updateIndexesNames() { logIndexName = updateIndexName(LOG_DOC_TYPE); eventIndexName = updateIndexName(EVENT_DOC_TYPE); messageIndexName = updateIndexName(MSG_DOC_TYPE); } private String updateIndexName(String type) { String indexName = this.indexPrefix + "_" + type + "_" + SIMPLE_DATE_FORMAT.format(new Date()); createIndex(indexName); return indexName; } private void createWorkflowIndex() { createIndex(workflowIndexName); addTypeMapping(workflowIndexName, WORKFLOW_DOC_TYPE, "/mappings_docType_workflow.json"); } private void createTaskIndex() { createIndex(taskIndexName); addTypeMapping(taskIndexName, TASK_DOC_TYPE, "/mappings_docType_task.json"); } private void createIndex(String indexName) { try { elasticSearchClient .admin() .indices() .prepareGetIndex() .addIndices(indexName) .execute() .actionGet(); } catch (IndexNotFoundException infe) { try { CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); createIndexRequest.settings( Settings.builder() .put("index.number_of_shards", properties.getIndexShardCount()) .put( "index.number_of_replicas", properties.getIndexReplicasCount())); elasticSearchClient.admin().indices().create(createIndexRequest).actionGet(); } catch (ResourceAlreadyExistsException done) { LOGGER.error("Failed to update log index name: {}", indexName, done); } } } private void addTypeMapping(String indexName, String type, String sourcePath) { GetMappingsResponse getMappingsResponse = elasticSearchClient .admin() .indices() .prepareGetMappings(indexName) .addTypes(type) .execute() .actionGet(); if (getMappingsResponse.mappings().isEmpty()) { LOGGER.info("Adding the {} type mappings", indexName); try { String source = loadTypeMappingSource(sourcePath); elasticSearchClient .admin() .indices() .preparePutMapping(indexName) .setType(type) .setSource(source, XContentType.JSON) .execute() .actionGet(); } catch (Exception e) { LOGGER.error("Failed to init index " + indexName + " mappings", e); } } } @Override public void indexWorkflow(WorkflowSummary workflow) { try { long startTime = Instant.now().toEpochMilli(); String id = workflow.getWorkflowId(); byte[] doc = objectMapper.writeValueAsBytes(workflow); String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; UpdateRequest req = buildUpdateRequest(id, doc, workflowIndexName, docType); elasticSearchClient.update(req).actionGet(); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing workflow: {}", endTime - startTime, workflow.getWorkflowId()); Monitors.recordESIndexTime("index_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { Monitors.error(CLASS_NAME, "indexWorkflow"); LOGGER.error("Failed to index workflow: {}", workflow.getWorkflowId(), e); } } @Override public CompletableFuture asyncIndexWorkflow(WorkflowSummary workflow) { return CompletableFuture.runAsync(() -> indexWorkflow(workflow), executorService); } @Override public void indexTask(TaskSummary task) { try { long startTime = Instant.now().toEpochMilli(); String id = task.getTaskId(); byte[] doc = objectMapper.writeValueAsBytes(task); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; UpdateRequest req = new UpdateRequest(taskIndexName, docType, id); req.doc(doc, XContentType.JSON); req.upsert(doc, XContentType.JSON); indexObject(req, TASK_DOC_TYPE); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing task:{} in workflow: {}", endTime - startTime, task.getTaskId(), task.getWorkflowId()); Monitors.recordESIndexTime("index_task", TASK_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to index task: {}", task.getTaskId(), e); } } @Override public CompletableFuture asyncIndexTask(TaskSummary task) { return CompletableFuture.runAsync(() -> indexTask(task), executorService); } private void indexObject(UpdateRequest req, String docType) { if (bulkRequests.get(docType) == null) { bulkRequests.put( docType, new BulkRequests( System.currentTimeMillis(), elasticSearchClient.prepareBulk())); } bulkRequests.get(docType).getBulkRequestBuilder().add(req); if (bulkRequests.get(docType).getBulkRequestBuilder().numberOfActions() >= this.indexBatchSize) { indexBulkRequest(docType); } } private synchronized void indexBulkRequest(String docType) { if (bulkRequests.get(docType).getBulkRequestBuilder() != null && bulkRequests.get(docType).getBulkRequestBuilder().numberOfActions() > 0) { updateWithRetry(bulkRequests.get(docType).getBulkRequestBuilder(), docType); bulkRequests.put( docType, new BulkRequests( System.currentTimeMillis(), elasticSearchClient.prepareBulk())); } } @Override public void addTaskExecutionLogs(List taskExecLogs) { if (taskExecLogs.isEmpty()) { return; } try { long startTime = Instant.now().toEpochMilli(); BulkRequestBuilderWrapper bulkRequestBuilder = new BulkRequestBuilderWrapper(elasticSearchClient.prepareBulk()); for (TaskExecLog log : taskExecLogs) { String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride; IndexRequest request = new IndexRequest(logIndexName, docType); request.source(objectMapper.writeValueAsBytes(log), XContentType.JSON); bulkRequestBuilder.add(request); } bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS); long endTime = Instant.now().toEpochMilli(); LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime); Monitors.recordESIndexTime( "index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size()); } catch (Exception e) { List taskIds = taskExecLogs.stream().map(TaskExecLog::getTaskId).collect(Collectors.toList()); LOGGER.error("Failed to index task execution logs for tasks: {}", taskIds, e); } } @Override public CompletableFuture asyncAddTaskExecutionLogs(List logs) { return CompletableFuture.runAsync(() -> addTaskExecutionLogs(logs), logExecutorService); } @Override public List getTaskExecutionLogs(String taskId) { try { BoolQueryBuilder query = boolQueryBuilder("taskId='" + taskId + "'", "*"); String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride; final SearchRequestBuilder srb = elasticSearchClient .prepareSearch(logIndexPrefix + "*") .setQuery(query) .setTypes(docType) .setSize(properties.getTaskLogResultLimit()) .addSort(SortBuilders.fieldSort("createdTime").order(SortOrder.ASC)); return mapTaskExecLogsResponse(srb.execute().actionGet()); } catch (Exception e) { LOGGER.error("Failed to get task execution logs for task: {}", taskId, e); } return null; } private List mapTaskExecLogsResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); List logs = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); TaskExecLog tel = objectMapper.readValue(source, TaskExecLog.class); logs.add(tel); } return logs; } @Override public void addMessage(String queue, Message message) { try { long startTime = Instant.now().toEpochMilli(); Map doc = new HashMap<>(); doc.put("messageId", message.getId()); doc.put("payload", message.getPayload()); doc.put("queue", queue); doc.put("created", System.currentTimeMillis()); String docType = StringUtils.isBlank(docTypeOverride) ? MSG_DOC_TYPE : docTypeOverride; UpdateRequest req = new UpdateRequest(messageIndexName, docType, message.getId()); req.doc(doc, XContentType.JSON); req.upsert(doc, XContentType.JSON); indexObject(req, MSG_DOC_TYPE); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing message: {}", endTime - startTime, message.getId()); Monitors.recordESIndexTime("add_message", MSG_DOC_TYPE, endTime - startTime); } catch (Exception e) { LOGGER.error("Failed to index message: {}", message.getId(), e); } } @Override public CompletableFuture asyncAddMessage(String queue, Message message) { return CompletableFuture.runAsync(() -> addMessage(queue, message), executorService); } @Override public List getMessages(String queue) { try { BoolQueryBuilder fq = boolQueryBuilder("queue='" + queue + "'", "*"); String docType = StringUtils.isBlank(docTypeOverride) ? MSG_DOC_TYPE : docTypeOverride; final SearchRequestBuilder srb = elasticSearchClient .prepareSearch(messageIndexPrefix + "*") .setQuery(fq) .setTypes(docType) .addSort(SortBuilders.fieldSort("created").order(SortOrder.ASC)); return mapGetMessagesResponse(srb.execute().actionGet()); } catch (Exception e) { LOGGER.error("Failed to get messages for queue: {}", queue, e); } return null; } private List mapGetMessagesResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); TypeFactory factory = TypeFactory.defaultInstance(); MapType type = factory.constructMapType(HashMap.class, String.class, String.class); List messages = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); Map mapSource = objectMapper.readValue(source, type); Message msg = new Message(mapSource.get("messageId"), mapSource.get("payload"), null); messages.add(msg); } return messages; } @Override public void addEventExecution(EventExecution eventExecution) { try { long startTime = Instant.now().toEpochMilli(); byte[] doc = objectMapper.writeValueAsBytes(eventExecution); String id = eventExecution.getName() + "." + eventExecution.getEvent() + "." + eventExecution.getMessageId() + "." + eventExecution.getId(); String docType = StringUtils.isBlank(docTypeOverride) ? EVENT_DOC_TYPE : docTypeOverride; UpdateRequest req = buildUpdateRequest(id, doc, eventIndexName, docType); indexObject(req, EVENT_DOC_TYPE); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing event execution: {}", endTime - startTime, eventExecution.getId()); Monitors.recordESIndexTime("add_event_execution", EVENT_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to index event execution: {}", eventExecution.getId(), e); } } @Override public CompletableFuture asyncAddEventExecution(EventExecution eventExecution) { return CompletableFuture.runAsync( () -> addEventExecution(eventExecution), logExecutorService); } @Override public List getEventExecutions(String event) { try { BoolQueryBuilder fq = boolQueryBuilder("event='" + event + "'", "*"); String docType = StringUtils.isBlank(docTypeOverride) ? EVENT_DOC_TYPE : docTypeOverride; final SearchRequestBuilder srb = elasticSearchClient .prepareSearch(eventIndexPrefix + "*") .setQuery(fq) .setTypes(docType) .addSort(SortBuilders.fieldSort("created").order(SortOrder.ASC)); return mapEventExecutionsResponse(srb.execute().actionGet()); } catch (Exception e) { LOGGER.error("Failed to get executions for event: {}", event, e); } return null; } private List mapEventExecutionsResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); List executions = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); EventExecution tel = objectMapper.readValue(source, EventExecution.class); executions.add(tel); } return executions; } private void updateWithRetry(BulkRequestBuilderWrapper request, String docType) { try { long startTime = Instant.now().toEpochMilli(); retryTemplate.execute(context -> request.execute().actionGet(5, TimeUnit.SECONDS)); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing object of type: {}", endTime - startTime, docType); Monitors.recordESIndexTime("index_object", docType, endTime - startTime); } catch (Exception e) { Monitors.error(CLASS_NAME, "index"); LOGGER.error("Failed to index {} for requests", request.numberOfActions(), e); } } @Override public SearchResult searchWorkflows( String query, String freeText, int start, int count, List sort) { return search(query, start, count, sort, freeText, WORKFLOW_DOC_TYPE, true, String.class); } @Override public SearchResult searchWorkflowSummary( String query, String freeText, int start, int count, List sort) { return search( query, start, count, sort, freeText, WORKFLOW_DOC_TYPE, false, WorkflowSummary.class); } @Override public long getWorkflowCount(String query, String freeText) { return count(query, freeText, WORKFLOW_DOC_TYPE); } @Override public SearchResult searchTasks( String query, String freeText, int start, int count, List sort) { return search(query, start, count, sort, freeText, TASK_DOC_TYPE, true, String.class); } @Override public SearchResult searchTaskSummary( String query, String freeText, int start, int count, List sort) { return search(query, start, count, sort, freeText, TASK_DOC_TYPE, false, TaskSummary.class); } @Override public void removeWorkflow(String workflowId) { try { long startTime = Instant.now().toEpochMilli(); DeleteRequest request = new DeleteRequest(workflowIndexName, WORKFLOW_DOC_TYPE, workflowId); DeleteResponse response = elasticSearchClient.delete(request).actionGet(); if (response.getResult() == DocWriteResponse.Result.DELETED) { LOGGER.error("Index removal failed - document not found by id: {}", workflowId); } long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for removing workflow: {}", endTime - startTime, workflowId); Monitors.recordESIndexTime("remove_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Throwable e) { LOGGER.error("Failed to remove workflow {} from index", workflowId, e); Monitors.error(CLASS_NAME, "remove"); } } @Override public CompletableFuture asyncRemoveWorkflow(String workflowId) { return CompletableFuture.runAsync(() -> removeWorkflow(workflowId), executorService); } @Override public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) { if (keys.length != values.length) { throw new IllegalArgumentException("Number of keys and values do not match"); } long startTime = Instant.now().toEpochMilli(); UpdateRequest request = new UpdateRequest(workflowIndexName, WORKFLOW_DOC_TYPE, workflowInstanceId); Map source = IntStream.range(0, keys.length) .boxed() .collect(Collectors.toMap(i -> keys[i], i -> values[i])); request.doc(source); LOGGER.debug( "Updating workflow {} in elasticsearch index: {}", workflowInstanceId, workflowIndexName); elasticSearchClient.update(request).actionGet(); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId); Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } @Override public CompletableFuture asyncUpdateWorkflow( String workflowInstanceId, String[] keys, Object[] values) { return CompletableFuture.runAsync( () -> updateWorkflow(workflowInstanceId, keys, values), executorService); } @Override public void removeTask(String workflowId, String taskId) { try { long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; SearchResult taskSearchResult = searchTasks( String.format( "(taskId='%s') AND (workflowId='%s')", taskId, workflowId), "*", 0, 1, null); if (taskSearchResult.getTotalHits() == 0) { LOGGER.error("Task: {} does not belong to workflow: {}", taskId, workflowId); Monitors.error(CLASS_NAME, "removeTask"); return; } DeleteRequest request = new DeleteRequest(taskIndexName, docType, taskId); DeleteResponse response = elasticSearchClient.delete(request).actionGet(); long endTime = Instant.now().toEpochMilli(); if (response.getResult() != DocWriteResponse.Result.DELETED) { LOGGER.error( "Index removal failed - task not found by id: {} of workflow: {}", taskId, workflowId); Monitors.error(CLASS_NAME, "removeTask"); return; } LOGGER.debug( "Time taken {} for removing task:{} of workflow: {}", endTime - startTime, taskId, workflowId); Monitors.recordESIndexTime("remove_task", docType, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { LOGGER.error( "Failed to remove task: {} of workflow: {} from index", taskId, workflowId, e); Monitors.error(CLASS_NAME, "removeTask"); } } @Override public CompletableFuture asyncRemoveTask(String workflowId, String taskId) { return CompletableFuture.runAsync(() -> removeTask(workflowId, taskId), executorService); } @Override public void updateTask(String workflowId, String taskId, String[] keys, Object[] values) { if (keys.length != values.length) { throw new IllegalArgumentException("Number of keys and values do not match"); } long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; UpdateRequest request = new UpdateRequest(taskIndexName, docType, taskId); Map source = IntStream.range(0, keys.length) .boxed() .collect(Collectors.toMap(i -> keys[i], i -> values[i])); request.doc(source); LOGGER.debug( "Updating task: {} of workflow: {} in elasticsearch index: {}", taskId, workflowId, taskIndexName); elasticSearchClient.update(request).actionGet(); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for updating task: {} of workflow: {}", endTime - startTime, taskId, workflowId); Monitors.recordESIndexTime("update_task", docType, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } @Override public CompletableFuture asyncUpdateTask( String workflowId, String taskId, String[] keys, Object[] values) { return CompletableFuture.runAsync( () -> updateTask(workflowId, taskId, keys, values), executorService); } @Override public String get(String workflowInstanceId, String fieldToGet) { String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; GetRequest request = new GetRequest(workflowIndexName, docType, workflowInstanceId) .fetchSourceContext( new FetchSourceContext( true, new String[] {fieldToGet}, Strings.EMPTY_ARRAY)); GetResponse response = elasticSearchClient.get(request).actionGet(); if (response.isExists()) { Map sourceAsMap = response.getSourceAsMap(); if (sourceAsMap.get(fieldToGet) != null) { return sourceAsMap.get(fieldToGet).toString(); } } LOGGER.debug( "Unable to find Workflow: {} in ElasticSearch index: {}.", workflowInstanceId, workflowIndexName); return null; } private long count(String structuredQuery, String freeTextQuery, String docType) { try { docType = StringUtils.isBlank(docTypeOverride) ? docType : docTypeOverride; BoolQueryBuilder fq = boolQueryBuilder(structuredQuery, freeTextQuery); // The count api has been removed from the Java api, use the search api instead and set // size to 0. final SearchRequestBuilder srb = elasticSearchClient .prepareSearch(getIndexName(docType)) .setQuery(fq) .setTypes(docType) .storedFields("_id") .setSize(0); SearchResponse response = srb.get(); return response.getHits().getTotalHits(); } catch (ParserException e) { throw new TransientException(e.getMessage(), e); } } private SearchResult search( String structuredQuery, int start, int size, List sortOptions, String freeTextQuery, String docType, boolean idOnly, Class clazz) { try { docType = StringUtils.isBlank(docTypeOverride) ? docType : docTypeOverride; BoolQueryBuilder fq = boolQueryBuilder(structuredQuery, freeTextQuery); final SearchRequestBuilder srb = elasticSearchClient .prepareSearch(getIndexName(docType)) .setQuery(fq) .setTypes(docType) .setFrom(start) .setSize(size); if (idOnly) { srb.storedFields("_id"); } addSortOptions(srb, sortOptions); return mapSearchResult(srb.get(), idOnly, clazz); } catch (ParserException e) { throw new TransientException(e.getMessage(), e); } } private void addSortOptions(SearchRequestBuilder srb, List sortOptions) { if (sortOptions != null) { sortOptions.forEach( sortOption -> { SortOrder order = SortOrder.ASC; String field = sortOption; int indx = sortOption.indexOf(':'); // Can't be 0, need the field name at-least if (indx > 0) { field = sortOption.substring(0, indx); order = SortOrder.valueOf(sortOption.substring(indx + 1)); } srb.addSort(field, order); }); } } private SearchResult mapSearchResult( SearchResponse response, boolean idOnly, Class clazz) { SearchHits searchHits = response.getHits(); long count = searchHits.getTotalHits(); List result; if (idOnly) { result = Arrays.stream(searchHits.getHits()) .map(hit -> clazz.cast(hit.getId())) .collect(Collectors.toList()); } else { result = Arrays.stream(searchHits.getHits()) .map( hit -> { try { return objectMapper.readValue( hit.getSourceAsString(), clazz); } catch (JsonProcessingException e) { LOGGER.error( "Failed to de-serialize elasticsearch from source: {}", hit.getSourceAsString(), e); } return null; }) .collect(Collectors.toList()); } return new SearchResult<>(count, result); } @Override public List searchArchivableWorkflows(String indexName, long archiveTtlDays) { QueryBuilder q = QueryBuilders.boolQuery() .must( QueryBuilders.rangeQuery("endTime") .lt(LocalDate.now().minusDays(archiveTtlDays).toString()) .gte( LocalDate.now() .minusDays(archiveTtlDays) .minusDays(1) .toString())) .should(QueryBuilders.termQuery("status", "COMPLETED")) .should(QueryBuilders.termQuery("status", "FAILED")) .should(QueryBuilders.termQuery("status", "TIMED_OUT")) .should(QueryBuilders.termQuery("status", "TERMINATED")) .mustNot(QueryBuilders.existsQuery("archived")) .minimumShouldMatch(1); String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; SearchRequestBuilder s = elasticSearchClient .prepareSearch(indexName) .setTypes(docType) .setQuery(q) .setSize(1000); return extractSearchIds(s); } private UpdateRequest buildUpdateRequest( String id, byte[] doc, String indexName, String docType) { UpdateRequest req = new UpdateRequest(indexName, docType, id); req.doc(doc, XContentType.JSON); req.upsert(doc, XContentType.JSON); req.retryOnConflict(UPDATE_REQUEST_RETRY_COUNT); return req; } private List extractSearchIds(SearchRequestBuilder s) { SearchResponse response = s.execute().actionGet(); SearchHits hits = response.getHits(); List ids = new LinkedList<>(); for (SearchHit hit : hits.getHits()) { ids.add(hit.getId()); } return ids; } /** * Flush the buffers if bulk requests have not been indexed for the past {@link * ElasticSearchProperties#getAsyncBufferFlushTimeout()} seconds. This is to prevent data loss * in case the instance is terminated, while the buffer still holds documents to be indexed. */ private void flushBulkRequests() { bulkRequests.entrySet().stream() .filter( entry -> (System.currentTimeMillis() - entry.getValue().getLastFlushTime()) >= asyncBufferFlushTimeout) .filter( entry -> entry.getValue().getBulkRequestBuilder() != null && entry.getValue() .getBulkRequestBuilder() .numberOfActions() > 0) .forEach( entry -> { LOGGER.debug( "Flushing bulk request buffer for type {}, size: {}", entry.getKey(), entry.getValue().getBulkRequestBuilder().numberOfActions()); indexBulkRequest(entry.getKey()); }); } private static class BulkRequests { private long lastFlushTime; private BulkRequestBuilderWrapper bulkRequestBuilder; public long getLastFlushTime() { return lastFlushTime; } public void setLastFlushTime(long lastFlushTime) { this.lastFlushTime = lastFlushTime; } public BulkRequestBuilderWrapper getBulkRequestBuilder() { return bulkRequestBuilder; } public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) { this.bulkRequestBuilder = new BulkRequestBuilderWrapper(bulkRequestBuilder); } BulkRequests(long lastFlushTime, BulkRequestBuilder bulkRequestBuilder) { this.lastFlushTime = lastFlushTime; this.bulkRequestBuilder = new BulkRequestBuilderWrapper(bulkRequestBuilder); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDAOV6.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.io.IOException; import java.io.InputStream; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.LocalDate; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NByteArrayEntity; import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.*; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.NonTransientException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es6.config.ElasticSearchProperties; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; import com.netflix.conductor.metrics.Monitors; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.type.MapType; import com.fasterxml.jackson.databind.type.TypeFactory; @Trace public class ElasticSearchRestDAOV6 extends ElasticSearchBaseDAO implements IndexDAO { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRestDAOV6.class); private static final int CORE_POOL_SIZE = 6; private static final long KEEP_ALIVE_TIME = 1L; private static final String WORKFLOW_DOC_TYPE = "workflow"; private static final String TASK_DOC_TYPE = "task"; private static final String LOG_DOC_TYPE = "task_log"; private static final String EVENT_DOC_TYPE = "event"; private static final String MSG_DOC_TYPE = "message"; private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); private static final SimpleDateFormat SIMPLE_DATE_FORMAT = new SimpleDateFormat("yyyyMMWW"); private @interface HttpMethod { String GET = "GET"; String POST = "POST"; String PUT = "PUT"; String HEAD = "HEAD"; } private static final String className = ElasticSearchRestDAOV6.class.getSimpleName(); private final String workflowIndexName; private final String taskIndexName; private final String eventIndexPrefix; private String eventIndexName; private final String messageIndexPrefix; private String messageIndexName; private String logIndexName; private final String logIndexPrefix; private final String docTypeOverride; private final String clusterHealthColor; private final ObjectMapper objectMapper; private final RestHighLevelClient elasticSearchClient; private final RestClient elasticSearchAdminClient; private final ExecutorService executorService; private final ExecutorService logExecutorService; private final ConcurrentHashMap bulkRequests; private final int indexBatchSize; private final long asyncBufferFlushTimeout; private final ElasticSearchProperties properties; private final RetryTemplate retryTemplate; static { SIMPLE_DATE_FORMAT.setTimeZone(GMT); } public ElasticSearchRestDAOV6( RestClientBuilder restClientBuilder, RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { this.objectMapper = objectMapper; this.elasticSearchAdminClient = restClientBuilder.build(); this.elasticSearchClient = new RestHighLevelClient(restClientBuilder); this.clusterHealthColor = properties.getClusterHealthColor(); this.bulkRequests = new ConcurrentHashMap<>(); this.indexBatchSize = properties.getIndexBatchSize(); this.asyncBufferFlushTimeout = properties.getAsyncBufferFlushTimeout().toMillis(); this.properties = properties; this.indexPrefix = properties.getIndexPrefix(); if (!properties.isAutoIndexManagementEnabled() && StringUtils.isNotBlank(properties.getDocumentTypeOverride())) { docTypeOverride = properties.getDocumentTypeOverride(); } else { docTypeOverride = ""; } this.workflowIndexName = getIndexName(WORKFLOW_DOC_TYPE); this.taskIndexName = getIndexName(TASK_DOC_TYPE); this.logIndexPrefix = this.indexPrefix + "_" + LOG_DOC_TYPE; this.messageIndexPrefix = this.indexPrefix + "_" + MSG_DOC_TYPE; this.eventIndexPrefix = this.indexPrefix + "_" + EVENT_DOC_TYPE; int workerQueueSize = properties.getAsyncWorkerQueueSize(); int maximumPoolSize = properties.getAsyncMaxPoolSize(); // Set up a workerpool for performing async operations. this.executorService = new ThreadPoolExecutor( CORE_POOL_SIZE, maximumPoolSize, KEEP_ALIVE_TIME, TimeUnit.MINUTES, new LinkedBlockingQueue<>(workerQueueSize), (runnable, executor) -> { LOGGER.warn( "Request {} to async dao discarded in executor {}", runnable, executor); Monitors.recordDiscardedIndexingCount("indexQueue"); }); // Set up a workerpool for performing async operations for task_logs, event_executions, // message int corePoolSize = 1; maximumPoolSize = 2; long keepAliveTime = 30L; this.logExecutorService = new ThreadPoolExecutor( corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<>(workerQueueSize), (runnable, executor) -> { LOGGER.warn( "Request {} to async log dao discarded in executor {}", runnable, executor); Monitors.recordDiscardedIndexingCount("logQueue"); }); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS); this.retryTemplate = retryTemplate; } @PreDestroy private void shutdown() { LOGGER.info("Gracefully shutdown executor service"); shutdownExecutorService(logExecutorService); shutdownExecutorService(executorService); } private void shutdownExecutorService(ExecutorService execService) { try { execService.shutdown(); if (execService.awaitTermination(30, TimeUnit.SECONDS)) { LOGGER.debug("tasks completed, shutting down"); } else { LOGGER.warn("Forcing shutdown after waiting for 30 seconds"); execService.shutdownNow(); } } catch (InterruptedException ie) { LOGGER.warn( "Shutdown interrupted, invoking shutdownNow on scheduledThreadPoolExecutor for delay queue"); execService.shutdownNow(); Thread.currentThread().interrupt(); } } @Override @PostConstruct public void setup() throws Exception { waitForHealthyCluster(); if (properties.isAutoIndexManagementEnabled()) { createIndexesTemplates(); createWorkflowIndex(); createTaskIndex(); } } private void createIndexesTemplates() { try { initIndexesTemplates(); updateIndexesNames(); Executors.newScheduledThreadPool(1) .scheduleAtFixedRate(this::updateIndexesNames, 0, 1, TimeUnit.HOURS); } catch (Exception e) { LOGGER.error("Error creating index templates!", e); } } private void initIndexesTemplates() { initIndexTemplate(LOG_DOC_TYPE); initIndexTemplate(EVENT_DOC_TYPE); initIndexTemplate(MSG_DOC_TYPE); } /** Initializes the index with the required templates and mappings. */ private void initIndexTemplate(String type) { String template = "template_" + type; try { if (doesResourceNotExist("/_template/" + template)) { LOGGER.info("Creating the index template '" + template + "'"); InputStream stream = ElasticSearchDAOV6.class.getResourceAsStream("/" + template + ".json"); byte[] templateSource = IOUtils.toByteArray(stream); HttpEntity entity = new NByteArrayEntity(templateSource, ContentType.APPLICATION_JSON); elasticSearchAdminClient.performRequest( HttpMethod.PUT, "/_template/" + template, Collections.emptyMap(), entity); } } catch (Exception e) { LOGGER.error("Failed to init " + template, e); } } private void updateIndexesNames() { logIndexName = updateIndexName(LOG_DOC_TYPE); eventIndexName = updateIndexName(EVENT_DOC_TYPE); messageIndexName = updateIndexName(MSG_DOC_TYPE); } private String updateIndexName(String type) { String indexName = this.indexPrefix + "_" + type + "_" + SIMPLE_DATE_FORMAT.format(new Date()); try { addIndex(indexName); return indexName; } catch (IOException e) { LOGGER.error("Failed to update log index name: {}", indexName, e); throw new NonTransientException("Failed to update log index name: " + indexName, e); } } private void createWorkflowIndex() { String indexName = getIndexName(WORKFLOW_DOC_TYPE); try { addIndex(indexName); } catch (IOException e) { LOGGER.error("Failed to initialize index '{}'", indexName, e); } try { addMappingToIndex(indexName, WORKFLOW_DOC_TYPE, "/mappings_docType_workflow.json"); } catch (IOException e) { LOGGER.error("Failed to add {} mapping", WORKFLOW_DOC_TYPE); } } private void createTaskIndex() { String indexName = getIndexName(TASK_DOC_TYPE); try { addIndex(indexName); } catch (IOException e) { LOGGER.error("Failed to initialize index '{}'", indexName, e); } try { addMappingToIndex(indexName, TASK_DOC_TYPE, "/mappings_docType_task.json"); } catch (IOException e) { LOGGER.error("Failed to add {} mapping", TASK_DOC_TYPE); } } /** * Waits for the ES cluster to become green. * * @throws Exception If there is an issue connecting with the ES cluster. */ private void waitForHealthyCluster() throws Exception { Map params = new HashMap<>(); params.put("wait_for_status", this.clusterHealthColor); params.put("timeout", "30s"); elasticSearchAdminClient.performRequest("GET", "/_cluster/health", params); } /** * Adds an index to elasticsearch if it does not exist. * * @param index The name of the index to create. * @throws IOException If an error occurred during requests to ES. */ private void addIndex(final String index) throws IOException { LOGGER.info("Adding index '{}'...", index); String resourcePath = "/" + index; if (doesResourceNotExist(resourcePath)) { try { ObjectNode setting = objectMapper.createObjectNode(); ObjectNode indexSetting = objectMapper.createObjectNode(); indexSetting.put("number_of_shards", properties.getIndexShardCount()); indexSetting.put("number_of_replicas", properties.getIndexReplicasCount()); setting.set("index", indexSetting); elasticSearchAdminClient.performRequest( HttpMethod.PUT, resourcePath, Collections.emptyMap(), new NStringEntity(setting.toString(), ContentType.APPLICATION_JSON)); LOGGER.info("Added '{}' index", index); } catch (ResponseException e) { boolean errorCreatingIndex = true; Response errorResponse = e.getResponse(); if (errorResponse.getStatusLine().getStatusCode() == HttpStatus.SC_BAD_REQUEST) { JsonNode root = objectMapper.readTree(EntityUtils.toString(errorResponse.getEntity())); String errorCode = root.get("error").get("type").asText(); if ("index_already_exists_exception".equals(errorCode)) { errorCreatingIndex = false; } } if (errorCreatingIndex) { throw e; } } } else { LOGGER.info("Index '{}' already exists", index); } } /** * Adds a mapping type to an index if it does not exist. * * @param index The name of the index. * @param mappingType The name of the mapping type. * @param mappingFilename The name of the mapping file to use to add the mapping if it does not * exist. * @throws IOException If an error occurred during requests to ES. */ private void addMappingToIndex( final String index, final String mappingType, final String mappingFilename) throws IOException { LOGGER.info("Adding '{}' mapping to index '{}'...", mappingType, index); String resourcePath = "/" + index + "/_mapping/" + mappingType; if (doesResourceNotExist(resourcePath)) { HttpEntity entity = new NByteArrayEntity( loadTypeMappingSource(mappingFilename).getBytes(), ContentType.APPLICATION_JSON); elasticSearchAdminClient.performRequest( HttpMethod.PUT, resourcePath, Collections.emptyMap(), entity); LOGGER.info("Added '{}' mapping", mappingType); } else { LOGGER.info("Mapping '{}' already exists", mappingType); } } /** * Determines whether a resource exists in ES. This will call a GET method to a particular path * and return true if status 200; false otherwise. * * @param resourcePath The path of the resource to get. * @return True if it exists; false otherwise. * @throws IOException If an error occurred during requests to ES. */ public boolean doesResourceExist(final String resourcePath) throws IOException { Response response = elasticSearchAdminClient.performRequest(HttpMethod.HEAD, resourcePath); return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; } /** * The inverse of doesResourceExist. * * @param resourcePath The path of the resource to check. * @return True if it does not exist; false otherwise. * @throws IOException If an error occurred during requests to ES. */ public boolean doesResourceNotExist(final String resourcePath) throws IOException { return !doesResourceExist(resourcePath); } @Override public void indexWorkflow(WorkflowSummary workflow) { try { long startTime = Instant.now().toEpochMilli(); String workflowId = workflow.getWorkflowId(); byte[] docBytes = objectMapper.writeValueAsBytes(workflow); String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; IndexRequest request = new IndexRequest(workflowIndexName, docType, workflowId); request.source(docBytes, XContentType.JSON); elasticSearchClient.index(request, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing workflow: {}", endTime - startTime, workflowId); Monitors.recordESIndexTime("index_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { Monitors.error(className, "indexWorkflow"); LOGGER.error("Failed to index workflow: {}", workflow.getWorkflowId(), e); } } @Override public CompletableFuture asyncIndexWorkflow(WorkflowSummary workflow) { return CompletableFuture.runAsync(() -> indexWorkflow(workflow), executorService); } @Override public void indexTask(TaskSummary task) { try { long startTime = Instant.now().toEpochMilli(); String taskId = task.getTaskId(); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; indexObject(taskIndexName, docType, taskId, task); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing task:{} in workflow: {}", endTime - startTime, taskId, task.getWorkflowId()); Monitors.recordESIndexTime("index_task", TASK_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to index task: {}", task.getTaskId(), e); } } @Override public CompletableFuture asyncIndexTask(TaskSummary task) { return CompletableFuture.runAsync(() -> indexTask(task), executorService); } @Override public void addTaskExecutionLogs(List taskExecLogs) { if (taskExecLogs.isEmpty()) { return; } long startTime = Instant.now().toEpochMilli(); BulkRequest bulkRequest = new BulkRequest(); for (TaskExecLog log : taskExecLogs) { byte[] docBytes; try { docBytes = objectMapper.writeValueAsBytes(log); } catch (JsonProcessingException e) { LOGGER.error("Failed to convert task log to JSON for task {}", log.getTaskId()); continue; } String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride; IndexRequest request = new IndexRequest(logIndexName, docType); request.source(docBytes, XContentType.JSON); bulkRequest.add(request); } try { elasticSearchClient.bulk(bulkRequest, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime); Monitors.recordESIndexTime( "index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size()); } catch (Exception e) { List taskIds = taskExecLogs.stream().map(TaskExecLog::getTaskId).collect(Collectors.toList()); LOGGER.error("Failed to index task execution logs for tasks: {}", taskIds, e); } } @Override public CompletableFuture asyncAddTaskExecutionLogs(List logs) { return CompletableFuture.runAsync(() -> addTaskExecutionLogs(logs), logExecutorService); } @Override public List getTaskExecutionLogs(String taskId) { try { BoolQueryBuilder query = boolQueryBuilder("taskId='" + taskId + "'", "*"); // Create the searchObjectIdsViaExpression source SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(query); searchSourceBuilder.sort(new FieldSortBuilder("createdTime").order(SortOrder.ASC)); searchSourceBuilder.size(properties.getTaskLogResultLimit()); // Generate the actual request to send to ES. String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride; SearchRequest searchRequest = new SearchRequest(logIndexPrefix + "*"); searchRequest.types(docType); searchRequest.source(searchSourceBuilder); SearchResponse response = elasticSearchClient.search(searchRequest); return mapTaskExecLogsResponse(response); } catch (Exception e) { LOGGER.error("Failed to get task execution logs for task: {}", taskId, e); } return null; } private List mapTaskExecLogsResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); List logs = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); TaskExecLog tel = objectMapper.readValue(source, TaskExecLog.class); logs.add(tel); } return logs; } @Override public List getMessages(String queue) { try { BoolQueryBuilder query = boolQueryBuilder("queue='" + queue + "'", "*"); // Create the searchObjectIdsViaExpression source SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(query); searchSourceBuilder.sort(new FieldSortBuilder("created").order(SortOrder.ASC)); // Generate the actual request to send to ES. String docType = StringUtils.isBlank(docTypeOverride) ? MSG_DOC_TYPE : docTypeOverride; SearchRequest searchRequest = new SearchRequest(messageIndexPrefix + "*"); searchRequest.types(docType); searchRequest.source(searchSourceBuilder); SearchResponse response = elasticSearchClient.search(searchRequest); return mapGetMessagesResponse(response); } catch (Exception e) { LOGGER.error("Failed to get messages for queue: {}", queue, e); } return null; } private List mapGetMessagesResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); TypeFactory factory = TypeFactory.defaultInstance(); MapType type = factory.constructMapType(HashMap.class, String.class, String.class); List messages = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); Map mapSource = objectMapper.readValue(source, type); Message msg = new Message(mapSource.get("messageId"), mapSource.get("payload"), null); messages.add(msg); } return messages; } @Override public List getEventExecutions(String event) { try { BoolQueryBuilder query = boolQueryBuilder("event='" + event + "'", "*"); // Create the searchObjectIdsViaExpression source SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(query); searchSourceBuilder.sort(new FieldSortBuilder("created").order(SortOrder.ASC)); // Generate the actual request to send to ES. String docType = StringUtils.isBlank(docTypeOverride) ? EVENT_DOC_TYPE : docTypeOverride; SearchRequest searchRequest = new SearchRequest(eventIndexPrefix + "*"); searchRequest.types(docType); searchRequest.source(searchSourceBuilder); SearchResponse response = elasticSearchClient.search(searchRequest); return mapEventExecutionsResponse(response); } catch (Exception e) { LOGGER.error("Failed to get executions for event: {}", event, e); } return null; } private List mapEventExecutionsResponse(SearchResponse response) throws IOException { SearchHit[] hits = response.getHits().getHits(); List executions = new ArrayList<>(hits.length); for (SearchHit hit : hits) { String source = hit.getSourceAsString(); EventExecution tel = objectMapper.readValue(source, EventExecution.class); executions.add(tel); } return executions; } @Override public void addMessage(String queue, Message message) { try { long startTime = Instant.now().toEpochMilli(); Map doc = new HashMap<>(); doc.put("messageId", message.getId()); doc.put("payload", message.getPayload()); doc.put("queue", queue); doc.put("created", System.currentTimeMillis()); String docType = StringUtils.isBlank(docTypeOverride) ? MSG_DOC_TYPE : docTypeOverride; indexObject(messageIndexName, docType, doc); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing message: {}", endTime - startTime, message.getId()); Monitors.recordESIndexTime("add_message", MSG_DOC_TYPE, endTime - startTime); } catch (Exception e) { LOGGER.error("Failed to index message: {}", message.getId(), e); } } @Override public CompletableFuture asyncAddMessage(String queue, Message message) { return CompletableFuture.runAsync(() -> addMessage(queue, message), executorService); } @Override public void addEventExecution(EventExecution eventExecution) { try { long startTime = Instant.now().toEpochMilli(); String id = eventExecution.getName() + "." + eventExecution.getEvent() + "." + eventExecution.getMessageId() + "." + eventExecution.getId(); String docType = StringUtils.isBlank(docTypeOverride) ? EVENT_DOC_TYPE : docTypeOverride; indexObject(eventIndexName, docType, id, eventExecution); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing event execution: {}", endTime - startTime, eventExecution.getId()); Monitors.recordESIndexTime("add_event_execution", EVENT_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to index event execution: {}", eventExecution.getId(), e); } } @Override public CompletableFuture asyncAddEventExecution(EventExecution eventExecution) { return CompletableFuture.runAsync( () -> addEventExecution(eventExecution), logExecutorService); } @Override public SearchResult searchWorkflows( String query, String freeText, int start, int count, List sort) { try { return searchObjectsViaExpression( query, start, count, sort, freeText, WORKFLOW_DOC_TYPE, true, String.class); } catch (Exception e) { throw new TransientException(e.getMessage(), e); } } @Override public SearchResult searchWorkflowSummary( String query, String freeText, int start, int count, List sort) { try { return searchObjectsViaExpression( query, start, count, sort, freeText, WORKFLOW_DOC_TYPE, false, WorkflowSummary.class); } catch (Exception e) { throw new TransientException(e.getMessage(), e); } } @Override public SearchResult searchTasks( String query, String freeText, int start, int count, List sort) { try { return searchObjectsViaExpression( query, start, count, sort, freeText, TASK_DOC_TYPE, true, String.class); } catch (Exception e) { throw new TransientException(e.getMessage(), e); } } @Override public SearchResult searchTaskSummary( String query, String freeText, int start, int count, List sort) { try { return searchObjectsViaExpression( query, start, count, sort, freeText, TASK_DOC_TYPE, false, TaskSummary.class); } catch (Exception e) { throw new TransientException(e.getMessage(), e); } } @Override public void removeWorkflow(String workflowId) { long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; DeleteRequest request = new DeleteRequest(workflowIndexName, docType, workflowId); try { DeleteResponse response = elasticSearchClient.delete(request); if (response.getResult() == DocWriteResponse.Result.NOT_FOUND) { LOGGER.error("Index removal failed - document not found by id: {}", workflowId); } long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for removing workflow: {}", endTime - startTime, workflowId); Monitors.recordESIndexTime("remove_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (IOException e) { LOGGER.error("Failed to remove workflow {} from index", workflowId, e); Monitors.error(className, "remove"); } } @Override public CompletableFuture asyncRemoveWorkflow(String workflowId) { return CompletableFuture.runAsync(() -> removeWorkflow(workflowId), executorService); } @Override public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) { try { if (keys.length != values.length) { throw new IllegalArgumentException("Number of keys and values do not match"); } long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; UpdateRequest request = new UpdateRequest(workflowIndexName, docType, workflowInstanceId); Map source = IntStream.range(0, keys.length) .boxed() .collect(Collectors.toMap(i -> keys[i], i -> values[i])); request.doc(source); LOGGER.debug("Updating workflow {} with {}", workflowInstanceId, source); elasticSearchClient.update(request, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId); Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to update workflow {}", workflowInstanceId, e); Monitors.error(className, "update"); } } @Override public CompletableFuture asyncUpdateWorkflow( String workflowInstanceId, String[] keys, Object[] values) { return CompletableFuture.runAsync( () -> updateWorkflow(workflowInstanceId, keys, values), executorService); } @Override public void removeTask(String workflowId, String taskId) { long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; SearchResult taskSearchResult = searchTasks( String.format("(taskId='%s') AND (workflowId='%s')", taskId, workflowId), "*", 0, 1, null); if (taskSearchResult.getTotalHits() == 0) { LOGGER.error("Task: {} does not belong to workflow: {}", taskId, workflowId); Monitors.error(className, "removeTask"); return; } DeleteRequest request = new DeleteRequest(taskIndexName, docType, taskId); try { DeleteResponse response = elasticSearchClient.delete(request); if (response.getResult() != DocWriteResponse.Result.DELETED) { LOGGER.error("Index removal failed - task not found by id: {}", workflowId); Monitors.error(className, "removeTask"); return; } long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for removing task:{} of workflow: {}", endTime - startTime, taskId, workflowId); Monitors.recordESIndexTime("remove_task", docType, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (IOException e) { LOGGER.error( "Failed to remove task {} of workflow: {} from index", taskId, workflowId, e); Monitors.error(className, "removeTask"); } } @Override public CompletableFuture asyncRemoveTask(String workflowId, String taskId) { return CompletableFuture.runAsync(() -> removeTask(workflowId, taskId), executorService); } @Override public void updateTask(String workflowId, String taskId, String[] keys, Object[] values) { try { if (keys.length != values.length) { throw new IllegalArgumentException("Number of keys and values do not match"); } long startTime = Instant.now().toEpochMilli(); String docType = StringUtils.isBlank(docTypeOverride) ? TASK_DOC_TYPE : docTypeOverride; UpdateRequest request = new UpdateRequest(taskIndexName, docType, taskId); Map source = IntStream.range(0, keys.length) .boxed() .collect(Collectors.toMap(i -> keys[i], i -> values[i])); request.doc(source); LOGGER.debug("Updating task: {} of workflow: {} with {}", taskId, workflowId, source); elasticSearchClient.update(request, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for updating task: {} of workflow: {}", endTime - startTime, taskId, workflowId); Monitors.recordESIndexTime("update_task", docType, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); } catch (Exception e) { LOGGER.error("Failed to update task: {} of workflow: {}", taskId, workflowId, e); Monitors.error(className, "update"); } } @Override public CompletableFuture asyncUpdateTask( String workflowId, String taskId, String[] keys, Object[] values) { return CompletableFuture.runAsync( () -> updateTask(workflowId, taskId, keys, values), executorService); } @Override public String get(String workflowInstanceId, String fieldToGet) { String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; GetRequest request = new GetRequest(workflowIndexName, docType, workflowInstanceId); GetResponse response; try { response = elasticSearchClient.get(request); } catch (IOException e) { LOGGER.error( "Unable to get Workflow: {} from ElasticSearch index: {}", workflowInstanceId, workflowIndexName, e); return null; } if (response.isExists()) { Map sourceAsMap = response.getSourceAsMap(); if (sourceAsMap.get(fieldToGet) != null) { return sourceAsMap.get(fieldToGet).toString(); } } LOGGER.debug( "Unable to find Workflow: {} in ElasticSearch index: {}.", workflowInstanceId, workflowIndexName); return null; } private SearchResult searchObjectsViaExpression( String structuredQuery, int start, int size, List sortOptions, String freeTextQuery, String docType, boolean idOnly, Class clazz) throws ParserException, IOException { QueryBuilder queryBuilder = boolQueryBuilder(structuredQuery, freeTextQuery); return searchObjects( getIndexName(docType), queryBuilder, start, size, sortOptions, docType, idOnly, clazz); } private SearchResult searchObjectIds( String indexName, QueryBuilder queryBuilder, int start, int size, String docType) throws IOException { return searchObjects( indexName, queryBuilder, start, size, null, docType, true, String.class); } /** * Tries to find objects for a given query in an index. * * @param indexName The name of the index. * @param queryBuilder The query to use for searching. * @param start The start to use. * @param size The total return size. * @param sortOptions A list of string options to sort in the form VALUE:ORDER; where ORDER is * optional and can be either ASC OR DESC. * @param docType The document type to searchObjectIdsViaExpression for. * @return The SearchResults which includes the count and objects that were found. * @throws IOException If we cannot communicate with ES. */ private SearchResult searchObjects( String indexName, QueryBuilder queryBuilder, int start, int size, List sortOptions, String docType, boolean idOnly, Class clazz) throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(queryBuilder); searchSourceBuilder.from(start); searchSourceBuilder.size(size); if (idOnly) { searchSourceBuilder.fetchSource(false); } if (sortOptions != null && !sortOptions.isEmpty()) { for (String sortOption : sortOptions) { SortOrder order = SortOrder.ASC; String field = sortOption; int index = sortOption.indexOf(":"); if (index > 0) { field = sortOption.substring(0, index); order = SortOrder.valueOf(sortOption.substring(index + 1)); } searchSourceBuilder.sort(new FieldSortBuilder(field).order(order)); } } // Generate the actual request to send to ES. docType = StringUtils.isBlank(docTypeOverride) ? docType : docTypeOverride; SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.types(docType); searchRequest.source(searchSourceBuilder); SearchResponse response = elasticSearchClient.search(searchRequest); return mapSearchResult(response, idOnly, clazz); } private SearchResult mapSearchResult( SearchResponse response, boolean idOnly, Class clazz) { SearchHits searchHits = response.getHits(); long count = searchHits.getTotalHits(); List result; if (idOnly) { result = Arrays.stream(searchHits.getHits()) .map(hit -> clazz.cast(hit.getId())) .collect(Collectors.toList()); } else { result = Arrays.stream(searchHits.getHits()) .map( hit -> { try { return objectMapper.readValue( hit.getSourceAsString(), clazz); } catch (JsonProcessingException e) { LOGGER.error( "Failed to de-serialize elasticsearch from source: {}", hit.getSourceAsString(), e); } return null; }) .collect(Collectors.toList()); } return new SearchResult<>(count, result); } @Override public List searchArchivableWorkflows(String indexName, long archiveTtlDays) { QueryBuilder q = QueryBuilders.boolQuery() .must( QueryBuilders.rangeQuery("endTime") .lt(LocalDate.now().minusDays(archiveTtlDays).toString()) .gte( LocalDate.now() .minusDays(archiveTtlDays) .minusDays(1) .toString())) .should(QueryBuilders.termQuery("status", "COMPLETED")) .should(QueryBuilders.termQuery("status", "FAILED")) .should(QueryBuilders.termQuery("status", "TIMED_OUT")) .should(QueryBuilders.termQuery("status", "TERMINATED")) .mustNot(QueryBuilders.existsQuery("archived")) .minimumShouldMatch(1); SearchResult workflowIds; try { workflowIds = searchObjectIds(indexName, q, 0, 1000, WORKFLOW_DOC_TYPE); } catch (IOException e) { LOGGER.error("Unable to communicate with ES to find archivable workflows", e); return Collections.emptyList(); } return workflowIds.getResults(); } @Override public long getWorkflowCount(String query, String freeText) { try { return getObjectCounts(query, freeText, WORKFLOW_DOC_TYPE); } catch (Exception e) { throw new TransientException(e.getMessage(), e); } } private long getObjectCounts(String structuredQuery, String freeTextQuery, String docType) throws ParserException, IOException { QueryBuilder queryBuilder = boolQueryBuilder(structuredQuery, freeTextQuery); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(queryBuilder); String indexName = getIndexName(docType); CountRequest countRequest = new CountRequest(new String[] {indexName}, sourceBuilder); CountResponse countResponse = elasticSearchClient.count(countRequest, RequestOptions.DEFAULT); return countResponse.getCount(); } private void indexObject(final String index, final String docType, final Object doc) { indexObject(index, docType, null, doc); } private void indexObject( final String index, final String docType, final String docId, final Object doc) { byte[] docBytes; try { docBytes = objectMapper.writeValueAsBytes(doc); } catch (JsonProcessingException e) { LOGGER.error("Failed to convert {} '{}' to byte string", docType, docId); return; } IndexRequest request = new IndexRequest(index, docType, docId); request.source(docBytes, XContentType.JSON); if (bulkRequests.get(docType) == null) { bulkRequests.put( docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest())); } bulkRequests.get(docType).getBulkRequest().add(request); if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) { indexBulkRequest(docType); } } private synchronized void indexBulkRequest(String docType) { if (bulkRequests.get(docType).getBulkRequest() != null && bulkRequests.get(docType).getBulkRequest().numberOfActions() > 0) { synchronized (bulkRequests.get(docType).getBulkRequest()) { indexWithRetry( bulkRequests.get(docType).getBulkRequest().get(), "Bulk Indexing " + docType, docType); bulkRequests.put( docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest())); } } } /** * Performs an index operation with a retry. * * @param request The index request that we want to perform. * @param operationDescription The type of operation that we are performing. */ private void indexWithRetry( final BulkRequest request, final String operationDescription, String docType) { try { long startTime = Instant.now().toEpochMilli(); retryTemplate.execute( context -> elasticSearchClient.bulk(request, RequestOptions.DEFAULT)); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing object of type: {}", endTime - startTime, docType); Monitors.recordESIndexTime("index_object", docType, endTime - startTime); Monitors.recordWorkerQueueSize( "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); Monitors.recordWorkerQueueSize( "logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size()); } catch (Exception e) { Monitors.error(className, "index"); LOGGER.error("Failed to index {} for request type: {}", request, docType, e); } } /** * Flush the buffers if bulk requests have not been indexed for the past {@link * ElasticSearchProperties#getAsyncBufferFlushTimeout()} seconds. This is to prevent data loss * in case the instance is terminated, while the buffer still holds documents to be indexed. */ private void flushBulkRequests() { bulkRequests.entrySet().stream() .filter( entry -> (System.currentTimeMillis() - entry.getValue().getLastFlushTime()) >= asyncBufferFlushTimeout) .filter( entry -> entry.getValue().getBulkRequest() != null && entry.getValue().getBulkRequest().numberOfActions() > 0) .forEach( entry -> { LOGGER.debug( "Flushing bulk request buffer for type {}, size: {}", entry.getKey(), entry.getValue().getBulkRequest().numberOfActions()); indexBulkRequest(entry.getKey()); }); } private static class BulkRequests { private final long lastFlushTime; private final BulkRequestWrapper bulkRequest; long getLastFlushTime() { return lastFlushTime; } BulkRequestWrapper getBulkRequest() { return bulkRequest; } BulkRequests(long lastFlushTime, BulkRequest bulkRequest) { this.lastFlushTime = lastFlushTime; this.bulkRequest = new BulkRequestWrapper(bulkRequest); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/Expression.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.InputStream; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import com.netflix.conductor.es6.dao.query.parser.internal.AbstractNode; import com.netflix.conductor.es6.dao.query.parser.internal.BooleanOp; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; public class Expression extends AbstractNode implements FilterProvider { private NameValue nameVal; private GroupedExpression ge; private BooleanOp op; private Expression rhs; public Expression(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] peeked = peek(1); if (peeked[0] == '(') { this.ge = new GroupedExpression(is); } else { this.nameVal = new NameValue(is); } peeked = peek(3); if (isBoolOpr(peeked)) { // we have an expression next this.op = new BooleanOp(is); this.rhs = new Expression(is); } } public boolean isBinaryExpr() { return this.op != null; } public BooleanOp getOperator() { return this.op; } public Expression getRightHandSide() { return this.rhs; } public boolean isNameValue() { return this.nameVal != null; } public NameValue getNameValue() { return this.nameVal; } public GroupedExpression getGroupedExpression() { return this.ge; } @Override public QueryBuilder getFilterBuilder() { QueryBuilder lhs = null; if (nameVal != null) { lhs = nameVal.getFilterBuilder(); } else { lhs = ge.getFilterBuilder(); } if (this.isBinaryExpr()) { QueryBuilder rhsFilter = rhs.getFilterBuilder(); if (this.op.isAnd()) { return QueryBuilders.boolQuery().must(lhs).must(rhsFilter); } else { return QueryBuilders.boolQuery().should(lhs).should(rhsFilter); } } else { return lhs; } } @Override public String toString() { if (isBinaryExpr()) { return "" + (nameVal == null ? ge : nameVal) + op + rhs; } else { return "" + (nameVal == null ? ge : nameVal); } } public static Expression fromString(String value) throws ParserException { return new Expression(new BufferedInputStream(new ByteArrayInputStream(value.getBytes()))); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/FilterProvider.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser; import org.elasticsearch.index.query.QueryBuilder; public interface FilterProvider { /** * @return FilterBuilder for elasticsearch */ public QueryBuilder getFilterBuilder(); } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/GroupedExpression.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser; import java.io.InputStream; import org.elasticsearch.index.query.QueryBuilder; import com.netflix.conductor.es6.dao.query.parser.internal.AbstractNode; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; public class GroupedExpression extends AbstractNode implements FilterProvider { private Expression expression; public GroupedExpression(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] peeked = read(1); assertExpected(peeked, "("); this.expression = new Expression(is); peeked = read(1); assertExpected(peeked, ")"); } @Override public String toString() { return "(" + expression + ")"; } /** * @return the expression */ public Expression getExpression() { return expression; } @Override public QueryBuilder getFilterBuilder() { return expression.getFilterBuilder(); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/NameValue.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser; import java.io.InputStream; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import com.netflix.conductor.es6.dao.query.parser.internal.AbstractNode; import com.netflix.conductor.es6.dao.query.parser.internal.ComparisonOp; import com.netflix.conductor.es6.dao.query.parser.internal.ComparisonOp.Operators; import com.netflix.conductor.es6.dao.query.parser.internal.ConstValue; import com.netflix.conductor.es6.dao.query.parser.internal.ListConst; import com.netflix.conductor.es6.dao.query.parser.internal.Name; import com.netflix.conductor.es6.dao.query.parser.internal.ParserException; import com.netflix.conductor.es6.dao.query.parser.internal.Range; /** * * * 

     * Represents an expression of the form as below:
 * key OPR value
 * OPR is the comparison operator which could be one of the following:
 * 	>, <, = , !=, IN, BETWEEN
 *
    */ public class NameValue extends AbstractNode implements FilterProvider { private Name name; private ComparisonOp op; private ConstValue value; private Range range; private ListConst valueList; public NameValue(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { this.name = new Name(is); this.op = new ComparisonOp(is); if (this.op.getOperator().equals(Operators.BETWEEN.value())) { this.range = new Range(is); } if (this.op.getOperator().equals(Operators.IN.value())) { this.valueList = new ListConst(is); } else { this.value = new ConstValue(is); } } @Override public String toString() { return "" + name + op + value; } /** * @return the name */ public Name getName() { return name; } /** * @return the op */ public ComparisonOp getOp() { return op; } /** * @return the value */ public ConstValue getValue() { return value; } @Override public QueryBuilder getFilterBuilder() { if (op.getOperator().equals(Operators.EQUALS.value())) { return QueryBuilders.queryStringQuery( name.getName() + ":" + value.getValue().toString()); } else if (op.getOperator().equals(Operators.BETWEEN.value())) { return QueryBuilders.rangeQuery(name.getName()) .from(range.getLow()) .to(range.getHigh()); } else if (op.getOperator().equals(Operators.IN.value())) { return QueryBuilders.termsQuery(name.getName(), valueList.getList()); } else if (op.getOperator().equals(Operators.NOT_EQUALS.value())) { return QueryBuilders.queryStringQuery( "NOT " + name.getName() + ":" + value.getValue().toString()); } else if (op.getOperator().equals(Operators.GREATER_THAN.value())) { return QueryBuilders.rangeQuery(name.getName()) .from(value.getValue()) .includeLower(false) .includeUpper(false); } else if (op.getOperator().equals(Operators.IS.value())) { if (value.getSysConstant().equals(ConstValue.SystemConsts.NULL)) { return QueryBuilders.boolQuery() .mustNot( QueryBuilders.boolQuery() .must(QueryBuilders.matchAllQuery()) .mustNot(QueryBuilders.existsQuery(name.getName()))); } else if (value.getSysConstant().equals(ConstValue.SystemConsts.NOT_NULL)) { return QueryBuilders.boolQuery() .mustNot( QueryBuilders.boolQuery() .must(QueryBuilders.matchAllQuery()) .must(QueryBuilders.existsQuery(name.getName()))); } } else if (op.getOperator().equals(Operators.LESS_THAN.value())) { return QueryBuilders.rangeQuery(name.getName()) .to(value.getValue()) .includeLower(false) .includeUpper(false); } else if (op.getOperator().equals(Operators.STARTS_WITH.value())) { return QueryBuilders.prefixQuery(name.getName(), value.getUnquotedValue()); } throw new IllegalStateException("Incorrect/unsupported operators"); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/AbstractNode.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; import java.math.BigDecimal; import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; public abstract class AbstractNode { public static final Pattern WHITESPACE = Pattern.compile("\\s"); protected static Set comparisonOprs = new HashSet<>(); static { comparisonOprs.add('>'); comparisonOprs.add('<'); comparisonOprs.add('='); } protected InputStream is; protected AbstractNode(InputStream is) throws ParserException { this.is = is; this.parse(); } protected boolean isNumber(String test) { try { // If you can convert to a big decimal value, then it is a number. new BigDecimal(test); return true; } catch (NumberFormatException e) { // Ignore } return false; } protected boolean isBoolOpr(byte[] buffer) { if (buffer.length > 1 && buffer[0] == 'O' && buffer[1] == 'R') { return true; } else { return buffer.length > 2 && buffer[0] == 'A' && buffer[1] == 'N' && buffer[2] == 'D'; } } protected boolean isComparisonOpr(byte[] buffer) { if (buffer[0] == 'I' && buffer[1] == 'N') { return true; } else if (buffer[0] == '!' && buffer[1] == '=') { return true; } else { return comparisonOprs.contains((char) buffer[0]); } } protected byte[] peek(int length) throws Exception { return read(length, true); } protected byte[] read(int length) throws Exception { return read(length, false); } protected String readToken() throws Exception { skipWhitespace(); StringBuilder sb = new StringBuilder(); while (is.available() > 0) { char c = (char) peek(1)[0]; if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { is.skip(1); break; } else if (c == '=' || c == '>' || c == '<' || c == '!') { // do not skip break; } sb.append(c); is.skip(1); } return sb.toString().trim(); } protected boolean isNumeric(char c) { return c == '-' || c == 'e' || (c >= '0' && c <= '9') || c == '.'; } protected void assertExpected(byte[] found, String expected) throws ParserException { assertExpected(new String(found), expected); } protected void assertExpected(String found, String expected) throws ParserException { if (!found.equals(expected)) { throw new ParserException("Expected " + expected + ", found " + found); } } protected void assertExpected(char found, char expected) throws ParserException { if (found != expected) { throw new ParserException("Expected " + expected + ", found " + found); } } protected static void efor(int length, FunctionThrowingException consumer) throws Exception { for (int i = 0; i < length; i++) { consumer.accept(i); } } protected abstract void _parse() throws Exception; // Public stuff here private void parse() throws ParserException { // skip white spaces skipWhitespace(); try { _parse(); } catch (Exception e) { if (!(e instanceof ParserException)) { throw new ParserException("Error parsing", e); } else { throw (ParserException) e; } } skipWhitespace(); } // Private methods private byte[] read(int length, boolean peekOnly) throws Exception { byte[] buf = new byte[length]; if (peekOnly) { is.mark(length); } efor(length, (Integer c) -> buf[c] = (byte) is.read()); if (peekOnly) { is.reset(); } return buf; } protected void skipWhitespace() throws ParserException { try { while (is.available() > 0) { byte c = peek(1)[0]; if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { // skip read(1); } else { break; } } } catch (Exception e) { throw new ParserException(e.getMessage(), e); } } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/BooleanOp.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; public class BooleanOp extends AbstractNode { private String value; public BooleanOp(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] buffer = peek(3); if (buffer.length > 1 && buffer[0] == 'O' && buffer[1] == 'R') { this.value = "OR"; } else if (buffer.length > 2 && buffer[0] == 'A' && buffer[1] == 'N' && buffer[2] == 'D') { this.value = "AND"; } else { throw new ParserException("No valid boolean operator found..."); } read(this.value.length()); } @Override public String toString() { return " " + value + " "; } public String getOperator() { return value; } public boolean isAnd() { return "AND".equals(value); } public boolean isOr() { return "OR".equals(value); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/ComparisonOp.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; public class ComparisonOp extends AbstractNode { public enum Operators { BETWEEN("BETWEEN"), EQUALS("="), LESS_THAN("<"), GREATER_THAN(">"), IN("IN"), NOT_EQUALS("!="), IS("IS"), STARTS_WITH("STARTS_WITH"); private final String value; Operators(String value) { this.value = value; } public String value() { return value; } } static { int max = 0; for (Operators op : Operators.values()) { max = Math.max(max, op.value().length()); } maxOperatorLength = max; } private static final int maxOperatorLength; private static final int betweenLen = Operators.BETWEEN.value().length(); private static final int startsWithLen = Operators.STARTS_WITH.value().length(); private String value; public ComparisonOp(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] peeked = peek(maxOperatorLength); if (peeked[0] == '=' || peeked[0] == '>' || peeked[0] == '<') { this.value = new String(peeked, 0, 1); } else if (peeked[0] == 'I' && peeked[1] == 'N') { this.value = "IN"; } else if (peeked[0] == 'I' && peeked[1] == 'S') { this.value = "IS"; } else if (peeked[0] == '!' && peeked[1] == '=') { this.value = "!="; } else if (peeked.length >= betweenLen && peeked[0] == 'B' && peeked[1] == 'E' && peeked[2] == 'T' && peeked[3] == 'W' && peeked[4] == 'E' && peeked[5] == 'E' && peeked[6] == 'N') { this.value = Operators.BETWEEN.value(); } else if (peeked.length == startsWithLen && new String(peeked).equals(Operators.STARTS_WITH.value())) { this.value = Operators.STARTS_WITH.value(); } else { throw new ParserException( "Expecting an operator (=, >, <, !=, BETWEEN, IN, STARTS_WITH), but found none. Peeked=>" + new String(peeked)); } read(this.value.length()); } @Override public String toString() { return " " + value + " "; } public String getOperator() { return value; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/ConstValue.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; /** * Constant value can be: * *

      *
    1. List of values (a,b,c) *
    2. Range of values (m AND n) *
    3. A value (x) *
    4. A value is either a string or a number *
    */ public class ConstValue extends AbstractNode { public enum SystemConsts { NULL("null"), NOT_NULL("not null"); private final String value; SystemConsts(String value) { this.value = value; } public String value() { return value; } } private static final String QUOTE = "\""; private Object value; private SystemConsts sysConsts; public ConstValue(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] peeked = peek(4); String sp = new String(peeked).trim(); // Read a constant value (number or a string) if (peeked[0] == '"' || peeked[0] == '\'') { this.value = readString(is); } else if (sp.toLowerCase().startsWith("not")) { this.value = SystemConsts.NOT_NULL.value(); sysConsts = SystemConsts.NOT_NULL; read(SystemConsts.NOT_NULL.value().length()); } else if (sp.equalsIgnoreCase(SystemConsts.NULL.value())) { this.value = SystemConsts.NULL.value(); sysConsts = SystemConsts.NULL; read(SystemConsts.NULL.value().length()); } else { this.value = readNumber(is); } } private String readNumber(InputStream is) throws Exception { StringBuilder sb = new StringBuilder(); while (is.available() > 0) { is.mark(1); char c = (char) is.read(); if (!isNumeric(c)) { is.reset(); break; } else { sb.append(c); } } return sb.toString().trim(); } /** * Reads an escaped string * * @throws Exception */ private String readString(InputStream is) throws Exception { char delim = (char) read(1)[0]; StringBuilder sb = new StringBuilder(); boolean valid = false; while (is.available() > 0) { char c = (char) is.read(); if (c == delim) { valid = true; break; } else if (c == '\\') { // read the next character as part of the value c = (char) is.read(); sb.append(c); } else { sb.append(c); } } if (!valid) { throw new ParserException( "String constant is not quoted with <" + delim + "> : " + sb.toString()); } return QUOTE + sb.toString() + QUOTE; } public Object getValue() { return value; } @Override public String toString() { return "" + value; } public String getUnquotedValue() { String result = toString(); if (result.length() >= 2 && result.startsWith(QUOTE) && result.endsWith(QUOTE)) { result = result.substring(1, result.length() - 1); } return result; } public boolean isSysConstant() { return this.sysConsts != null; } public SystemConsts getSysConstant() { return this.sysConsts; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/FunctionThrowingException.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; @FunctionalInterface public interface FunctionThrowingException { void accept(T t) throws Exception; } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/ListConst.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; import java.util.LinkedList; import java.util.List; /** List of constants */ public class ListConst extends AbstractNode { private List values; public ListConst(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { byte[] peeked = read(1); assertExpected(peeked, "("); this.values = readList(); } private List readList() throws Exception { List list = new LinkedList<>(); boolean valid = false; char c; StringBuilder sb = new StringBuilder(); while (is.available() > 0) { c = (char) is.read(); if (c == ')') { valid = true; break; } else if (c == ',') { list.add(sb.toString().trim()); sb = new StringBuilder(); } else { sb.append(c); } } list.add(sb.toString().trim()); if (!valid) { throw new ParserException("Expected ')' but never encountered in the stream"); } return list; } public List getList() { return values; } @Override public String toString() { return values.toString(); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/Name.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; /** Represents the name of the field to be searched against. */ public class Name extends AbstractNode { private String value; public Name(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { this.value = readToken(); } @Override public String toString() { return value; } public String getName() { return value; } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/ParserException.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; @SuppressWarnings("serial") public class ParserException extends Exception { public ParserException(String message) { super(message); } public ParserException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: es6-persistence/src/main/java/com/netflix/conductor/es6/dao/query/parser/internal/Range.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.InputStream; public class Range extends AbstractNode { private String low; private String high; public Range(InputStream is) throws ParserException { super(is); } @Override protected void _parse() throws Exception { this.low = readNumber(is); skipWhitespace(); byte[] peeked = read(3); assertExpected(peeked, "AND"); skipWhitespace(); String num = readNumber(is); if ("".equals(num)) { throw new ParserException("Missing the upper range value..."); } this.high = num; } private String readNumber(InputStream is) throws Exception { StringBuilder sb = new StringBuilder(); while (is.available() > 0) { is.mark(1); char c = (char) is.read(); if (!isNumeric(c)) { is.reset(); break; } else { sb.append(c); } } return sb.toString().trim(); } /** * @return the low */ public String getLow() { return low; } /** * @return the high */ public String getHigh() { return high; } @Override public String toString() { return low + " AND " + high; } } ================================================ FILE: es6-persistence/src/main/resources/mappings_docType_task.json ================================================ { "task": { "properties": { "correlationId": { "type": "keyword", "index": true }, "endTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, "executionTime": { "type": "long" }, "input": { "type": "text", "index": true }, "output": { "type": "text", "index": true }, "queueWaitTime": { "type": "long" }, "reasonForIncompletion": { "type": "keyword", "index": true }, "scheduledTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, "startTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, "status": { "type": "keyword", "index": true }, "taskDefName": { "type": "keyword", "index": true }, "taskId": { "type": "keyword", "index": true }, "taskType": { "type": "keyword", "index": true }, "updateTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, "workflowId": { "type": "keyword", "index": true }, "workflowType": { "type": "keyword", "index": true }, "domain": { "type": "keyword", "index": true } } } } ================================================ FILE: es6-persistence/src/main/resources/mappings_docType_workflow.json ================================================ { "workflow": { "properties": { "correlationId": { "type": "keyword", "index": true, "doc_values": true }, "endTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis", "doc_values": true }, "executionTime": { "type": "long", "doc_values": true }, "failedReferenceTaskNames": { "type": "text", "index": false }, "failedTaskNames": { "type": "text", "index": true }, "input": { "type": "text", "index": true }, "output": { "type": "text", "index": true }, "reasonForIncompletion": { "type": "keyword", "index": true, "doc_values": true }, "startTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis", "doc_values": true }, "status": { "type": "keyword", "index": true, "doc_values": true }, "updateTime": { "type": "date", "format": "strict_date_optional_time||epoch_millis", "doc_values": true }, "version": { "type": "long", "doc_values": true }, "workflowId": { "type": "keyword", "index": true, "doc_values": true }, "workflowType": { "type": "keyword", "index": true, "doc_values": true }, "rawJSON": { "type": "text", "index": false }, "event": { "type": "keyword", "index": true } } } } ================================================ FILE: es6-persistence/src/main/resources/template_event.json ================================================ { "order": 0, "template": "*event*", "settings": { "index": { "refresh_interval": "1s" } }, "mappings": { "event": { "properties": { "action": { "type": "keyword", "index": true }, "created": { "type": "long" }, "event": { "type": "keyword", "index": true }, "id": { "type": "keyword", "index": true }, "messageId": { "type": "keyword", "index": true }, "name": { "type": "keyword", "index": true }, "output": { "properties": { "workflowId": { "type": "keyword", "index": true } } }, "status": { "type": "keyword", "index": true } } } }, "aliases": {} } ================================================ FILE: es6-persistence/src/main/resources/template_message.json ================================================ { "order": 0, "template": "*message*", "settings": { "index": { "refresh_interval": "1s" } }, "mappings": { "message": { "properties": { "created": { "type": "long" }, "messageId": { "type": "keyword", "index": true }, "payload": { "type": "keyword", "index": true }, "queue": { "type": "keyword", "index": true } } } }, "aliases": {} } ================================================ FILE: es6-persistence/src/main/resources/template_task_log.json ================================================ { "order": 0, "template": "*task*log*", "settings": { "index": { "refresh_interval": "1s" } }, "mappings": { "task_log": { "properties": { "createdTime": { "type": "long" }, "log": { "type": "keyword", "index": true }, "taskId": { "type": "keyword", "index": true } } } }, "aliases": {} } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchDaoBaseTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.net.InetAddress; import java.util.concurrent.ExecutionException; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.springframework.retry.support.RetryTemplate; abstract class ElasticSearchDaoBaseTest extends ElasticSearchTest { protected TransportClient elasticSearchClient; protected ElasticSearchDAOV6 indexDAO; @Before public void setup() throws Exception { int mappedPort = container.getMappedPort(9300); properties.setUrl("tcp://localhost:" + mappedPort); Settings settings = Settings.builder().put("client.transport.ignore_cluster_name", true).build(); elasticSearchClient = new PreBuiltTransportClient(settings) .addTransportAddress( new TransportAddress( InetAddress.getByName("localhost"), mappedPort)); indexDAO = new ElasticSearchDAOV6( elasticSearchClient, new RetryTemplate(), properties, objectMapper); indexDAO.setup(); } @AfterClass public static void closeClient() { container.stop(); } @After public void tearDown() { deleteAllIndices(); if (elasticSearchClient != null) { elasticSearchClient.close(); } } private void deleteAllIndices() { ImmutableOpenMap indices = elasticSearchClient .admin() .cluster() .prepareState() .get() .getState() .getMetaData() .getIndices(); indices.forEach( cursor -> { try { elasticSearchClient .admin() .indices() .delete(new DeleteIndexRequest(cursor.value.getIndex().getName())) .get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } }); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDaoBaseTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import org.apache.http.HttpHost; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.junit.After; import org.junit.Before; import org.springframework.retry.support.RetryTemplate; abstract class ElasticSearchRestDaoBaseTest extends ElasticSearchTest { protected RestClient restClient; protected ElasticSearchRestDAOV6 indexDAO; @Before public void setup() throws Exception { String httpHostAddress = container.getHttpHostAddress(); String host = httpHostAddress.split(":")[0]; int port = Integer.parseInt(httpHostAddress.split(":")[1]); properties.setUrl("http://" + httpHostAddress); RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "http")); restClient = restClientBuilder.build(); indexDAO = new ElasticSearchRestDAOV6( restClientBuilder, new RetryTemplate(), properties, objectMapper); indexDAO.setup(); } @After public void tearDown() throws Exception { deleteAllIndices(); if (restClient != null) { restClient.close(); } } private void deleteAllIndices() throws IOException { Response beforeResponse = restClient.performRequest("GET", "/_cat/indices"); Reader streamReader = new InputStreamReader(beforeResponse.getEntity().getContent()); BufferedReader bufferedReader = new BufferedReader(streamReader); String line; while ((line = bufferedReader.readLine()) != null) { String[] fields = line.split("\\s"); String endpoint = String.format("/%s", fields[2]); restClient.performRequest("DELETE", endpoint); } } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.util.Map; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.es6.config.ElasticSearchProperties; import com.fasterxml.jackson.databind.ObjectMapper; @ContextConfiguration( classes = {TestObjectMapperConfiguration.class, ElasticSearchTest.TestConfiguration.class}) @RunWith(SpringRunner.class) @TestPropertySource( properties = {"conductor.indexing.enabled=true", "conductor.elasticsearch.version=6"}) abstract class ElasticSearchTest { @Configuration static class TestConfiguration { @Bean public ElasticSearchProperties elasticSearchProperties() { return new ElasticSearchProperties(); } } protected static final ElasticsearchContainer container = new ElasticsearchContainer( DockerImageName.parse( "docker.elastic.co/elasticsearch/elasticsearch-oss") .withTag("6.8.17")) // this should match the client version // Resolve issue with es container not starting on m1/m2 macs .withEnv(Map.of("bootstrap.system_call_filter", "false")); @Autowired protected ObjectMapper objectMapper; @Autowired protected ElasticSearchProperties properties; @BeforeClass public static void startServer() { container.start(); } @AfterClass public static void stopServer() { container.stop(); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/TestElasticSearchDAOV6.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.function.Supplier; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.junit.Test; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow.WorkflowStatus; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.es6.utils.TestUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; public class TestElasticSearchDAOV6 extends ElasticSearchDaoBaseTest { private static final SimpleDateFormat SIMPLE_DATE_FORMAT = new SimpleDateFormat("yyyyMMWW"); private static final String INDEX_PREFIX = "conductor"; private static final String WORKFLOW_DOC_TYPE = "workflow"; private static final String TASK_DOC_TYPE = "task"; private static final String MSG_DOC_TYPE = "message"; private static final String EVENT_DOC_TYPE = "event"; private static final String LOG_INDEX_PREFIX = "task_log"; @Test public void assertInitialSetup() { SIMPLE_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); String workflowIndex = INDEX_PREFIX + "_" + WORKFLOW_DOC_TYPE; String taskIndex = INDEX_PREFIX + "_" + TASK_DOC_TYPE; String taskLogIndex = INDEX_PREFIX + "_" + LOG_INDEX_PREFIX + "_" + SIMPLE_DATE_FORMAT.format(new Date()); String messageIndex = INDEX_PREFIX + "_" + MSG_DOC_TYPE + "_" + SIMPLE_DATE_FORMAT.format(new Date()); String eventIndex = INDEX_PREFIX + "_" + EVENT_DOC_TYPE + "_" + SIMPLE_DATE_FORMAT.format(new Date()); assertTrue("Index 'conductor_workflow' should exist", indexExists("conductor_workflow")); assertTrue("Index 'conductor_task' should exist", indexExists("conductor_task")); assertTrue("Index '" + taskLogIndex + "' should exist", indexExists(taskLogIndex)); assertTrue("Index '" + messageIndex + "' should exist", indexExists(messageIndex)); assertTrue("Index '" + eventIndex + "' should exist", indexExists(eventIndex)); assertTrue( "Mapping 'workflow' for index 'conductor' should exist", doesMappingExist(workflowIndex, WORKFLOW_DOC_TYPE)); assertTrue( "Mapping 'task' for index 'conductor' should exist", doesMappingExist(taskIndex, TASK_DOC_TYPE)); } private boolean indexExists(final String index) { IndicesExistsRequest request = new IndicesExistsRequest(index); try { return elasticSearchClient.admin().indices().exists(request).get().isExists(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } } private boolean doesMappingExist(final String index, final String mappingName) { GetMappingsRequest request = new GetMappingsRequest().indices(index); try { GetMappingsResponse response = elasticSearchClient.admin().indices().getMappings(request).get(); return response.getMappings().get(index).containsKey(mappingName); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } } @Test public void shouldIndexWorkflow() throws JsonProcessingException { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); assertWorkflowSummary(workflow.getWorkflowId(), workflow); } @Test public void shouldIndexWorkflowAsync() throws Exception { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.asyncIndexWorkflow(workflow).get(); assertWorkflowSummary(workflow.getWorkflowId(), workflow); } @Test public void shouldRemoveWorkflow() { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflows(workflow.getWorkflowId()), 1); assertEquals(1, workflows.size()); indexDAO.removeWorkflow(workflow.getWorkflowId()); workflows = tryFindResults(() -> searchWorkflows(workflow.getWorkflowId()), 0); assertTrue("Workflow was not removed.", workflows.isEmpty()); } @Test public void shouldAsyncRemoveWorkflow() throws Exception { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflows(workflow.getWorkflowId()), 1); assertEquals(1, workflows.size()); indexDAO.asyncRemoveWorkflow(workflow.getWorkflowId()).get(); workflows = tryFindResults(() -> searchWorkflows(workflow.getWorkflowId()), 0); assertTrue("Workflow was not removed.", workflows.isEmpty()); } @Test public void shouldUpdateWorkflow() throws JsonProcessingException { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); indexDAO.updateWorkflow( workflow.getWorkflowId(), new String[] {"status"}, new Object[] {WorkflowStatus.COMPLETED}); workflow.setStatus(WorkflowStatus.COMPLETED); assertWorkflowSummary(workflow.getWorkflowId(), workflow); } @Test public void shouldAsyncUpdateWorkflow() throws Exception { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); indexDAO.asyncUpdateWorkflow( workflow.getWorkflowId(), new String[] {"status"}, new Object[] {WorkflowStatus.FAILED}) .get(); workflow.setStatus(WorkflowStatus.FAILED); assertWorkflowSummary(workflow.getWorkflowId(), workflow); } @Test public void shouldIndexTask() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); List tasks = tryFindResults(() -> searchTasks(taskSummary)); assertEquals(taskSummary.getTaskId(), tasks.get(0)); } @Test public void shouldIndexTaskAsync() throws Exception { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.asyncIndexTask(taskSummary).get(); List tasks = tryFindResults(() -> searchTasks(taskSummary)); assertEquals(taskSummary.getTaskId(), tasks.get(0)); } @Test public void shouldRemoveTask() { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); TaskSummary taskSummary = TestUtils.loadTaskSnapshot( objectMapper, "task_summary", workflowSummary.getWorkflowId()); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.removeTask(workflowSummary.getWorkflowId(), taskSummary.getTaskId()); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertTrue("Task was not removed.", tasks.isEmpty()); } @Test public void shouldAsyncRemoveTask() throws Exception { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); TaskSummary taskSummary = TestUtils.loadTaskSnapshot( objectMapper, "task_summary", workflowSummary.getWorkflowId()); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.asyncRemoveTask(workflowSummary.getWorkflowId(), taskSummary.getTaskId()).get(); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertTrue("Task was not removed.", tasks.isEmpty()); } @Test public void shouldNotRemoveTaskWhenNotAssociatedWithWorkflow() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.removeTask("InvalidWorkflow", taskSummary.getTaskId()); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertFalse("Task was removed.", tasks.isEmpty()); } @Test public void shouldNotAsyncRemoveTaskWhenNotAssociatedWithWorkflow() throws Exception { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.asyncRemoveTask("InvalidWorkflow", taskSummary.getTaskId()).get(); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertFalse("Task was removed.", tasks.isEmpty()); } @Test public void shouldAddTaskExecutionLogs() { List logs = new ArrayList<>(); String taskId = uuid(); logs.add(createLog(taskId, "log1")); logs.add(createLog(taskId, "log2")); logs.add(createLog(taskId, "log3")); indexDAO.addTaskExecutionLogs(logs); List indexedLogs = tryFindResults(() -> indexDAO.getTaskExecutionLogs(taskId), 3); assertEquals(3, indexedLogs.size()); assertTrue("Not all logs was indexed", indexedLogs.containsAll(logs)); } @Test public void shouldAddTaskExecutionLogsAsync() throws Exception { List logs = new ArrayList<>(); String taskId = uuid(); logs.add(createLog(taskId, "log1")); logs.add(createLog(taskId, "log2")); logs.add(createLog(taskId, "log3")); indexDAO.asyncAddTaskExecutionLogs(logs).get(); List indexedLogs = tryFindResults(() -> indexDAO.getTaskExecutionLogs(taskId), 3); assertEquals(3, indexedLogs.size()); assertTrue("Not all logs was indexed", indexedLogs.containsAll(logs)); } @Test public void shouldAddMessage() { String queue = "queue"; Message message1 = new Message(uuid(), "payload1", null); Message message2 = new Message(uuid(), "payload2", null); indexDAO.addMessage(queue, message1); indexDAO.addMessage(queue, message2); List indexedMessages = tryFindResults(() -> indexDAO.getMessages(queue), 2); assertEquals(2, indexedMessages.size()); assertTrue( "Not all messages was indexed", indexedMessages.containsAll(Arrays.asList(message1, message2))); } @Test public void shouldAddEventExecution() { String event = "event"; EventExecution execution1 = createEventExecution(event); EventExecution execution2 = createEventExecution(event); indexDAO.addEventExecution(execution1); indexDAO.addEventExecution(execution2); List indexedExecutions = tryFindResults(() -> indexDAO.getEventExecutions(event), 2); assertEquals(2, indexedExecutions.size()); assertTrue( "Not all event executions was indexed", indexedExecutions.containsAll(Arrays.asList(execution1, execution2))); } @Test public void shouldAsyncAddEventExecution() throws Exception { String event = "event2"; EventExecution execution1 = createEventExecution(event); EventExecution execution2 = createEventExecution(event); indexDAO.asyncAddEventExecution(execution1).get(); indexDAO.asyncAddEventExecution(execution2).get(); List indexedExecutions = tryFindResults(() -> indexDAO.getEventExecutions(event), 2); assertEquals(2, indexedExecutions.size()); assertTrue( "Not all event executions was indexed", indexedExecutions.containsAll(Arrays.asList(execution1, execution2))); } @Test public void shouldAddIndexPrefixToIndexTemplate() throws Exception { String json = TestUtils.loadJsonResource("expected_template_task_log"); String content = indexDAO.loadTypeMappingSource("/template_task_log.json"); assertEquals(json, content); } @Test public void shouldCountWorkflows() { int counts = 1100; for (int i = 0; i < counts; i++) { WorkflowSummary workflow = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflow); } // wait for workflow to be indexed long result = tryGetCount(() -> getWorkflowCount("template_workflow", "RUNNING"), counts); assertEquals(counts, result); } @Test public void shouldFindWorkflow() { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflowSummary(workflowSummary.getWorkflowId()), 1); assertEquals(1, workflows.size()); assertEquals(workflowSummary, workflows.get(0)); } @Test public void shouldFindTask() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); List tasks = tryFindResults(() -> searchTaskSummary(taskSummary)); assertEquals(1, tasks.size()); assertEquals(taskSummary, tasks.get(0)); } private long tryGetCount(Supplier countFunction, int resultsCount) { long result = 0; for (int i = 0; i < 20; i++) { result = countFunction.get(); if (result == resultsCount) { return result; } try { Thread.sleep(100); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } return result; } // Get total workflow counts given the name and status private long getWorkflowCount(String workflowName, String status) { return indexDAO.getWorkflowCount( "status=\"" + status + "\" AND workflowType=\"" + workflowName + "\"", "*"); } private void assertWorkflowSummary(String workflowId, WorkflowSummary summary) throws JsonProcessingException { assertEquals(summary.getWorkflowType(), indexDAO.get(workflowId, "workflowType")); assertEquals(String.valueOf(summary.getVersion()), indexDAO.get(workflowId, "version")); assertEquals(summary.getWorkflowId(), indexDAO.get(workflowId, "workflowId")); assertEquals(summary.getCorrelationId(), indexDAO.get(workflowId, "correlationId")); assertEquals(summary.getStartTime(), indexDAO.get(workflowId, "startTime")); assertEquals(summary.getUpdateTime(), indexDAO.get(workflowId, "updateTime")); assertEquals(summary.getEndTime(), indexDAO.get(workflowId, "endTime")); assertEquals(summary.getStatus().name(), indexDAO.get(workflowId, "status")); assertEquals(summary.getInput(), indexDAO.get(workflowId, "input")); assertEquals(summary.getOutput(), indexDAO.get(workflowId, "output")); assertEquals( summary.getReasonForIncompletion(), indexDAO.get(workflowId, "reasonForIncompletion")); assertEquals( String.valueOf(summary.getExecutionTime()), indexDAO.get(workflowId, "executionTime")); assertEquals(summary.getEvent(), indexDAO.get(workflowId, "event")); assertEquals( summary.getFailedReferenceTaskNames(), indexDAO.get(workflowId, "failedReferenceTaskNames")); assertEquals( summary.getFailedTaskNames(), objectMapper.readValue(indexDAO.get(workflowId, "failedTaskNames"), Set.class)); } private List tryFindResults(Supplier> searchFunction) { return tryFindResults(searchFunction, 1); } private List tryFindResults(Supplier> searchFunction, int resultsCount) { List result = Collections.emptyList(); for (int i = 0; i < 20; i++) { result = searchFunction.get(); if (result.size() == resultsCount) { return result; } try { Thread.sleep(100); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } return result; } private List searchWorkflows(String workflowId) { return indexDAO.searchWorkflows( "", "workflowId:\"" + workflowId + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchWorkflowSummary(String workflowId) { return indexDAO.searchWorkflowSummary( "", "workflowId:\"" + workflowId + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchTasks(TaskSummary taskSummary) { return indexDAO.searchTasks( "", "workflowId:\"" + taskSummary.getWorkflowId() + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchTaskSummary(TaskSummary taskSummary) { return indexDAO.searchTaskSummary( "", "workflowId:\"" + taskSummary.getWorkflowId() + "\"", 0, 100, Collections.emptyList()) .getResults(); } private TaskExecLog createLog(String taskId, String log) { TaskExecLog taskExecLog = new TaskExecLog(log); taskExecLog.setTaskId(taskId); return taskExecLog; } private EventExecution createEventExecution(String event) { EventExecution execution = new EventExecution(uuid(), uuid()); execution.setName("name"); execution.setEvent(event); execution.setCreated(System.currentTimeMillis()); execution.setStatus(EventExecution.Status.COMPLETED); execution.setAction(EventHandler.Action.Type.start_workflow); execution.setOutput(ImmutableMap.of("a", 1, "b", 2, "c", 3)); return execution; } private String uuid() { return UUID.randomUUID().toString(); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/TestElasticSearchDAOV6Batch.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.util.HashMap; import java.util.concurrent.TimeUnit; import org.junit.Test; import org.springframework.test.context.TestPropertySource; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.fasterxml.jackson.core.JsonProcessingException; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @TestPropertySource(properties = "conductor.elasticsearch.indexBatchSize=2") public class TestElasticSearchDAOV6Batch extends ElasticSearchDaoBaseTest { @Test public void indexTaskWithBatchSizeTwo() { String correlationId = "some-correlation-id"; TaskSummary taskSummary = new TaskSummary(); taskSummary.setTaskId("some-task-id"); taskSummary.setWorkflowId("some-workflow-instance-id"); taskSummary.setTaskType("some-task-type"); taskSummary.setStatus(Status.FAILED); try { taskSummary.setInput( objectMapper.writeValueAsString( new HashMap() { { put("input_key", "input_value"); } })); } catch (JsonProcessingException e) { throw new RuntimeException(e); } taskSummary.setCorrelationId(correlationId); taskSummary.setTaskDefName("some-task-def-name"); taskSummary.setReasonForIncompletion("some-failure-reason"); indexDAO.indexTask(taskSummary); indexDAO.indexTask(taskSummary); await().atMost(5, TimeUnit.SECONDS) .untilAsserted( () -> { SearchResult result = indexDAO.searchTasks( "correlationId='" + correlationId + "'", "*", 0, 10000, null); assertTrue( "should return 1 or more search results", result.getResults().size() > 0); assertEquals( "taskId should match the indexed task", "some-task-id", result.getResults().get(0)); }); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/TestElasticSearchRestDAOV6.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Set; import java.util.TimeZone; import java.util.UUID; import java.util.function.Supplier; import org.junit.Test; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow.WorkflowStatus; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.es6.utils.TestUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; import static org.junit.Assert.*; public class TestElasticSearchRestDAOV6 extends ElasticSearchRestDaoBaseTest { private static final SimpleDateFormat SIMPLE_DATE_FORMAT = new SimpleDateFormat("yyyyMMWW"); private static final String INDEX_PREFIX = "conductor"; private static final String WORKFLOW_DOC_TYPE = "workflow"; private static final String TASK_DOC_TYPE = "task"; private static final String MSG_DOC_TYPE = "message"; private static final String EVENT_DOC_TYPE = "event"; private static final String LOG_INDEX_PREFIX = "task_log"; private boolean indexExists(final String index) throws IOException { return indexDAO.doesResourceExist("/" + index); } private boolean doesMappingExist(final String index, final String mappingName) throws IOException { return indexDAO.doesResourceExist("/" + index + "/_mapping/" + mappingName); } @Test public void assertInitialSetup() throws IOException { SIMPLE_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); String workflowIndex = INDEX_PREFIX + "_" + WORKFLOW_DOC_TYPE; String taskIndex = INDEX_PREFIX + "_" + TASK_DOC_TYPE; String taskLogIndex = INDEX_PREFIX + "_" + LOG_INDEX_PREFIX + "_" + SIMPLE_DATE_FORMAT.format(new Date()); String messageIndex = INDEX_PREFIX + "_" + MSG_DOC_TYPE + "_" + SIMPLE_DATE_FORMAT.format(new Date()); String eventIndex = INDEX_PREFIX + "_" + EVENT_DOC_TYPE + "_" + SIMPLE_DATE_FORMAT.format(new Date()); assertTrue("Index 'conductor_workflow' should exist", indexExists("conductor_workflow")); assertTrue("Index 'conductor_task' should exist", indexExists("conductor_task")); assertTrue("Index '" + taskLogIndex + "' should exist", indexExists(taskLogIndex)); assertTrue("Index '" + messageIndex + "' should exist", indexExists(messageIndex)); assertTrue("Index '" + eventIndex + "' should exist", indexExists(eventIndex)); assertTrue( "Mapping 'workflow' for index 'conductor' should exist", doesMappingExist(workflowIndex, WORKFLOW_DOC_TYPE)); assertTrue( "Mapping 'task' for index 'conductor' should exist", doesMappingExist(taskIndex, TASK_DOC_TYPE)); } @Test public void shouldIndexWorkflow() throws JsonProcessingException { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); assertWorkflowSummary(workflowSummary.getWorkflowId(), workflowSummary); } @Test public void shouldIndexWorkflowAsync() throws Exception { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.asyncIndexWorkflow(workflowSummary).get(); assertWorkflowSummary(workflowSummary.getWorkflowId(), workflowSummary); } @Test public void shouldRemoveWorkflow() { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); assertEquals(1, workflows.size()); indexDAO.removeWorkflow(workflowSummary.getWorkflowId()); workflows = tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 0); assertTrue("Workflow was not removed.", workflows.isEmpty()); } @Test public void shouldAsyncRemoveWorkflow() throws Exception { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); assertEquals(1, workflows.size()); indexDAO.asyncRemoveWorkflow(workflowSummary.getWorkflowId()).get(); workflows = tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 0); assertTrue("Workflow was not removed.", workflows.isEmpty()); } @Test public void shouldUpdateWorkflow() throws JsonProcessingException { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); indexDAO.updateWorkflow( workflowSummary.getWorkflowId(), new String[] {"status"}, new Object[] {WorkflowStatus.COMPLETED}); workflowSummary.setStatus(WorkflowStatus.COMPLETED); assertWorkflowSummary(workflowSummary.getWorkflowId(), workflowSummary); } @Test public void shouldAsyncUpdateWorkflow() throws Exception { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); indexDAO.asyncUpdateWorkflow( workflowSummary.getWorkflowId(), new String[] {"status"}, new Object[] {WorkflowStatus.FAILED}) .get(); workflowSummary.setStatus(WorkflowStatus.FAILED); assertWorkflowSummary(workflowSummary.getWorkflowId(), workflowSummary); } @Test public void shouldIndexTask() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); List tasks = tryFindResults(() -> searchTasks(taskSummary)); assertEquals(taskSummary.getTaskId(), tasks.get(0)); } @Test public void shouldIndexTaskAsync() throws Exception { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.asyncIndexTask(taskSummary).get(); List tasks = tryFindResults(() -> searchTasks(taskSummary)); assertEquals(taskSummary.getTaskId(), tasks.get(0)); } @Test public void shouldRemoveTask() { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); TaskSummary taskSummary = TestUtils.loadTaskSnapshot( objectMapper, "task_summary", workflowSummary.getWorkflowId()); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.removeTask(workflowSummary.getWorkflowId(), taskSummary.getTaskId()); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertTrue("Task was not removed.", tasks.isEmpty()); } @Test public void shouldAsyncRemoveTask() throws Exception { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed tryFindResults(() -> searchWorkflows(workflowSummary.getWorkflowId()), 1); TaskSummary taskSummary = TestUtils.loadTaskSnapshot( objectMapper, "task_summary", workflowSummary.getWorkflowId()); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.asyncRemoveTask(workflowSummary.getWorkflowId(), taskSummary.getTaskId()).get(); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertTrue("Task was not removed.", tasks.isEmpty()); } @Test public void shouldNotRemoveTaskWhenNotAssociatedWithWorkflow() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.removeTask("InvalidWorkflow", taskSummary.getTaskId()); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertFalse("Task was removed.", tasks.isEmpty()); } @Test public void shouldNotAsyncRemoveTaskWhenNotAssociatedWithWorkflow() throws Exception { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); // Wait for the task to be indexed List tasks = tryFindResults(() -> searchTasks(taskSummary), 1); indexDAO.asyncRemoveTask("InvalidWorkflow", taskSummary.getTaskId()).get(); tasks = tryFindResults(() -> searchTasks(taskSummary), 0); assertFalse("Task was removed.", tasks.isEmpty()); } @Test public void shouldAddTaskExecutionLogs() { List logs = new ArrayList<>(); String taskId = uuid(); logs.add(createLog(taskId, "log1")); logs.add(createLog(taskId, "log2")); logs.add(createLog(taskId, "log3")); indexDAO.addTaskExecutionLogs(logs); List indexedLogs = tryFindResults(() -> indexDAO.getTaskExecutionLogs(taskId), 3); assertEquals(3, indexedLogs.size()); assertTrue("Not all logs was indexed", indexedLogs.containsAll(logs)); } @Test public void shouldAddTaskExecutionLogsAsync() throws Exception { List logs = new ArrayList<>(); String taskId = uuid(); logs.add(createLog(taskId, "log1")); logs.add(createLog(taskId, "log2")); logs.add(createLog(taskId, "log3")); indexDAO.asyncAddTaskExecutionLogs(logs).get(); List indexedLogs = tryFindResults(() -> indexDAO.getTaskExecutionLogs(taskId), 3); assertEquals(3, indexedLogs.size()); assertTrue("Not all logs was indexed", indexedLogs.containsAll(logs)); } @Test public void shouldAddMessage() { String queue = "queue"; Message message1 = new Message(uuid(), "payload1", null); Message message2 = new Message(uuid(), "payload2", null); indexDAO.addMessage(queue, message1); indexDAO.addMessage(queue, message2); List indexedMessages = tryFindResults(() -> indexDAO.getMessages(queue), 2); assertEquals(2, indexedMessages.size()); assertTrue( "Not all messages was indexed", indexedMessages.containsAll(Arrays.asList(message1, message2))); } @Test public void shouldAddEventExecution() { String event = "event"; EventExecution execution1 = createEventExecution(event); EventExecution execution2 = createEventExecution(event); indexDAO.addEventExecution(execution1); indexDAO.addEventExecution(execution2); List indexedExecutions = tryFindResults(() -> indexDAO.getEventExecutions(event), 2); assertEquals(2, indexedExecutions.size()); assertTrue( "Not all event executions was indexed", indexedExecutions.containsAll(Arrays.asList(execution1, execution2))); } @Test public void shouldAsyncAddEventExecution() throws Exception { String event = "event2"; EventExecution execution1 = createEventExecution(event); EventExecution execution2 = createEventExecution(event); indexDAO.asyncAddEventExecution(execution1).get(); indexDAO.asyncAddEventExecution(execution2).get(); List indexedExecutions = tryFindResults(() -> indexDAO.getEventExecutions(event), 2); assertEquals(2, indexedExecutions.size()); assertTrue( "Not all event executions was indexed", indexedExecutions.containsAll(Arrays.asList(execution1, execution2))); } @Test public void shouldAddIndexPrefixToIndexTemplate() throws Exception { String json = TestUtils.loadJsonResource("expected_template_task_log"); String content = indexDAO.loadTypeMappingSource("/template_task_log.json"); assertEquals(json, content); } @Test public void shouldCountWorkflows() { int counts = 1100; for (int i = 0; i < counts; i++) { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); } // wait for workflow to be indexed long result = tryGetCount(() -> getWorkflowCount("template_workflow", "RUNNING"), counts); assertEquals(counts, result); } @Test public void shouldFindWorkflow() { WorkflowSummary workflowSummary = TestUtils.loadWorkflowSnapshot(objectMapper, "workflow_summary"); indexDAO.indexWorkflow(workflowSummary); // wait for workflow to be indexed List workflows = tryFindResults(() -> searchWorkflowSummary(workflowSummary.getWorkflowId()), 1); assertEquals(1, workflows.size()); assertEquals(workflowSummary, workflows.get(0)); } @Test public void shouldFindTask() { TaskSummary taskSummary = TestUtils.loadTaskSnapshot(objectMapper, "task_summary"); indexDAO.indexTask(taskSummary); List tasks = tryFindResults(() -> searchTaskSummary(taskSummary)); assertEquals(1, tasks.size()); assertEquals(taskSummary, tasks.get(0)); } private long tryGetCount(Supplier countFunction, int resultsCount) { long result = 0; for (int i = 0; i < 20; i++) { result = countFunction.get(); if (result == resultsCount) { return result; } try { Thread.sleep(100); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } return result; } // Get total workflow counts given the name and status private long getWorkflowCount(String workflowName, String status) { return indexDAO.getWorkflowCount( "status=\"" + status + "\" AND workflowType=\"" + workflowName + "\"", "*"); } private void assertWorkflowSummary(String workflowId, WorkflowSummary summary) throws JsonProcessingException { assertEquals(summary.getWorkflowType(), indexDAO.get(workflowId, "workflowType")); assertEquals(String.valueOf(summary.getVersion()), indexDAO.get(workflowId, "version")); assertEquals(summary.getWorkflowId(), indexDAO.get(workflowId, "workflowId")); assertEquals(summary.getCorrelationId(), indexDAO.get(workflowId, "correlationId")); assertEquals(summary.getStartTime(), indexDAO.get(workflowId, "startTime")); assertEquals(summary.getUpdateTime(), indexDAO.get(workflowId, "updateTime")); assertEquals(summary.getEndTime(), indexDAO.get(workflowId, "endTime")); assertEquals(summary.getStatus().name(), indexDAO.get(workflowId, "status")); assertEquals(summary.getInput(), indexDAO.get(workflowId, "input")); assertEquals(summary.getOutput(), indexDAO.get(workflowId, "output")); assertEquals( summary.getReasonForIncompletion(), indexDAO.get(workflowId, "reasonForIncompletion")); assertEquals( String.valueOf(summary.getExecutionTime()), indexDAO.get(workflowId, "executionTime")); assertEquals(summary.getEvent(), indexDAO.get(workflowId, "event")); assertEquals( summary.getFailedReferenceTaskNames(), indexDAO.get(workflowId, "failedReferenceTaskNames")); assertEquals( summary.getFailedTaskNames(), objectMapper.readValue(indexDAO.get(workflowId, "failedTaskNames"), Set.class)); } private List tryFindResults(Supplier> searchFunction) { return tryFindResults(searchFunction, 1); } private List tryFindResults(Supplier> searchFunction, int resultsCount) { List result = Collections.emptyList(); for (int i = 0; i < 20; i++) { result = searchFunction.get(); if (result.size() == resultsCount) { return result; } try { Thread.sleep(100); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } return result; } private List searchWorkflows(String workflowId) { return indexDAO.searchWorkflows( "", "workflowId:\"" + workflowId + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchWorkflowSummary(String workflowId) { return indexDAO.searchWorkflowSummary( "", "workflowId:\"" + workflowId + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchWorkflows(String workflowName, String status) { List sortOptions = new ArrayList<>(); sortOptions.add("startTime:DESC"); return indexDAO.searchWorkflows( "status=\"" + status + "\" AND workflowType=\"" + workflowName + "\"", "*", 0, 1000, sortOptions) .getResults(); } private List searchTasks(TaskSummary taskSummary) { return indexDAO.searchTasks( "", "workflowId:\"" + taskSummary.getWorkflowId() + "\"", 0, 100, Collections.emptyList()) .getResults(); } private List searchTaskSummary(TaskSummary taskSummary) { return indexDAO.searchTaskSummary( "", "workflowId:\"" + taskSummary.getWorkflowId() + "\"", 0, 100, Collections.emptyList()) .getResults(); } private TaskExecLog createLog(String taskId, String log) { TaskExecLog taskExecLog = new TaskExecLog(log); taskExecLog.setTaskId(taskId); return taskExecLog; } private EventExecution createEventExecution(String event) { EventExecution execution = new EventExecution(uuid(), uuid()); execution.setName("name"); execution.setEvent(event); execution.setCreated(System.currentTimeMillis()); execution.setStatus(EventExecution.Status.COMPLETED); execution.setAction(EventHandler.Action.Type.start_workflow); execution.setOutput(ImmutableMap.of("a", 1, "b", 2, "c", 3)); return execution; } private String uuid() { return UUID.randomUUID().toString(); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/TestElasticSearchRestDAOV6Batch.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.index; import java.util.HashMap; import java.util.concurrent.TimeUnit; import org.junit.Test; import org.springframework.test.context.TestPropertySource; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.fasterxml.jackson.core.JsonProcessingException; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @TestPropertySource(properties = "conductor.elasticsearch.indexBatchSize=2") public class TestElasticSearchRestDAOV6Batch extends ElasticSearchRestDaoBaseTest { @Test public void indexTaskWithBatchSizeTwo() { String correlationId = "some-correlation-id"; TaskSummary taskSummary = new TaskSummary(); taskSummary.setTaskId("some-task-id"); taskSummary.setWorkflowId("some-workflow-instance-id"); taskSummary.setTaskType("some-task-type"); taskSummary.setStatus(Status.FAILED); try { taskSummary.setInput( objectMapper.writeValueAsString( new HashMap() { { put("input_key", "input_value"); } })); } catch (JsonProcessingException e) { throw new RuntimeException(e); } taskSummary.setCorrelationId(correlationId); taskSummary.setTaskDefName("some-task-def-name"); taskSummary.setReasonForIncompletion("some-failure-reason"); indexDAO.indexTask(taskSummary); indexDAO.indexTask(taskSummary); await().atMost(5, TimeUnit.SECONDS) .untilAsserted( () -> { SearchResult result = indexDAO.searchTasks( "correlationId='" + correlationId + "'", "*", 0, 10000, null); assertTrue( "should return 1 or more search results", result.getResults().size() > 0); assertEquals( "taskId should match the indexed task", "some-task-id", result.getResults().get(0)); }); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/TestExpression.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.InputStream; import org.junit.Test; import com.netflix.conductor.es6.dao.query.parser.internal.ConstValue; import com.netflix.conductor.es6.dao.query.parser.internal.TestAbstractParser; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class TestExpression extends TestAbstractParser { @Test public void test() throws Exception { String test = "type='IMAGE' AND subType ='sdp' AND (metadata.width > 50 OR metadata.height > 50)"; InputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(test.getBytes())); Expression expression = new Expression(inputStream); assertTrue(expression.isBinaryExpr()); assertNull(expression.getGroupedExpression()); assertNotNull(expression.getNameValue()); NameValue nameValue = expression.getNameValue(); assertEquals("type", nameValue.getName().getName()); assertEquals("=", nameValue.getOp().getOperator()); assertEquals("\"IMAGE\"", nameValue.getValue().getValue()); Expression rightHandSide = expression.getRightHandSide(); assertNotNull(rightHandSide); assertTrue(rightHandSide.isBinaryExpr()); nameValue = rightHandSide.getNameValue(); assertNotNull(nameValue); // subType = sdp assertNull(rightHandSide.getGroupedExpression()); assertEquals("subType", nameValue.getName().getName()); assertEquals("=", nameValue.getOp().getOperator()); assertEquals("\"sdp\"", nameValue.getValue().getValue()); assertEquals("AND", rightHandSide.getOperator().getOperator()); rightHandSide = rightHandSide.getRightHandSide(); assertNotNull(rightHandSide); assertFalse(rightHandSide.isBinaryExpr()); GroupedExpression groupedExpression = rightHandSide.getGroupedExpression(); assertNotNull(groupedExpression); expression = groupedExpression.getExpression(); assertNotNull(expression); assertTrue(expression.isBinaryExpr()); nameValue = expression.getNameValue(); assertNotNull(nameValue); assertEquals("metadata.width", nameValue.getName().getName()); assertEquals(">", nameValue.getOp().getOperator()); assertEquals("50", nameValue.getValue().getValue()); assertEquals("OR", expression.getOperator().getOperator()); rightHandSide = expression.getRightHandSide(); assertNotNull(rightHandSide); assertFalse(rightHandSide.isBinaryExpr()); nameValue = rightHandSide.getNameValue(); assertNotNull(nameValue); assertEquals("metadata.height", nameValue.getName().getName()); assertEquals(">", nameValue.getOp().getOperator()); assertEquals("50", nameValue.getValue().getValue()); } @Test public void testWithSysConstants() throws Exception { String test = "type='IMAGE' AND subType ='sdp' AND description IS null"; InputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(test.getBytes())); Expression expression = new Expression(inputStream); assertTrue(expression.isBinaryExpr()); assertNull(expression.getGroupedExpression()); assertNotNull(expression.getNameValue()); NameValue nameValue = expression.getNameValue(); assertEquals("type", nameValue.getName().getName()); assertEquals("=", nameValue.getOp().getOperator()); assertEquals("\"IMAGE\"", nameValue.getValue().getValue()); Expression rightHandSide = expression.getRightHandSide(); assertNotNull(rightHandSide); assertTrue(rightHandSide.isBinaryExpr()); nameValue = rightHandSide.getNameValue(); assertNotNull(nameValue); // subType = sdp assertNull(rightHandSide.getGroupedExpression()); assertEquals("subType", nameValue.getName().getName()); assertEquals("=", nameValue.getOp().getOperator()); assertEquals("\"sdp\"", nameValue.getValue().getValue()); assertEquals("AND", rightHandSide.getOperator().getOperator()); rightHandSide = rightHandSide.getRightHandSide(); assertNotNull(rightHandSide); assertFalse(rightHandSide.isBinaryExpr()); GroupedExpression groupedExpression = rightHandSide.getGroupedExpression(); assertNull(groupedExpression); nameValue = rightHandSide.getNameValue(); assertNotNull(nameValue); assertEquals("description", nameValue.getName().getName()); assertEquals("IS", nameValue.getOp().getOperator()); ConstValue constValue = nameValue.getValue(); assertNotNull(constValue); assertEquals(constValue.getSysConstant(), ConstValue.SystemConsts.NULL); test = "description IS not null"; inputStream = new BufferedInputStream(new ByteArrayInputStream(test.getBytes())); expression = new Expression(inputStream); nameValue = expression.getNameValue(); assertNotNull(nameValue); assertEquals("description", nameValue.getName().getName()); assertEquals("IS", nameValue.getOp().getOperator()); constValue = nameValue.getValue(); assertNotNull(constValue); assertEquals(constValue.getSysConstant(), ConstValue.SystemConsts.NOT_NULL); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/internal/TestAbstractParser.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.InputStream; public abstract class TestAbstractParser { protected InputStream getInputStream(String expression) { return new BufferedInputStream(new ByteArrayInputStream(expression.getBytes())); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/internal/TestBooleanOp.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class TestBooleanOp extends TestAbstractParser { @Test public void test() throws Exception { String[] tests = new String[] {"AND", "OR"}; for (String test : tests) { BooleanOp name = new BooleanOp(getInputStream(test)); String nameVal = name.getOperator(); assertNotNull(nameVal); assertEquals(test, nameVal); } } @Test(expected = ParserException.class) public void testInvalid() throws Exception { String test = "<"; BooleanOp name = new BooleanOp(getInputStream(test)); String nameVal = name.getOperator(); assertNotNull(nameVal); assertEquals(test, nameVal); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/internal/TestComparisonOp.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class TestComparisonOp extends TestAbstractParser { @Test public void test() throws Exception { String[] tests = new String[] {"<", ">", "=", "!=", "IN", "BETWEEN", "STARTS_WITH"}; for (String test : tests) { ComparisonOp name = new ComparisonOp(getInputStream(test)); String nameVal = name.getOperator(); assertNotNull(nameVal); assertEquals(test, nameVal); } } @Test(expected = ParserException.class) public void testInvalidOp() throws Exception { String test = "AND"; ComparisonOp name = new ComparisonOp(getInputStream(test)); String nameVal = name.getOperator(); assertNotNull(nameVal); assertEquals(test, nameVal); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/internal/TestConstValue.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import java.util.List; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class TestConstValue extends TestAbstractParser { @Test public void testStringConst() throws Exception { String test = "'string value'"; String expected = test.replaceAll( "'", "\""); // Quotes are removed but then the result is double quoted. ConstValue constValue = new ConstValue(getInputStream(test)); assertNotNull(constValue.getValue()); assertEquals(expected, constValue.getValue()); assertTrue(constValue.getValue() instanceof String); test = "\"string value\""; constValue = new ConstValue(getInputStream(test)); assertNotNull(constValue.getValue()); assertEquals(expected, constValue.getValue()); assertTrue(constValue.getValue() instanceof String); } @Test public void testSystemConst() throws Exception { String test = "null"; ConstValue constValue = new ConstValue(getInputStream(test)); assertNotNull(constValue.getValue()); assertTrue(constValue.getValue() instanceof String); assertEquals(constValue.getSysConstant(), ConstValue.SystemConsts.NULL); test = "not null"; constValue = new ConstValue(getInputStream(test)); assertNotNull(constValue.getValue()); assertEquals(constValue.getSysConstant(), ConstValue.SystemConsts.NOT_NULL); } @Test(expected = ParserException.class) public void testInvalid() throws Exception { String test = "'string value"; new ConstValue(getInputStream(test)); } @Test public void testNumConst() throws Exception { String test = "12345.89"; ConstValue cv = new ConstValue(getInputStream(test)); assertNotNull(cv.getValue()); assertTrue( cv.getValue() instanceof String); // Numeric values are stored as string as we are just passing thru // them to ES assertEquals(test, cv.getValue()); } @Test public void testRange() throws Exception { String test = "50 AND 100"; Range range = new Range(getInputStream(test)); assertEquals("50", range.getLow()); assertEquals("100", range.getHigh()); } @Test(expected = ParserException.class) public void testBadRange() throws Exception { String test = "50 AND"; new Range(getInputStream(test)); } @Test public void testArray() throws Exception { String test = "(1, 3, 'name', 'value2')"; ListConst listConst = new ListConst(getInputStream(test)); List list = listConst.getList(); assertEquals(4, list.size()); assertTrue(list.contains("1")); assertEquals("'value2'", list.get(3)); // Values are preserved as it is... } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/dao/query/parser/internal/TestName.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.dao.query.parser.internal; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class TestName extends TestAbstractParser { @Test public void test() throws Exception { String test = "metadata.en_US.lang "; Name name = new Name(getInputStream(test)); String nameVal = name.getName(); assertNotNull(nameVal); assertEquals(test.trim(), nameVal); } } ================================================ FILE: es6-persistence/src/test/java/com/netflix/conductor/es6/utils/TestUtils.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.es6.utils; import java.nio.charset.StandardCharsets; import org.apache.commons.io.FileUtils; import org.springframework.util.ResourceUtils; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.utils.IDGenerator; import com.fasterxml.jackson.databind.ObjectMapper; public class TestUtils { private static final String WORKFLOW_INSTANCE_ID_PLACEHOLDER = "WORKFLOW_INSTANCE_ID"; public static WorkflowSummary loadWorkflowSnapshot( ObjectMapper objectMapper, String resourceFileName) { try { String content = loadJsonResource(resourceFileName); String workflowId = new IDGenerator().generate(); content = content.replace(WORKFLOW_INSTANCE_ID_PLACEHOLDER, workflowId); return objectMapper.readValue(content, WorkflowSummary.class); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } public static TaskSummary loadTaskSnapshot(ObjectMapper objectMapper, String resourceFileName) { try { String content = loadJsonResource(resourceFileName); String workflowId = new IDGenerator().generate(); content = content.replace(WORKFLOW_INSTANCE_ID_PLACEHOLDER, workflowId); return objectMapper.readValue(content, TaskSummary.class); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } public static TaskSummary loadTaskSnapshot( ObjectMapper objectMapper, String resourceFileName, String workflowId) { try { String content = loadJsonResource(resourceFileName); content = content.replace(WORKFLOW_INSTANCE_ID_PLACEHOLDER, workflowId); return objectMapper.readValue(content, TaskSummary.class); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } public static String loadJsonResource(String resourceFileName) { try { return FileUtils.readFileToString( ResourceUtils.getFile("classpath:" + resourceFileName + ".json"), StandardCharsets.UTF_8); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } } ================================================ FILE: es6-persistence/src/test/resources/expected_template_task_log.json ================================================ { "order": 0, "template": "*conductor_task*log*", "settings": { "index": { "refresh_interval": "1s" } }, "mappings": { "task_log": { "properties": { "createdTime": { "type": "long" }, "log": { "type": "keyword", "index": true }, "taskId": { "type": "keyword", "index": true } } } }, "aliases": {} } ================================================ FILE: es6-persistence/src/test/resources/task_summary.json ================================================ { "taskId": "9dea4567-0240-4eab-bde8-99f4535ea3fc", "taskDefName": "templated_task", "taskType": "templated_task", "workflowId": "WORKFLOW_INSTANCE_ID", "workflowType": "template_workflow", "correlationId": "testTaskDefTemplate", "scheduledTime": "2021-08-22T05:18:25.121Z", "startTime": "0", "endTime": "0", "updateTime": "2021-08-23T00:18:25.121Z", "status": "SCHEDULED", "workflowPriority": 1, "queueWaitTime": 0, "executionTime": 0, "input": "{http_request={method=GET, vipStack=test_stack, body={requestDetails={key1=value1, key2=42}, outputPath=s3://bucket/outputPath, inputPaths=[file://path1, file://path2]}, uri=/get/something}}" } ================================================ FILE: es6-persistence/src/test/resources/workflow_summary.json ================================================ { "workflowType": "template_workflow", "version": 1, "workflowId": "WORKFLOW_INSTANCE_ID", "priority": 1, "correlationId": "testTaskDefTemplate", "startTime": 1534983505050, "updateTime": 1534983505131, "endTime": 0, "status": "RUNNING", "input": "{path1=file://path1, path2=file://path2, requestDetails={key1=value1, key2=42}, outputPath=s3://bucket/outputPath}" } ================================================ FILE: family.properties ================================================ generation=1 ================================================ FILE: gradle/wrapper/gradle-wrapper.properties ================================================ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists distributionSha256Sum=a01b6587e15fe7ed120a0ee299c25982a1eee045abd6a9dd5e216b2f628ef9ac distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists ================================================ FILE: gradlew ================================================ #!/bin/sh # # Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ############################################################################## # # Gradle start up script for POSIX generated by Gradle. # # Important for running: # # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is # noncompliant, but you have some other compliant shell such as ksh or # bash, then to run this script, type that shell name before the whole # command line, like: # # ksh Gradle # # Busybox and similar reduced shells will NOT work, because this script # requires all of these POSIX shell features: # * functions; # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», # «${var#prefix}», «${var%suffix}», and «$( cmd )»; # * compound commands having a testable exit status, especially «case»; # * various built-in commands including «command», «set», and «ulimit». # # Important for patching: # # (2) This script targets any POSIX shell, so it avoids extensions provided # by Bash, Ksh, etc; in particular arrays are avoided. # # The "traditional" practice of packing multiple parameters into a # space-separated string is a well documented source of bugs and security # problems, so this is (mostly) avoided, by progressively accumulating # options in "$@", and eventually passing that to Java. # # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; # see the in-line comments for details. # # There are tweaks for specific operating systems such as AIX, CygWin, # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. # ############################################################################## # Attempt to set APP_HOME # Resolve links: $0 may be a link app_path=$0 # Need this for daisy-chained symlinks. while APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path [ -h "$app_path" ] do ls=$( ls -ld "$app_path" ) link=${ls#*' -> '} case $link in #( /*) app_path=$link ;; #( *) app_path=$APP_HOME$link ;; esac done APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum warn () { echo "$*" } >&2 die () { echo echo "$*" echo exit 1 } >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false case "$( uname )" in #( CYGWIN* ) cygwin=true ;; #( Darwin* ) darwin=true ;; #( MSYS* | MINGW* ) msys=true ;; #( NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD=$JAVA_HOME/jre/sh/java else JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac fi # Collect all arguments for the java command, stacking in reverse order: # * args from the command line # * the main class name # * -classpath # * -D...appname settings # * --module-path (only if needed) # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) # Now convert the arguments - kludge to limit ourselves to /bin/sh for arg do if case $arg in #( -*) false ;; # don't mess with options #( /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath [ -e "$t" ] ;; #( *) false ;; esac then arg=$( cygpath --path --ignore --mixed "$arg" ) fi # Roll the args list around exactly as many times as the number of # args, so each arg winds up back in the position where it started, but # possibly modified. # # NB: a `for` loop captures its iteration list before it begins, so # changing the positional parameters here affects neither the number of # iterations, nor the values presented in `arg`. shift # remove old arg set -- "$@" "$arg" # push replacement arg done fi # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in # double quotes to make sure that they get re-expanded; and # * put everything else in single quotes, so that it's not re-expanded. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ org.gradle.wrapper.GradleWrapperMain \ "$@" # Stop when "xargs" is not available. if ! command -v xargs >/dev/null 2>&1 then die "xargs is not available" fi # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. # # In Bash we could simply go: # # readarray ARGS < <( xargs -n1 <<<"$var" ) && # set -- "${ARGS[@]}" "$@" # # but POSIX shell has neither arrays nor command substitution, so instead we # post-process each arg (as a line of input to sed) to backslash-escape any # character that might be a shell metacharacter, then use eval to reverse # that process (while maintaining the separation between arguments), and wrap # the whole thing up as a single "set" statement. # # This will of course break if any of these variables contains a newline or # an unmatched quote. # eval "set -- $( printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | xargs -n1 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | tr '\n' ' ' )" '"$@"' exec "$JAVACMD" "$@" ================================================ FILE: gradlew.bat ================================================ @rem @rem Copyright 2015 the original author or authors. @rem @rem Licensed under the Apache License, Version 2.0 (the "License"); @rem you may not use this file except in compliance with the License. @rem You may obtain a copy of the License at @rem @rem https://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, software @rem distributed under the License is distributed on an "AS IS" BASIS, @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Resolve any "." and ".." in APP_HOME to make it shorter. for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! set EXIT_CODE=%ERRORLEVEL% if %EXIT_CODE% equ 0 set EXIT_CODE=1 if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal :omega ================================================ FILE: grpc/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ buildscript { dependencies { classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.19' } } plugins { id 'java' id 'idea' id "com.google.protobuf" version "0.8.19" } repositories{ maven { url "https://mvnrepository.com/artifact" } } dependencies { implementation project(':conductor-common') implementation "com.google.protobuf:protobuf-java:${revProtoBuf}" implementation "io.grpc:grpc-protobuf:${revGrpc}" implementation "io.grpc:grpc-stub:${revGrpc}" implementation "javax.annotation:javax.annotation-api:1.3.2" } def artifactName = 'com.google.protobuf:protoc:3.14.0:osx-x86_64' switch (org.gradle.internal.os.OperatingSystem.current()) { case org.gradle.internal.os.OperatingSystem.LINUX: artifactName = "com.google.protobuf:protoc:3.21.12" break; case org.gradle.internal.os.OperatingSystem.MAC_OS: artifactName = "com.google.protobuf:protoc:3.14.0:osx-x86_64" break; case org.gradle.internal.os.OperatingSystem.WINDOWS: artifactName = "com.google.protobuf:protoc:3.21.12" break; } protobuf { protoc { artifact = artifactName } plugins { grpc { artifact = "io.grpc:protoc-gen-grpc-java:${revGrpc}" } } generateProtoTasks { processResources.dependsOn extractProto all()*.plugins { grpc {} } } } idea { module { sourceDirs += file("${projectDir}/build/generated/source/proto/main/java"); sourceDirs += file("${projectDir}/build/generated/source/proto/main/grpc"); } } sourceSets { main { java { srcDir 'build/generated/source/proto/main/java' srcDir 'build/generated/source/proto/main/grpc' } } } compileJava.dependsOn(tasks.getByPath(':conductor-common:protogen')) ================================================ FILE: grpc/src/main/java/com/netflix/conductor/grpc/AbstractProtoMapper.java ================================================ package com.netflix.conductor.grpc; import com.google.protobuf.Any; import com.google.protobuf.Value; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.DynamicForkJoinTask; import com.netflix.conductor.common.metadata.workflow.DynamicForkJoinTaskList; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDefSummary; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.proto.DynamicForkJoinTaskListPb; import com.netflix.conductor.proto.DynamicForkJoinTaskPb; import com.netflix.conductor.proto.EventExecutionPb; import com.netflix.conductor.proto.EventHandlerPb; import com.netflix.conductor.proto.PollDataPb; import com.netflix.conductor.proto.RerunWorkflowRequestPb; import com.netflix.conductor.proto.SkipTaskRequestPb; import com.netflix.conductor.proto.StartWorkflowRequestPb; import com.netflix.conductor.proto.SubWorkflowParamsPb; import com.netflix.conductor.proto.TaskDefPb; import com.netflix.conductor.proto.TaskExecLogPb; import com.netflix.conductor.proto.TaskPb; import com.netflix.conductor.proto.TaskResultPb; import com.netflix.conductor.proto.TaskSummaryPb; import com.netflix.conductor.proto.WorkflowDefPb; import com.netflix.conductor.proto.WorkflowDefSummaryPb; import com.netflix.conductor.proto.WorkflowPb; import com.netflix.conductor.proto.WorkflowSummaryPb; import com.netflix.conductor.proto.WorkflowTaskPb; import java.lang.IllegalArgumentException; import java.lang.Object; import java.lang.String; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Generated; @Generated("com.netflix.conductor.annotationsprocessor.protogen") public abstract class AbstractProtoMapper { public DynamicForkJoinTaskPb.DynamicForkJoinTask toProto(DynamicForkJoinTask from) { DynamicForkJoinTaskPb.DynamicForkJoinTask.Builder to = DynamicForkJoinTaskPb.DynamicForkJoinTask.newBuilder(); if (from.getTaskName() != null) { to.setTaskName( from.getTaskName() ); } if (from.getWorkflowName() != null) { to.setWorkflowName( from.getWorkflowName() ); } if (from.getReferenceName() != null) { to.setReferenceName( from.getReferenceName() ); } for (Map.Entry pair : from.getInput().entrySet()) { to.putInput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getType() != null) { to.setType( from.getType() ); } return to.build(); } public DynamicForkJoinTask fromProto(DynamicForkJoinTaskPb.DynamicForkJoinTask from) { DynamicForkJoinTask to = new DynamicForkJoinTask(); to.setTaskName( from.getTaskName() ); to.setWorkflowName( from.getWorkflowName() ); to.setReferenceName( from.getReferenceName() ); Map inputMap = new HashMap(); for (Map.Entry pair : from.getInputMap().entrySet()) { inputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInput(inputMap); to.setType( from.getType() ); return to; } public DynamicForkJoinTaskListPb.DynamicForkJoinTaskList toProto(DynamicForkJoinTaskList from) { DynamicForkJoinTaskListPb.DynamicForkJoinTaskList.Builder to = DynamicForkJoinTaskListPb.DynamicForkJoinTaskList.newBuilder(); for (DynamicForkJoinTask elem : from.getDynamicTasks()) { to.addDynamicTasks( toProto(elem) ); } return to.build(); } public DynamicForkJoinTaskList fromProto( DynamicForkJoinTaskListPb.DynamicForkJoinTaskList from) { DynamicForkJoinTaskList to = new DynamicForkJoinTaskList(); to.setDynamicTasks( from.getDynamicTasksList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); return to; } public EventExecutionPb.EventExecution toProto(EventExecution from) { EventExecutionPb.EventExecution.Builder to = EventExecutionPb.EventExecution.newBuilder(); if (from.getId() != null) { to.setId( from.getId() ); } if (from.getMessageId() != null) { to.setMessageId( from.getMessageId() ); } if (from.getName() != null) { to.setName( from.getName() ); } if (from.getEvent() != null) { to.setEvent( from.getEvent() ); } to.setCreated( from.getCreated() ); if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } if (from.getAction() != null) { to.setAction( toProto( from.getAction() ) ); } for (Map.Entry pair : from.getOutput().entrySet()) { to.putOutput( pair.getKey(), toProto( pair.getValue() ) ); } return to.build(); } public EventExecution fromProto(EventExecutionPb.EventExecution from) { EventExecution to = new EventExecution(); to.setId( from.getId() ); to.setMessageId( from.getMessageId() ); to.setName( from.getName() ); to.setEvent( from.getEvent() ); to.setCreated( from.getCreated() ); to.setStatus( fromProto( from.getStatus() ) ); to.setAction( fromProto( from.getAction() ) ); Map outputMap = new HashMap(); for (Map.Entry pair : from.getOutputMap().entrySet()) { outputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutput(outputMap); return to; } public EventExecutionPb.EventExecution.Status toProto(EventExecution.Status from) { EventExecutionPb.EventExecution.Status to; switch (from) { case IN_PROGRESS: to = EventExecutionPb.EventExecution.Status.IN_PROGRESS; break; case COMPLETED: to = EventExecutionPb.EventExecution.Status.COMPLETED; break; case FAILED: to = EventExecutionPb.EventExecution.Status.FAILED; break; case SKIPPED: to = EventExecutionPb.EventExecution.Status.SKIPPED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public EventExecution.Status fromProto(EventExecutionPb.EventExecution.Status from) { EventExecution.Status to; switch (from) { case IN_PROGRESS: to = EventExecution.Status.IN_PROGRESS; break; case COMPLETED: to = EventExecution.Status.COMPLETED; break; case FAILED: to = EventExecution.Status.FAILED; break; case SKIPPED: to = EventExecution.Status.SKIPPED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public EventHandlerPb.EventHandler toProto(EventHandler from) { EventHandlerPb.EventHandler.Builder to = EventHandlerPb.EventHandler.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getEvent() != null) { to.setEvent( from.getEvent() ); } if (from.getCondition() != null) { to.setCondition( from.getCondition() ); } for (EventHandler.Action elem : from.getActions()) { to.addActions( toProto(elem) ); } to.setActive( from.isActive() ); if (from.getEvaluatorType() != null) { to.setEvaluatorType( from.getEvaluatorType() ); } return to.build(); } public EventHandler fromProto(EventHandlerPb.EventHandler from) { EventHandler to = new EventHandler(); to.setName( from.getName() ); to.setEvent( from.getEvent() ); to.setCondition( from.getCondition() ); to.setActions( from.getActionsList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); to.setActive( from.getActive() ); to.setEvaluatorType( from.getEvaluatorType() ); return to; } public EventHandlerPb.EventHandler.StartWorkflow toProto(EventHandler.StartWorkflow from) { EventHandlerPb.EventHandler.StartWorkflow.Builder to = EventHandlerPb.EventHandler.StartWorkflow.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getVersion() != null) { to.setVersion( from.getVersion() ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } for (Map.Entry pair : from.getInput().entrySet()) { to.putInput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getInputMessage() != null) { to.setInputMessage( toProto( from.getInputMessage() ) ); } to.putAllTaskToDomain( from.getTaskToDomain() ); return to.build(); } public EventHandler.StartWorkflow fromProto(EventHandlerPb.EventHandler.StartWorkflow from) { EventHandler.StartWorkflow to = new EventHandler.StartWorkflow(); to.setName( from.getName() ); to.setVersion( from.getVersion() ); to.setCorrelationId( from.getCorrelationId() ); Map inputMap = new HashMap(); for (Map.Entry pair : from.getInputMap().entrySet()) { inputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInput(inputMap); if (from.hasInputMessage()) { to.setInputMessage( fromProto( from.getInputMessage() ) ); } to.setTaskToDomain( from.getTaskToDomainMap() ); return to; } public EventHandlerPb.EventHandler.TaskDetails toProto(EventHandler.TaskDetails from) { EventHandlerPb.EventHandler.TaskDetails.Builder to = EventHandlerPb.EventHandler.TaskDetails.newBuilder(); if (from.getWorkflowId() != null) { to.setWorkflowId( from.getWorkflowId() ); } if (from.getTaskRefName() != null) { to.setTaskRefName( from.getTaskRefName() ); } for (Map.Entry pair : from.getOutput().entrySet()) { to.putOutput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getOutputMessage() != null) { to.setOutputMessage( toProto( from.getOutputMessage() ) ); } if (from.getTaskId() != null) { to.setTaskId( from.getTaskId() ); } return to.build(); } public EventHandler.TaskDetails fromProto(EventHandlerPb.EventHandler.TaskDetails from) { EventHandler.TaskDetails to = new EventHandler.TaskDetails(); to.setWorkflowId( from.getWorkflowId() ); to.setTaskRefName( from.getTaskRefName() ); Map outputMap = new HashMap(); for (Map.Entry pair : from.getOutputMap().entrySet()) { outputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutput(outputMap); if (from.hasOutputMessage()) { to.setOutputMessage( fromProto( from.getOutputMessage() ) ); } to.setTaskId( from.getTaskId() ); return to; } public EventHandlerPb.EventHandler.Action toProto(EventHandler.Action from) { EventHandlerPb.EventHandler.Action.Builder to = EventHandlerPb.EventHandler.Action.newBuilder(); if (from.getAction() != null) { to.setAction( toProto( from.getAction() ) ); } if (from.getStart_workflow() != null) { to.setStartWorkflow( toProto( from.getStart_workflow() ) ); } if (from.getComplete_task() != null) { to.setCompleteTask( toProto( from.getComplete_task() ) ); } if (from.getFail_task() != null) { to.setFailTask( toProto( from.getFail_task() ) ); } to.setExpandInlineJson( from.isExpandInlineJSON() ); return to.build(); } public EventHandler.Action fromProto(EventHandlerPb.EventHandler.Action from) { EventHandler.Action to = new EventHandler.Action(); to.setAction( fromProto( from.getAction() ) ); if (from.hasStartWorkflow()) { to.setStart_workflow( fromProto( from.getStartWorkflow() ) ); } if (from.hasCompleteTask()) { to.setComplete_task( fromProto( from.getCompleteTask() ) ); } if (from.hasFailTask()) { to.setFail_task( fromProto( from.getFailTask() ) ); } to.setExpandInlineJSON( from.getExpandInlineJson() ); return to; } public EventHandlerPb.EventHandler.Action.Type toProto(EventHandler.Action.Type from) { EventHandlerPb.EventHandler.Action.Type to; switch (from) { case start_workflow: to = EventHandlerPb.EventHandler.Action.Type.START_WORKFLOW; break; case complete_task: to = EventHandlerPb.EventHandler.Action.Type.COMPLETE_TASK; break; case fail_task: to = EventHandlerPb.EventHandler.Action.Type.FAIL_TASK; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public EventHandler.Action.Type fromProto(EventHandlerPb.EventHandler.Action.Type from) { EventHandler.Action.Type to; switch (from) { case START_WORKFLOW: to = EventHandler.Action.Type.start_workflow; break; case COMPLETE_TASK: to = EventHandler.Action.Type.complete_task; break; case FAIL_TASK: to = EventHandler.Action.Type.fail_task; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public PollDataPb.PollData toProto(PollData from) { PollDataPb.PollData.Builder to = PollDataPb.PollData.newBuilder(); if (from.getQueueName() != null) { to.setQueueName( from.getQueueName() ); } if (from.getDomain() != null) { to.setDomain( from.getDomain() ); } if (from.getWorkerId() != null) { to.setWorkerId( from.getWorkerId() ); } to.setLastPollTime( from.getLastPollTime() ); return to.build(); } public PollData fromProto(PollDataPb.PollData from) { PollData to = new PollData(); to.setQueueName( from.getQueueName() ); to.setDomain( from.getDomain() ); to.setWorkerId( from.getWorkerId() ); to.setLastPollTime( from.getLastPollTime() ); return to; } public RerunWorkflowRequestPb.RerunWorkflowRequest toProto(RerunWorkflowRequest from) { RerunWorkflowRequestPb.RerunWorkflowRequest.Builder to = RerunWorkflowRequestPb.RerunWorkflowRequest.newBuilder(); if (from.getReRunFromWorkflowId() != null) { to.setReRunFromWorkflowId( from.getReRunFromWorkflowId() ); } for (Map.Entry pair : from.getWorkflowInput().entrySet()) { to.putWorkflowInput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getReRunFromTaskId() != null) { to.setReRunFromTaskId( from.getReRunFromTaskId() ); } for (Map.Entry pair : from.getTaskInput().entrySet()) { to.putTaskInput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } return to.build(); } public RerunWorkflowRequest fromProto(RerunWorkflowRequestPb.RerunWorkflowRequest from) { RerunWorkflowRequest to = new RerunWorkflowRequest(); to.setReRunFromWorkflowId( from.getReRunFromWorkflowId() ); Map workflowInputMap = new HashMap(); for (Map.Entry pair : from.getWorkflowInputMap().entrySet()) { workflowInputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setWorkflowInput(workflowInputMap); to.setReRunFromTaskId( from.getReRunFromTaskId() ); Map taskInputMap = new HashMap(); for (Map.Entry pair : from.getTaskInputMap().entrySet()) { taskInputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setTaskInput(taskInputMap); to.setCorrelationId( from.getCorrelationId() ); return to; } public SkipTaskRequest fromProto(SkipTaskRequestPb.SkipTaskRequest from) { SkipTaskRequest to = new SkipTaskRequest(); Map taskInputMap = new HashMap(); for (Map.Entry pair : from.getTaskInputMap().entrySet()) { taskInputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setTaskInput(taskInputMap); Map taskOutputMap = new HashMap(); for (Map.Entry pair : from.getTaskOutputMap().entrySet()) { taskOutputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setTaskOutput(taskOutputMap); if (from.hasTaskInputMessage()) { to.setTaskInputMessage( fromProto( from.getTaskInputMessage() ) ); } if (from.hasTaskOutputMessage()) { to.setTaskOutputMessage( fromProto( from.getTaskOutputMessage() ) ); } return to; } public StartWorkflowRequestPb.StartWorkflowRequest toProto(StartWorkflowRequest from) { StartWorkflowRequestPb.StartWorkflowRequest.Builder to = StartWorkflowRequestPb.StartWorkflowRequest.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getVersion() != null) { to.setVersion( from.getVersion() ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } for (Map.Entry pair : from.getInput().entrySet()) { to.putInput( pair.getKey(), toProto( pair.getValue() ) ); } to.putAllTaskToDomain( from.getTaskToDomain() ); if (from.getWorkflowDef() != null) { to.setWorkflowDef( toProto( from.getWorkflowDef() ) ); } if (from.getExternalInputPayloadStoragePath() != null) { to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); } if (from.getPriority() != null) { to.setPriority( from.getPriority() ); } return to.build(); } public StartWorkflowRequest fromProto(StartWorkflowRequestPb.StartWorkflowRequest from) { StartWorkflowRequest to = new StartWorkflowRequest(); to.setName( from.getName() ); to.setVersion( from.getVersion() ); to.setCorrelationId( from.getCorrelationId() ); Map inputMap = new HashMap(); for (Map.Entry pair : from.getInputMap().entrySet()) { inputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInput(inputMap); to.setTaskToDomain( from.getTaskToDomainMap() ); if (from.hasWorkflowDef()) { to.setWorkflowDef( fromProto( from.getWorkflowDef() ) ); } to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); to.setPriority( from.getPriority() ); return to; } public SubWorkflowParamsPb.SubWorkflowParams toProto(SubWorkflowParams from) { SubWorkflowParamsPb.SubWorkflowParams.Builder to = SubWorkflowParamsPb.SubWorkflowParams.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getVersion() != null) { to.setVersion( from.getVersion() ); } to.putAllTaskToDomain( from.getTaskToDomain() ); if (from.getWorkflowDefinition() != null) { to.setWorkflowDefinition( toProto( from.getWorkflowDefinition() ) ); } return to.build(); } public SubWorkflowParams fromProto(SubWorkflowParamsPb.SubWorkflowParams from) { SubWorkflowParams to = new SubWorkflowParams(); to.setName( from.getName() ); to.setVersion( from.getVersion() ); to.setTaskToDomain( from.getTaskToDomainMap() ); if (from.hasWorkflowDefinition()) { to.setWorkflowDefinition( fromProto( from.getWorkflowDefinition() ) ); } return to; } public TaskPb.Task toProto(Task from) { TaskPb.Task.Builder to = TaskPb.Task.newBuilder(); if (from.getTaskType() != null) { to.setTaskType( from.getTaskType() ); } if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } for (Map.Entry pair : from.getInputData().entrySet()) { to.putInputData( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getReferenceTaskName() != null) { to.setReferenceTaskName( from.getReferenceTaskName() ); } to.setRetryCount( from.getRetryCount() ); to.setSeq( from.getSeq() ); if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } to.setPollCount( from.getPollCount() ); if (from.getTaskDefName() != null) { to.setTaskDefName( from.getTaskDefName() ); } to.setScheduledTime( from.getScheduledTime() ); to.setStartTime( from.getStartTime() ); to.setEndTime( from.getEndTime() ); to.setUpdateTime( from.getUpdateTime() ); to.setStartDelayInSeconds( from.getStartDelayInSeconds() ); if (from.getRetriedTaskId() != null) { to.setRetriedTaskId( from.getRetriedTaskId() ); } to.setRetried( from.isRetried() ); to.setExecuted( from.isExecuted() ); to.setCallbackFromWorker( from.isCallbackFromWorker() ); to.setResponseTimeoutSeconds( from.getResponseTimeoutSeconds() ); if (from.getWorkflowInstanceId() != null) { to.setWorkflowInstanceId( from.getWorkflowInstanceId() ); } if (from.getWorkflowType() != null) { to.setWorkflowType( from.getWorkflowType() ); } if (from.getTaskId() != null) { to.setTaskId( from.getTaskId() ); } if (from.getReasonForIncompletion() != null) { to.setReasonForIncompletion( from.getReasonForIncompletion() ); } to.setCallbackAfterSeconds( from.getCallbackAfterSeconds() ); if (from.getWorkerId() != null) { to.setWorkerId( from.getWorkerId() ); } for (Map.Entry pair : from.getOutputData().entrySet()) { to.putOutputData( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getWorkflowTask() != null) { to.setWorkflowTask( toProto( from.getWorkflowTask() ) ); } if (from.getDomain() != null) { to.setDomain( from.getDomain() ); } if (from.getInputMessage() != null) { to.setInputMessage( toProto( from.getInputMessage() ) ); } if (from.getOutputMessage() != null) { to.setOutputMessage( toProto( from.getOutputMessage() ) ); } to.setRateLimitPerFrequency( from.getRateLimitPerFrequency() ); to.setRateLimitFrequencyInSeconds( from.getRateLimitFrequencyInSeconds() ); if (from.getExternalInputPayloadStoragePath() != null) { to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); } if (from.getExternalOutputPayloadStoragePath() != null) { to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); } to.setWorkflowPriority( from.getWorkflowPriority() ); if (from.getExecutionNameSpace() != null) { to.setExecutionNameSpace( from.getExecutionNameSpace() ); } if (from.getIsolationGroupId() != null) { to.setIsolationGroupId( from.getIsolationGroupId() ); } to.setIteration( from.getIteration() ); if (from.getSubWorkflowId() != null) { to.setSubWorkflowId( from.getSubWorkflowId() ); } to.setSubworkflowChanged( from.isSubworkflowChanged() ); return to.build(); } public Task fromProto(TaskPb.Task from) { Task to = new Task(); to.setTaskType( from.getTaskType() ); to.setStatus( fromProto( from.getStatus() ) ); Map inputDataMap = new HashMap(); for (Map.Entry pair : from.getInputDataMap().entrySet()) { inputDataMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInputData(inputDataMap); to.setReferenceTaskName( from.getReferenceTaskName() ); to.setRetryCount( from.getRetryCount() ); to.setSeq( from.getSeq() ); to.setCorrelationId( from.getCorrelationId() ); to.setPollCount( from.getPollCount() ); to.setTaskDefName( from.getTaskDefName() ); to.setScheduledTime( from.getScheduledTime() ); to.setStartTime( from.getStartTime() ); to.setEndTime( from.getEndTime() ); to.setUpdateTime( from.getUpdateTime() ); to.setStartDelayInSeconds( from.getStartDelayInSeconds() ); to.setRetriedTaskId( from.getRetriedTaskId() ); to.setRetried( from.getRetried() ); to.setExecuted( from.getExecuted() ); to.setCallbackFromWorker( from.getCallbackFromWorker() ); to.setResponseTimeoutSeconds( from.getResponseTimeoutSeconds() ); to.setWorkflowInstanceId( from.getWorkflowInstanceId() ); to.setWorkflowType( from.getWorkflowType() ); to.setTaskId( from.getTaskId() ); to.setReasonForIncompletion( from.getReasonForIncompletion() ); to.setCallbackAfterSeconds( from.getCallbackAfterSeconds() ); to.setWorkerId( from.getWorkerId() ); Map outputDataMap = new HashMap(); for (Map.Entry pair : from.getOutputDataMap().entrySet()) { outputDataMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutputData(outputDataMap); if (from.hasWorkflowTask()) { to.setWorkflowTask( fromProto( from.getWorkflowTask() ) ); } to.setDomain( from.getDomain() ); if (from.hasInputMessage()) { to.setInputMessage( fromProto( from.getInputMessage() ) ); } if (from.hasOutputMessage()) { to.setOutputMessage( fromProto( from.getOutputMessage() ) ); } to.setRateLimitPerFrequency( from.getRateLimitPerFrequency() ); to.setRateLimitFrequencyInSeconds( from.getRateLimitFrequencyInSeconds() ); to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); to.setWorkflowPriority( from.getWorkflowPriority() ); to.setExecutionNameSpace( from.getExecutionNameSpace() ); to.setIsolationGroupId( from.getIsolationGroupId() ); to.setIteration( from.getIteration() ); to.setSubWorkflowId( from.getSubWorkflowId() ); to.setSubworkflowChanged( from.getSubworkflowChanged() ); return to; } public TaskPb.Task.Status toProto(Task.Status from) { TaskPb.Task.Status to; switch (from) { case IN_PROGRESS: to = TaskPb.Task.Status.IN_PROGRESS; break; case CANCELED: to = TaskPb.Task.Status.CANCELED; break; case FAILED: to = TaskPb.Task.Status.FAILED; break; case FAILED_WITH_TERMINAL_ERROR: to = TaskPb.Task.Status.FAILED_WITH_TERMINAL_ERROR; break; case COMPLETED: to = TaskPb.Task.Status.COMPLETED; break; case COMPLETED_WITH_ERRORS: to = TaskPb.Task.Status.COMPLETED_WITH_ERRORS; break; case SCHEDULED: to = TaskPb.Task.Status.SCHEDULED; break; case TIMED_OUT: to = TaskPb.Task.Status.TIMED_OUT; break; case SKIPPED: to = TaskPb.Task.Status.SKIPPED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public Task.Status fromProto(TaskPb.Task.Status from) { Task.Status to; switch (from) { case IN_PROGRESS: to = Task.Status.IN_PROGRESS; break; case CANCELED: to = Task.Status.CANCELED; break; case FAILED: to = Task.Status.FAILED; break; case FAILED_WITH_TERMINAL_ERROR: to = Task.Status.FAILED_WITH_TERMINAL_ERROR; break; case COMPLETED: to = Task.Status.COMPLETED; break; case COMPLETED_WITH_ERRORS: to = Task.Status.COMPLETED_WITH_ERRORS; break; case SCHEDULED: to = Task.Status.SCHEDULED; break; case TIMED_OUT: to = Task.Status.TIMED_OUT; break; case SKIPPED: to = Task.Status.SKIPPED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskDefPb.TaskDef toProto(TaskDef from) { TaskDefPb.TaskDef.Builder to = TaskDefPb.TaskDef.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getDescription() != null) { to.setDescription( from.getDescription() ); } to.setRetryCount( from.getRetryCount() ); to.setTimeoutSeconds( from.getTimeoutSeconds() ); to.addAllInputKeys( from.getInputKeys() ); to.addAllOutputKeys( from.getOutputKeys() ); if (from.getTimeoutPolicy() != null) { to.setTimeoutPolicy( toProto( from.getTimeoutPolicy() ) ); } if (from.getRetryLogic() != null) { to.setRetryLogic( toProto( from.getRetryLogic() ) ); } to.setRetryDelaySeconds( from.getRetryDelaySeconds() ); to.setResponseTimeoutSeconds( from.getResponseTimeoutSeconds() ); if (from.getConcurrentExecLimit() != null) { to.setConcurrentExecLimit( from.getConcurrentExecLimit() ); } for (Map.Entry pair : from.getInputTemplate().entrySet()) { to.putInputTemplate( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getRateLimitPerFrequency() != null) { to.setRateLimitPerFrequency( from.getRateLimitPerFrequency() ); } if (from.getRateLimitFrequencyInSeconds() != null) { to.setRateLimitFrequencyInSeconds( from.getRateLimitFrequencyInSeconds() ); } if (from.getIsolationGroupId() != null) { to.setIsolationGroupId( from.getIsolationGroupId() ); } if (from.getExecutionNameSpace() != null) { to.setExecutionNameSpace( from.getExecutionNameSpace() ); } if (from.getOwnerEmail() != null) { to.setOwnerEmail( from.getOwnerEmail() ); } if (from.getPollTimeoutSeconds() != null) { to.setPollTimeoutSeconds( from.getPollTimeoutSeconds() ); } if (from.getBackoffScaleFactor() != null) { to.setBackoffScaleFactor( from.getBackoffScaleFactor() ); } return to.build(); } public TaskDef fromProto(TaskDefPb.TaskDef from) { TaskDef to = new TaskDef(); to.setName( from.getName() ); to.setDescription( from.getDescription() ); to.setRetryCount( from.getRetryCount() ); to.setTimeoutSeconds( from.getTimeoutSeconds() ); to.setInputKeys( from.getInputKeysList().stream().collect(Collectors.toCollection(ArrayList::new)) ); to.setOutputKeys( from.getOutputKeysList().stream().collect(Collectors.toCollection(ArrayList::new)) ); to.setTimeoutPolicy( fromProto( from.getTimeoutPolicy() ) ); to.setRetryLogic( fromProto( from.getRetryLogic() ) ); to.setRetryDelaySeconds( from.getRetryDelaySeconds() ); to.setResponseTimeoutSeconds( from.getResponseTimeoutSeconds() ); to.setConcurrentExecLimit( from.getConcurrentExecLimit() ); Map inputTemplateMap = new HashMap(); for (Map.Entry pair : from.getInputTemplateMap().entrySet()) { inputTemplateMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInputTemplate(inputTemplateMap); to.setRateLimitPerFrequency( from.getRateLimitPerFrequency() ); to.setRateLimitFrequencyInSeconds( from.getRateLimitFrequencyInSeconds() ); to.setIsolationGroupId( from.getIsolationGroupId() ); to.setExecutionNameSpace( from.getExecutionNameSpace() ); to.setOwnerEmail( from.getOwnerEmail() ); to.setPollTimeoutSeconds( from.getPollTimeoutSeconds() ); to.setBackoffScaleFactor( from.getBackoffScaleFactor() ); return to; } public TaskDefPb.TaskDef.TimeoutPolicy toProto(TaskDef.TimeoutPolicy from) { TaskDefPb.TaskDef.TimeoutPolicy to; switch (from) { case RETRY: to = TaskDefPb.TaskDef.TimeoutPolicy.RETRY; break; case TIME_OUT_WF: to = TaskDefPb.TaskDef.TimeoutPolicy.TIME_OUT_WF; break; case ALERT_ONLY: to = TaskDefPb.TaskDef.TimeoutPolicy.ALERT_ONLY; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskDef.TimeoutPolicy fromProto(TaskDefPb.TaskDef.TimeoutPolicy from) { TaskDef.TimeoutPolicy to; switch (from) { case RETRY: to = TaskDef.TimeoutPolicy.RETRY; break; case TIME_OUT_WF: to = TaskDef.TimeoutPolicy.TIME_OUT_WF; break; case ALERT_ONLY: to = TaskDef.TimeoutPolicy.ALERT_ONLY; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskDefPb.TaskDef.RetryLogic toProto(TaskDef.RetryLogic from) { TaskDefPb.TaskDef.RetryLogic to; switch (from) { case FIXED: to = TaskDefPb.TaskDef.RetryLogic.FIXED; break; case EXPONENTIAL_BACKOFF: to = TaskDefPb.TaskDef.RetryLogic.EXPONENTIAL_BACKOFF; break; case LINEAR_BACKOFF: to = TaskDefPb.TaskDef.RetryLogic.LINEAR_BACKOFF; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskDef.RetryLogic fromProto(TaskDefPb.TaskDef.RetryLogic from) { TaskDef.RetryLogic to; switch (from) { case FIXED: to = TaskDef.RetryLogic.FIXED; break; case EXPONENTIAL_BACKOFF: to = TaskDef.RetryLogic.EXPONENTIAL_BACKOFF; break; case LINEAR_BACKOFF: to = TaskDef.RetryLogic.LINEAR_BACKOFF; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskExecLogPb.TaskExecLog toProto(TaskExecLog from) { TaskExecLogPb.TaskExecLog.Builder to = TaskExecLogPb.TaskExecLog.newBuilder(); if (from.getLog() != null) { to.setLog( from.getLog() ); } if (from.getTaskId() != null) { to.setTaskId( from.getTaskId() ); } to.setCreatedTime( from.getCreatedTime() ); return to.build(); } public TaskExecLog fromProto(TaskExecLogPb.TaskExecLog from) { TaskExecLog to = new TaskExecLog(); to.setLog( from.getLog() ); to.setTaskId( from.getTaskId() ); to.setCreatedTime( from.getCreatedTime() ); return to; } public TaskResultPb.TaskResult toProto(TaskResult from) { TaskResultPb.TaskResult.Builder to = TaskResultPb.TaskResult.newBuilder(); if (from.getWorkflowInstanceId() != null) { to.setWorkflowInstanceId( from.getWorkflowInstanceId() ); } if (from.getTaskId() != null) { to.setTaskId( from.getTaskId() ); } if (from.getReasonForIncompletion() != null) { to.setReasonForIncompletion( from.getReasonForIncompletion() ); } to.setCallbackAfterSeconds( from.getCallbackAfterSeconds() ); if (from.getWorkerId() != null) { to.setWorkerId( from.getWorkerId() ); } if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } for (Map.Entry pair : from.getOutputData().entrySet()) { to.putOutputData( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getOutputMessage() != null) { to.setOutputMessage( toProto( from.getOutputMessage() ) ); } return to.build(); } public TaskResult fromProto(TaskResultPb.TaskResult from) { TaskResult to = new TaskResult(); to.setWorkflowInstanceId( from.getWorkflowInstanceId() ); to.setTaskId( from.getTaskId() ); to.setReasonForIncompletion( from.getReasonForIncompletion() ); to.setCallbackAfterSeconds( from.getCallbackAfterSeconds() ); to.setWorkerId( from.getWorkerId() ); to.setStatus( fromProto( from.getStatus() ) ); Map outputDataMap = new HashMap(); for (Map.Entry pair : from.getOutputDataMap().entrySet()) { outputDataMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutputData(outputDataMap); if (from.hasOutputMessage()) { to.setOutputMessage( fromProto( from.getOutputMessage() ) ); } return to; } public TaskResultPb.TaskResult.Status toProto(TaskResult.Status from) { TaskResultPb.TaskResult.Status to; switch (from) { case IN_PROGRESS: to = TaskResultPb.TaskResult.Status.IN_PROGRESS; break; case FAILED: to = TaskResultPb.TaskResult.Status.FAILED; break; case FAILED_WITH_TERMINAL_ERROR: to = TaskResultPb.TaskResult.Status.FAILED_WITH_TERMINAL_ERROR; break; case COMPLETED: to = TaskResultPb.TaskResult.Status.COMPLETED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskResult.Status fromProto(TaskResultPb.TaskResult.Status from) { TaskResult.Status to; switch (from) { case IN_PROGRESS: to = TaskResult.Status.IN_PROGRESS; break; case FAILED: to = TaskResult.Status.FAILED; break; case FAILED_WITH_TERMINAL_ERROR: to = TaskResult.Status.FAILED_WITH_TERMINAL_ERROR; break; case COMPLETED: to = TaskResult.Status.COMPLETED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public TaskSummaryPb.TaskSummary toProto(TaskSummary from) { TaskSummaryPb.TaskSummary.Builder to = TaskSummaryPb.TaskSummary.newBuilder(); if (from.getWorkflowId() != null) { to.setWorkflowId( from.getWorkflowId() ); } if (from.getWorkflowType() != null) { to.setWorkflowType( from.getWorkflowType() ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } if (from.getScheduledTime() != null) { to.setScheduledTime( from.getScheduledTime() ); } if (from.getStartTime() != null) { to.setStartTime( from.getStartTime() ); } if (from.getUpdateTime() != null) { to.setUpdateTime( from.getUpdateTime() ); } if (from.getEndTime() != null) { to.setEndTime( from.getEndTime() ); } if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } if (from.getReasonForIncompletion() != null) { to.setReasonForIncompletion( from.getReasonForIncompletion() ); } to.setExecutionTime( from.getExecutionTime() ); to.setQueueWaitTime( from.getQueueWaitTime() ); if (from.getTaskDefName() != null) { to.setTaskDefName( from.getTaskDefName() ); } if (from.getTaskType() != null) { to.setTaskType( from.getTaskType() ); } if (from.getInput() != null) { to.setInput( from.getInput() ); } if (from.getOutput() != null) { to.setOutput( from.getOutput() ); } if (from.getTaskId() != null) { to.setTaskId( from.getTaskId() ); } if (from.getExternalInputPayloadStoragePath() != null) { to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); } if (from.getExternalOutputPayloadStoragePath() != null) { to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); } to.setWorkflowPriority( from.getWorkflowPriority() ); if (from.getDomain() != null) { to.setDomain( from.getDomain() ); } return to.build(); } public TaskSummary fromProto(TaskSummaryPb.TaskSummary from) { TaskSummary to = new TaskSummary(); to.setWorkflowId( from.getWorkflowId() ); to.setWorkflowType( from.getWorkflowType() ); to.setCorrelationId( from.getCorrelationId() ); to.setScheduledTime( from.getScheduledTime() ); to.setStartTime( from.getStartTime() ); to.setUpdateTime( from.getUpdateTime() ); to.setEndTime( from.getEndTime() ); to.setStatus( fromProto( from.getStatus() ) ); to.setReasonForIncompletion( from.getReasonForIncompletion() ); to.setExecutionTime( from.getExecutionTime() ); to.setQueueWaitTime( from.getQueueWaitTime() ); to.setTaskDefName( from.getTaskDefName() ); to.setTaskType( from.getTaskType() ); to.setInput( from.getInput() ); to.setOutput( from.getOutput() ); to.setTaskId( from.getTaskId() ); to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); to.setWorkflowPriority( from.getWorkflowPriority() ); to.setDomain( from.getDomain() ); return to; } public WorkflowPb.Workflow toProto(Workflow from) { WorkflowPb.Workflow.Builder to = WorkflowPb.Workflow.newBuilder(); if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } to.setEndTime( from.getEndTime() ); if (from.getWorkflowId() != null) { to.setWorkflowId( from.getWorkflowId() ); } if (from.getParentWorkflowId() != null) { to.setParentWorkflowId( from.getParentWorkflowId() ); } if (from.getParentWorkflowTaskId() != null) { to.setParentWorkflowTaskId( from.getParentWorkflowTaskId() ); } for (Task elem : from.getTasks()) { to.addTasks( toProto(elem) ); } for (Map.Entry pair : from.getInput().entrySet()) { to.putInput( pair.getKey(), toProto( pair.getValue() ) ); } for (Map.Entry pair : from.getOutput().entrySet()) { to.putOutput( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } if (from.getReRunFromWorkflowId() != null) { to.setReRunFromWorkflowId( from.getReRunFromWorkflowId() ); } if (from.getReasonForIncompletion() != null) { to.setReasonForIncompletion( from.getReasonForIncompletion() ); } if (from.getEvent() != null) { to.setEvent( from.getEvent() ); } to.putAllTaskToDomain( from.getTaskToDomain() ); to.addAllFailedReferenceTaskNames( from.getFailedReferenceTaskNames() ); if (from.getWorkflowDefinition() != null) { to.setWorkflowDefinition( toProto( from.getWorkflowDefinition() ) ); } if (from.getExternalInputPayloadStoragePath() != null) { to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); } if (from.getExternalOutputPayloadStoragePath() != null) { to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); } to.setPriority( from.getPriority() ); for (Map.Entry pair : from.getVariables().entrySet()) { to.putVariables( pair.getKey(), toProto( pair.getValue() ) ); } to.setLastRetriedTime( from.getLastRetriedTime() ); to.addAllFailedTaskNames( from.getFailedTaskNames() ); return to.build(); } public Workflow fromProto(WorkflowPb.Workflow from) { Workflow to = new Workflow(); to.setStatus( fromProto( from.getStatus() ) ); to.setEndTime( from.getEndTime() ); to.setWorkflowId( from.getWorkflowId() ); to.setParentWorkflowId( from.getParentWorkflowId() ); to.setParentWorkflowTaskId( from.getParentWorkflowTaskId() ); to.setTasks( from.getTasksList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); Map inputMap = new HashMap(); for (Map.Entry pair : from.getInputMap().entrySet()) { inputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInput(inputMap); Map outputMap = new HashMap(); for (Map.Entry pair : from.getOutputMap().entrySet()) { outputMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutput(outputMap); to.setCorrelationId( from.getCorrelationId() ); to.setReRunFromWorkflowId( from.getReRunFromWorkflowId() ); to.setReasonForIncompletion( from.getReasonForIncompletion() ); to.setEvent( from.getEvent() ); to.setTaskToDomain( from.getTaskToDomainMap() ); to.setFailedReferenceTaskNames( from.getFailedReferenceTaskNamesList().stream().collect(Collectors.toCollection(HashSet::new)) ); if (from.hasWorkflowDefinition()) { to.setWorkflowDefinition( fromProto( from.getWorkflowDefinition() ) ); } to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); to.setPriority( from.getPriority() ); Map variablesMap = new HashMap(); for (Map.Entry pair : from.getVariablesMap().entrySet()) { variablesMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setVariables(variablesMap); to.setLastRetriedTime( from.getLastRetriedTime() ); to.setFailedTaskNames( from.getFailedTaskNamesList().stream().collect(Collectors.toCollection(HashSet::new)) ); return to; } public WorkflowPb.Workflow.WorkflowStatus toProto(Workflow.WorkflowStatus from) { WorkflowPb.Workflow.WorkflowStatus to; switch (from) { case RUNNING: to = WorkflowPb.Workflow.WorkflowStatus.RUNNING; break; case COMPLETED: to = WorkflowPb.Workflow.WorkflowStatus.COMPLETED; break; case FAILED: to = WorkflowPb.Workflow.WorkflowStatus.FAILED; break; case TIMED_OUT: to = WorkflowPb.Workflow.WorkflowStatus.TIMED_OUT; break; case TERMINATED: to = WorkflowPb.Workflow.WorkflowStatus.TERMINATED; break; case PAUSED: to = WorkflowPb.Workflow.WorkflowStatus.PAUSED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public Workflow.WorkflowStatus fromProto(WorkflowPb.Workflow.WorkflowStatus from) { Workflow.WorkflowStatus to; switch (from) { case RUNNING: to = Workflow.WorkflowStatus.RUNNING; break; case COMPLETED: to = Workflow.WorkflowStatus.COMPLETED; break; case FAILED: to = Workflow.WorkflowStatus.FAILED; break; case TIMED_OUT: to = Workflow.WorkflowStatus.TIMED_OUT; break; case TERMINATED: to = Workflow.WorkflowStatus.TERMINATED; break; case PAUSED: to = Workflow.WorkflowStatus.PAUSED; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public WorkflowDefPb.WorkflowDef toProto(WorkflowDef from) { WorkflowDefPb.WorkflowDef.Builder to = WorkflowDefPb.WorkflowDef.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getDescription() != null) { to.setDescription( from.getDescription() ); } to.setVersion( from.getVersion() ); for (WorkflowTask elem : from.getTasks()) { to.addTasks( toProto(elem) ); } to.addAllInputParameters( from.getInputParameters() ); for (Map.Entry pair : from.getOutputParameters().entrySet()) { to.putOutputParameters( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getFailureWorkflow() != null) { to.setFailureWorkflow( from.getFailureWorkflow() ); } to.setSchemaVersion( from.getSchemaVersion() ); to.setRestartable( from.isRestartable() ); to.setWorkflowStatusListenerEnabled( from.isWorkflowStatusListenerEnabled() ); if (from.getOwnerEmail() != null) { to.setOwnerEmail( from.getOwnerEmail() ); } if (from.getTimeoutPolicy() != null) { to.setTimeoutPolicy( toProto( from.getTimeoutPolicy() ) ); } to.setTimeoutSeconds( from.getTimeoutSeconds() ); for (Map.Entry pair : from.getVariables().entrySet()) { to.putVariables( pair.getKey(), toProto( pair.getValue() ) ); } for (Map.Entry pair : from.getInputTemplate().entrySet()) { to.putInputTemplate( pair.getKey(), toProto( pair.getValue() ) ); } return to.build(); } public WorkflowDef fromProto(WorkflowDefPb.WorkflowDef from) { WorkflowDef to = new WorkflowDef(); to.setName( from.getName() ); to.setDescription( from.getDescription() ); to.setVersion( from.getVersion() ); to.setTasks( from.getTasksList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); to.setInputParameters( from.getInputParametersList().stream().collect(Collectors.toCollection(ArrayList::new)) ); Map outputParametersMap = new HashMap(); for (Map.Entry pair : from.getOutputParametersMap().entrySet()) { outputParametersMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setOutputParameters(outputParametersMap); to.setFailureWorkflow( from.getFailureWorkflow() ); to.setSchemaVersion( from.getSchemaVersion() ); to.setRestartable( from.getRestartable() ); to.setWorkflowStatusListenerEnabled( from.getWorkflowStatusListenerEnabled() ); to.setOwnerEmail( from.getOwnerEmail() ); to.setTimeoutPolicy( fromProto( from.getTimeoutPolicy() ) ); to.setTimeoutSeconds( from.getTimeoutSeconds() ); Map variablesMap = new HashMap(); for (Map.Entry pair : from.getVariablesMap().entrySet()) { variablesMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setVariables(variablesMap); Map inputTemplateMap = new HashMap(); for (Map.Entry pair : from.getInputTemplateMap().entrySet()) { inputTemplateMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInputTemplate(inputTemplateMap); return to; } public WorkflowDefPb.WorkflowDef.TimeoutPolicy toProto(WorkflowDef.TimeoutPolicy from) { WorkflowDefPb.WorkflowDef.TimeoutPolicy to; switch (from) { case TIME_OUT_WF: to = WorkflowDefPb.WorkflowDef.TimeoutPolicy.TIME_OUT_WF; break; case ALERT_ONLY: to = WorkflowDefPb.WorkflowDef.TimeoutPolicy.ALERT_ONLY; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public WorkflowDef.TimeoutPolicy fromProto(WorkflowDefPb.WorkflowDef.TimeoutPolicy from) { WorkflowDef.TimeoutPolicy to; switch (from) { case TIME_OUT_WF: to = WorkflowDef.TimeoutPolicy.TIME_OUT_WF; break; case ALERT_ONLY: to = WorkflowDef.TimeoutPolicy.ALERT_ONLY; break; default: throw new IllegalArgumentException("Unexpected enum constant: " + from); } return to; } public WorkflowDefSummaryPb.WorkflowDefSummary toProto(WorkflowDefSummary from) { WorkflowDefSummaryPb.WorkflowDefSummary.Builder to = WorkflowDefSummaryPb.WorkflowDefSummary.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } to.setVersion( from.getVersion() ); if (from.getCreateTime() != null) { to.setCreateTime( from.getCreateTime() ); } return to.build(); } public WorkflowDefSummary fromProto(WorkflowDefSummaryPb.WorkflowDefSummary from) { WorkflowDefSummary to = new WorkflowDefSummary(); to.setName( from.getName() ); to.setVersion( from.getVersion() ); to.setCreateTime( from.getCreateTime() ); return to; } public WorkflowSummaryPb.WorkflowSummary toProto(WorkflowSummary from) { WorkflowSummaryPb.WorkflowSummary.Builder to = WorkflowSummaryPb.WorkflowSummary.newBuilder(); if (from.getWorkflowType() != null) { to.setWorkflowType( from.getWorkflowType() ); } to.setVersion( from.getVersion() ); if (from.getWorkflowId() != null) { to.setWorkflowId( from.getWorkflowId() ); } if (from.getCorrelationId() != null) { to.setCorrelationId( from.getCorrelationId() ); } if (from.getStartTime() != null) { to.setStartTime( from.getStartTime() ); } if (from.getUpdateTime() != null) { to.setUpdateTime( from.getUpdateTime() ); } if (from.getEndTime() != null) { to.setEndTime( from.getEndTime() ); } if (from.getStatus() != null) { to.setStatus( toProto( from.getStatus() ) ); } if (from.getInput() != null) { to.setInput( from.getInput() ); } if (from.getOutput() != null) { to.setOutput( from.getOutput() ); } if (from.getReasonForIncompletion() != null) { to.setReasonForIncompletion( from.getReasonForIncompletion() ); } to.setExecutionTime( from.getExecutionTime() ); if (from.getEvent() != null) { to.setEvent( from.getEvent() ); } if (from.getFailedReferenceTaskNames() != null) { to.setFailedReferenceTaskNames( from.getFailedReferenceTaskNames() ); } if (from.getExternalInputPayloadStoragePath() != null) { to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); } if (from.getExternalOutputPayloadStoragePath() != null) { to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); } to.setPriority( from.getPriority() ); to.addAllFailedTaskNames( from.getFailedTaskNames() ); return to.build(); } public WorkflowSummary fromProto(WorkflowSummaryPb.WorkflowSummary from) { WorkflowSummary to = new WorkflowSummary(); to.setWorkflowType( from.getWorkflowType() ); to.setVersion( from.getVersion() ); to.setWorkflowId( from.getWorkflowId() ); to.setCorrelationId( from.getCorrelationId() ); to.setStartTime( from.getStartTime() ); to.setUpdateTime( from.getUpdateTime() ); to.setEndTime( from.getEndTime() ); to.setStatus( fromProto( from.getStatus() ) ); to.setInput( from.getInput() ); to.setOutput( from.getOutput() ); to.setReasonForIncompletion( from.getReasonForIncompletion() ); to.setExecutionTime( from.getExecutionTime() ); to.setEvent( from.getEvent() ); to.setFailedReferenceTaskNames( from.getFailedReferenceTaskNames() ); to.setExternalInputPayloadStoragePath( from.getExternalInputPayloadStoragePath() ); to.setExternalOutputPayloadStoragePath( from.getExternalOutputPayloadStoragePath() ); to.setPriority( from.getPriority() ); to.setFailedTaskNames( from.getFailedTaskNamesList().stream().collect(Collectors.toCollection(HashSet::new)) ); return to; } public WorkflowTaskPb.WorkflowTask toProto(WorkflowTask from) { WorkflowTaskPb.WorkflowTask.Builder to = WorkflowTaskPb.WorkflowTask.newBuilder(); if (from.getName() != null) { to.setName( from.getName() ); } if (from.getTaskReferenceName() != null) { to.setTaskReferenceName( from.getTaskReferenceName() ); } if (from.getDescription() != null) { to.setDescription( from.getDescription() ); } for (Map.Entry pair : from.getInputParameters().entrySet()) { to.putInputParameters( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getType() != null) { to.setType( from.getType() ); } if (from.getDynamicTaskNameParam() != null) { to.setDynamicTaskNameParam( from.getDynamicTaskNameParam() ); } if (from.getCaseValueParam() != null) { to.setCaseValueParam( from.getCaseValueParam() ); } if (from.getCaseExpression() != null) { to.setCaseExpression( from.getCaseExpression() ); } if (from.getScriptExpression() != null) { to.setScriptExpression( from.getScriptExpression() ); } for (Map.Entry> pair : from.getDecisionCases().entrySet()) { to.putDecisionCases( pair.getKey(), toProto( pair.getValue() ) ); } if (from.getDynamicForkTasksParam() != null) { to.setDynamicForkTasksParam( from.getDynamicForkTasksParam() ); } if (from.getDynamicForkTasksInputParamName() != null) { to.setDynamicForkTasksInputParamName( from.getDynamicForkTasksInputParamName() ); } for (WorkflowTask elem : from.getDefaultCase()) { to.addDefaultCase( toProto(elem) ); } for (List elem : from.getForkTasks()) { to.addForkTasks( toProto(elem) ); } to.setStartDelay( from.getStartDelay() ); if (from.getSubWorkflowParam() != null) { to.setSubWorkflowParam( toProto( from.getSubWorkflowParam() ) ); } to.addAllJoinOn( from.getJoinOn() ); if (from.getSink() != null) { to.setSink( from.getSink() ); } to.setOptional( from.isOptional() ); if (from.getTaskDefinition() != null) { to.setTaskDefinition( toProto( from.getTaskDefinition() ) ); } if (from.isRateLimited() != null) { to.setRateLimited( from.isRateLimited() ); } to.addAllDefaultExclusiveJoinTask( from.getDefaultExclusiveJoinTask() ); if (from.isAsyncComplete() != null) { to.setAsyncComplete( from.isAsyncComplete() ); } if (from.getLoopCondition() != null) { to.setLoopCondition( from.getLoopCondition() ); } for (WorkflowTask elem : from.getLoopOver()) { to.addLoopOver( toProto(elem) ); } if (from.getRetryCount() != null) { to.setRetryCount( from.getRetryCount() ); } if (from.getEvaluatorType() != null) { to.setEvaluatorType( from.getEvaluatorType() ); } if (from.getExpression() != null) { to.setExpression( from.getExpression() ); } return to.build(); } public WorkflowTask fromProto(WorkflowTaskPb.WorkflowTask from) { WorkflowTask to = new WorkflowTask(); to.setName( from.getName() ); to.setTaskReferenceName( from.getTaskReferenceName() ); to.setDescription( from.getDescription() ); Map inputParametersMap = new HashMap(); for (Map.Entry pair : from.getInputParametersMap().entrySet()) { inputParametersMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setInputParameters(inputParametersMap); to.setType( from.getType() ); to.setDynamicTaskNameParam( from.getDynamicTaskNameParam() ); to.setCaseValueParam( from.getCaseValueParam() ); to.setCaseExpression( from.getCaseExpression() ); to.setScriptExpression( from.getScriptExpression() ); Map> decisionCasesMap = new HashMap>(); for (Map.Entry pair : from.getDecisionCasesMap().entrySet()) { decisionCasesMap.put( pair.getKey(), fromProto( pair.getValue() ) ); } to.setDecisionCases(decisionCasesMap); to.setDynamicForkTasksParam( from.getDynamicForkTasksParam() ); to.setDynamicForkTasksInputParamName( from.getDynamicForkTasksInputParamName() ); to.setDefaultCase( from.getDefaultCaseList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); to.setForkTasks( from.getForkTasksList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); to.setStartDelay( from.getStartDelay() ); if (from.hasSubWorkflowParam()) { to.setSubWorkflowParam( fromProto( from.getSubWorkflowParam() ) ); } to.setJoinOn( from.getJoinOnList().stream().collect(Collectors.toCollection(ArrayList::new)) ); to.setSink( from.getSink() ); to.setOptional( from.getOptional() ); if (from.hasTaskDefinition()) { to.setTaskDefinition( fromProto( from.getTaskDefinition() ) ); } to.setRateLimited( from.getRateLimited() ); to.setDefaultExclusiveJoinTask( from.getDefaultExclusiveJoinTaskList().stream().collect(Collectors.toCollection(ArrayList::new)) ); to.setAsyncComplete( from.getAsyncComplete() ); to.setLoopCondition( from.getLoopCondition() ); to.setLoopOver( from.getLoopOverList().stream().map(this::fromProto).collect(Collectors.toCollection(ArrayList::new)) ); to.setRetryCount( from.getRetryCount() ); to.setEvaluatorType( from.getEvaluatorType() ); to.setExpression( from.getExpression() ); return to; } public abstract WorkflowTaskPb.WorkflowTask.WorkflowTaskList toProto(List in); public abstract List fromProto(WorkflowTaskPb.WorkflowTask.WorkflowTaskList in); public abstract Value toProto(Object in); public abstract Object fromProto(Value in); public abstract Any toProto(Any in); public abstract Any fromProto(Any in); } ================================================ FILE: grpc/src/main/java/com/netflix/conductor/grpc/ProtoMapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc; import com.google.protobuf.Any; import com.google.protobuf.ListValue; import com.google.protobuf.NullValue; import com.google.protobuf.Struct; import com.google.protobuf.Value; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.proto.WorkflowTaskPb; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; /** * ProtoMapper implements conversion code between the internal models * used by Conductor (POJOs) and their corresponding equivalents in * the exposed Protocol Buffers interface. * * The vast majority of the mapping logic is implemented in the autogenerated * {@link AbstractProtoMapper} class. This class only implements the custom * logic for objects that need to be special cased in the API. */ public final class ProtoMapper extends AbstractProtoMapper { public static final ProtoMapper INSTANCE = new ProtoMapper(); private static final int NO_RETRY_VALUE = -1; private ProtoMapper() {} /** * Convert an {@link Object} instance into its equivalent {@link Value} * ProtoBuf object. * * The {@link Value} ProtoBuf message is a variant type that can define any * value representable as a native JSON type. Consequently, this method expects * the given {@link Object} instance to be a Java object instance of JSON-native * value, namely: null, {@link Boolean}, {@link Double}, {@link String}, * {@link Map}, {@link List}. * * Any other values will cause an exception to be thrown. * See {@link ProtoMapper#fromProto(Value)} for the reverse mapping. * * @param val a Java object that can be represented natively in JSON * @return an instance of a {@link Value} ProtoBuf message */ @Override public Value toProto(Object val) { Value.Builder builder = Value.newBuilder(); if (val == null) { builder.setNullValue(NullValue.NULL_VALUE); } else if (val instanceof Boolean) { builder.setBoolValue((Boolean) val); } else if (val instanceof Double) { builder.setNumberValue((Double) val); } else if (val instanceof String) { builder.setStringValue((String) val); } else if (val instanceof Map) { Map map = (Map) val; Struct.Builder struct = Struct.newBuilder(); for (Map.Entry pair : map.entrySet()) { struct.putFields(pair.getKey(), toProto(pair.getValue())); } builder.setStructValue(struct.build()); } else if (val instanceof List) { ListValue.Builder list = ListValue.newBuilder(); for (Object obj : (List)val) { list.addValues(toProto(obj)); } builder.setListValue(list.build()); } else { throw new ClassCastException("cannot map to Value type: "+val); } return builder.build(); } /** * Convert a ProtoBuf {@link Value} message into its native Java object * equivalent. * * See {@link ProtoMapper#toProto(Object)} for the reverse mapping and the * possible values that can be returned from this method. * * @param any an instance of a ProtoBuf {@link Value} message * @return a native Java object representing the value */ @Override public Object fromProto(Value any) { switch (any.getKindCase()) { case NULL_VALUE: return null; case BOOL_VALUE: return any.getBoolValue(); case NUMBER_VALUE: return any.getNumberValue(); case STRING_VALUE: return any.getStringValue(); case STRUCT_VALUE: Struct struct = any.getStructValue(); Map map = new HashMap<>(); for (Map.Entry pair : struct.getFieldsMap().entrySet()) { map.put(pair.getKey(), fromProto(pair.getValue())); } return map; case LIST_VALUE: List list = new ArrayList<>(); for (Value val : any.getListValue().getValuesList()) { list.add(fromProto(val)); } return list; default: throw new ClassCastException("unset Value element: "+any); } } /** * Convert a WorkflowTaskList message wrapper into a {@link List} instance * with its contents. * * @param list an instance of a ProtoBuf message * @return a list with the contents of the message */ @Override public List fromProto(WorkflowTaskPb.WorkflowTask.WorkflowTaskList list) { return list.getTasksList().stream().map(this::fromProto).collect(Collectors.toList()); } @Override public WorkflowTaskPb.WorkflowTask toProto(final WorkflowTask from) { final WorkflowTaskPb.WorkflowTask.Builder to = WorkflowTaskPb.WorkflowTask.newBuilder(super.toProto(from)); if (from.getRetryCount() == null) { to.setRetryCount(NO_RETRY_VALUE); } return to.build(); } @Override public WorkflowTask fromProto(final WorkflowTaskPb.WorkflowTask from) { final WorkflowTask workflowTask = super.fromProto(from); if (from.getRetryCount() == NO_RETRY_VALUE) { workflowTask.setRetryCount(null); } return workflowTask; } /** * Convert a list of {@link WorkflowTask} instances into a ProtoBuf wrapper object. * * @param list a list of {@link WorkflowTask} instances * @return a ProtoBuf message wrapping the contents of the list */ @Override public WorkflowTaskPb.WorkflowTask.WorkflowTaskList toProto(List list) { return WorkflowTaskPb.WorkflowTask.WorkflowTaskList.newBuilder() .addAllTasks(list.stream().map(this::toProto)::iterator) .build(); } @Override public Any toProto(Any in) { return in; } @Override public Any fromProto(Any in) { return in; } } ================================================ FILE: grpc/src/main/proto/grpc/event_service.proto ================================================ syntax = "proto3"; package conductor.grpc.events; import "model/eventhandler.proto"; option java_package = "com.netflix.conductor.grpc"; option java_outer_classname = "EventServicePb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/grpc/events"; service EventService { // POST / rpc AddEventHandler(AddEventHandlerRequest) returns (AddEventHandlerResponse); // PUT / rpc UpdateEventHandler(UpdateEventHandlerRequest) returns (UpdateEventHandlerResponse); // DELETE /{name} rpc RemoveEventHandler(RemoveEventHandlerRequest) returns (RemoveEventHandlerResponse); // GET / rpc GetEventHandlers(GetEventHandlersRequest) returns (stream conductor.proto.EventHandler); // GET /{name} rpc GetEventHandlersForEvent(GetEventHandlersForEventRequest) returns (stream conductor.proto.EventHandler); } message AddEventHandlerRequest { conductor.proto.EventHandler handler = 1; } message AddEventHandlerResponse {} message UpdateEventHandlerRequest { conductor.proto.EventHandler handler = 1; } message UpdateEventHandlerResponse {} message RemoveEventHandlerRequest { string name = 1; } message RemoveEventHandlerResponse {} message GetEventHandlersRequest {} message GetEventHandlersForEventRequest { string event = 1; bool active_only = 2; } ================================================ FILE: grpc/src/main/proto/grpc/metadata_service.proto ================================================ syntax = "proto3"; package conductor.grpc.metadata; import "model/taskdef.proto"; import "model/workflowdef.proto"; option java_package = "com.netflix.conductor.grpc"; option java_outer_classname = "MetadataServicePb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/grpc/metadata"; service MetadataService { // POST /workflow rpc CreateWorkflow(CreateWorkflowRequest) returns (CreateWorkflowResponse); // POST /workflow/validate rpc ValidateWorkflow(ValidateWorkflowRequest) returns (ValidateWorkflowResponse); // PUT /workflow rpc UpdateWorkflows(UpdateWorkflowsRequest) returns (UpdateWorkflowsResponse); // GET /workflow/{name} rpc GetWorkflow(GetWorkflowRequest) returns (GetWorkflowResponse); // POST /taskdefs rpc CreateTasks(CreateTasksRequest) returns (CreateTasksResponse); // PUT /taskdefs rpc UpdateTask(UpdateTaskRequest) returns (UpdateTaskResponse); // GET /taskdefs/{tasktype} rpc GetTask(GetTaskRequest) returns (GetTaskResponse); // DELETE /taskdefs/{tasktype} rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); } message CreateWorkflowRequest { conductor.proto.WorkflowDef workflow = 1; } message CreateWorkflowResponse {} message ValidateWorkflowRequest { conductor.proto.WorkflowDef workflow = 1; } message ValidateWorkflowResponse {} message UpdateWorkflowsRequest { repeated conductor.proto.WorkflowDef defs = 1; } message UpdateWorkflowsResponse {} message GetWorkflowRequest { string name = 1; int32 version = 2; } message GetWorkflowResponse { conductor.proto.WorkflowDef workflow = 1; } message CreateTasksRequest { repeated conductor.proto.TaskDef defs = 1; } message CreateTasksResponse {} message UpdateTaskRequest { conductor.proto.TaskDef task = 1; } message UpdateTaskResponse {} message GetTaskRequest { string task_type = 1; } message GetTaskResponse { conductor.proto.TaskDef task = 1; } message DeleteTaskRequest { string task_type = 1; } message DeleteTaskResponse {} ================================================ FILE: grpc/src/main/proto/grpc/search.proto ================================================ syntax = "proto3"; package conductor.grpc.search; option java_package = "com.netflix.conductor.grpc"; option java_outer_classname = "SearchPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/grpc/search"; message Request { int32 start = 1; int32 size = 2; string sort = 3; string free_text = 4; string query = 5; } ================================================ FILE: grpc/src/main/proto/grpc/task_service.proto ================================================ syntax = "proto3"; package conductor.grpc.tasks; import "model/taskexeclog.proto"; import "model/taskresult.proto"; import "model/tasksummary.proto"; import "model/task.proto"; import "grpc/search.proto"; option java_package = "com.netflix.conductor.grpc"; option java_outer_classname = "TaskServicePb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/grpc/tasks"; service TaskService { // GET /poll/{tasktype} rpc Poll(PollRequest) returns (PollResponse); // /poll/batch/{tasktype} rpc BatchPoll(BatchPollRequest) returns (stream conductor.proto.Task); // POST / rpc UpdateTask(UpdateTaskRequest) returns (UpdateTaskResponse); // POST /{taskId}/log rpc AddLog(AddLogRequest) returns (AddLogResponse); // GET {taskId}/log rpc GetTaskLogs(GetTaskLogsRequest) returns (GetTaskLogsResponse); // GET /{taskId} rpc GetTask(GetTaskRequest) returns (GetTaskResponse); // GET /queue/sizes rpc GetQueueSizesForTasks(QueueSizesRequest) returns (QueueSizesResponse); // GET /queue/all rpc GetQueueInfo(QueueInfoRequest) returns (QueueInfoResponse); // GET /queue/all/verbose rpc GetQueueAllInfo(QueueAllInfoRequest) returns (QueueAllInfoResponse); // GET /search rpc Search(conductor.grpc.search.Request) returns (TaskSummarySearchResult); // GET /searchV2 rpc SearchV2(conductor.grpc.search.Request) returns (TaskSearchResult); } message PollRequest { string task_type = 1; string worker_id = 2; string domain = 3; } message PollResponse { conductor.proto.Task task = 1; } message BatchPollRequest { string task_type = 1; string worker_id = 2; string domain = 3; int32 count = 4; int32 timeout = 5; } message UpdateTaskRequest { conductor.proto.TaskResult result = 1; } message UpdateTaskResponse { string task_id = 1; } message AddLogRequest { string task_id = 1; string log = 2; } message AddLogResponse {} message GetTaskLogsRequest { string task_id = 1; } message GetTaskLogsResponse { repeated conductor.proto.TaskExecLog logs = 1; } message GetTaskRequest { string task_id = 1; } message GetTaskResponse { conductor.proto.Task task = 1; } message QueueSizesRequest { repeated string task_types = 1; } message QueueSizesResponse { map queue_for_task = 1; } message QueueInfoRequest {} message QueueInfoResponse { map queues = 1; } message QueueAllInfoRequest {} message QueueAllInfoResponse { message ShardInfo { int64 size = 1; int64 uacked = 2; } message QueueInfo { map shards = 1; } map queues = 1; } message TaskSummarySearchResult { int64 total_hits = 1; repeated conductor.proto.TaskSummary results = 2; } message TaskSearchResult { int64 total_hits = 1; repeated conductor.proto.Task results = 2; } ================================================ FILE: grpc/src/main/proto/grpc/workflow_service.proto ================================================ syntax = "proto3"; package conductor.grpc.workflows; import "grpc/search.proto"; import "model/workflow.proto"; import "model/workflowsummary.proto"; import "model/skiptaskrequest.proto"; import "model/startworkflowrequest.proto"; import "model/rerunworkflowrequest.proto"; option java_package = "com.netflix.conductor.grpc"; option java_outer_classname = "WorkflowServicePb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/grpc/workflows"; service WorkflowService { // POST / rpc StartWorkflow(conductor.proto.StartWorkflowRequest) returns (StartWorkflowResponse); // GET /{name}/correlated/{correlationId} rpc GetWorkflows(GetWorkflowsRequest) returns (GetWorkflowsResponse); // GET /{workflowId} rpc GetWorkflowStatus(GetWorkflowStatusRequest) returns (conductor.proto.Workflow); // DELETE /{workflodId}/remove rpc RemoveWorkflow(RemoveWorkflowRequest) returns (RemoveWorkflowResponse); // GET /running/{name} rpc GetRunningWorkflows(GetRunningWorkflowsRequest) returns (GetRunningWorkflowsResponse); // PUT /decide/{workflowId} rpc DecideWorkflow(DecideWorkflowRequest) returns (DecideWorkflowResponse); // PUT /{workflowId}/pause rpc PauseWorkflow(PauseWorkflowRequest) returns (PauseWorkflowResponse); // PUT /{workflowId}/pause rpc ResumeWorkflow(ResumeWorkflowRequest) returns (ResumeWorkflowResponse); // PUT /{workflowId}/skiptask/{taskReferenceName} rpc SkipTaskFromWorkflow(SkipTaskRequest) returns (SkipTaskResponse); // POST /{workflowId}/rerun rpc RerunWorkflow(conductor.proto.RerunWorkflowRequest) returns (RerunWorkflowResponse); // POST /{workflowId}/restart rpc RestartWorkflow(RestartWorkflowRequest) returns (RestartWorkflowResponse); // POST /{workflowId}retry rpc RetryWorkflow(RetryWorkflowRequest) returns (RetryWorkflowResponse); // POST /{workflowId}/resetcallbacks rpc ResetWorkflowCallbacks(ResetWorkflowCallbacksRequest) returns (ResetWorkflowCallbacksResponse); // DELETE /{workflowId} rpc TerminateWorkflow(TerminateWorkflowRequest) returns (TerminateWorkflowResponse); // GET /search rpc Search(conductor.grpc.search.Request) returns (WorkflowSummarySearchResult); rpc SearchByTasks(conductor.grpc.search.Request) returns (WorkflowSummarySearchResult); // GET /searchV2 rpc SearchV2(conductor.grpc.search.Request) returns (WorkflowSearchResult); rpc SearchByTasksV2(conductor.grpc.search.Request) returns (WorkflowSearchResult); } message StartWorkflowResponse { string workflow_id = 1; } message GetWorkflowsRequest { string name = 1; repeated string correlation_id = 2; bool include_closed = 3; bool include_tasks = 4; } message GetWorkflowsResponse { message Workflows { repeated conductor.proto.Workflow workflows = 1; } map workflows_by_id = 1; } message GetWorkflowStatusRequest { string workflow_id = 1; bool include_tasks = 2; } message GetWorkflowStatusResponse { conductor.proto.Workflow workflow = 1; } message RemoveWorkflowRequest { string workflod_id = 1; bool archive_workflow = 2; } message RemoveWorkflowResponse {} message GetRunningWorkflowsRequest { string name = 1; int32 version = 2; int64 start_time = 3; int64 end_time = 4; } message GetRunningWorkflowsResponse { repeated string workflow_ids = 1; } message DecideWorkflowRequest { string workflow_id = 1; } message DecideWorkflowResponse {} message PauseWorkflowRequest { string workflow_id = 1; } message PauseWorkflowResponse {} message ResumeWorkflowRequest { string workflow_id = 1; } message ResumeWorkflowResponse {} message SkipTaskRequest { string workflow_id = 1; string task_reference_name = 2; conductor.proto.SkipTaskRequest request = 3; } message SkipTaskResponse {} message RerunWorkflowResponse { string workflow_id = 1; } message RestartWorkflowRequest { string workflow_id = 1; bool use_latest_definitions = 2; } message RestartWorkflowResponse {} message RetryWorkflowRequest { string workflow_id = 1; bool resume_subworkflow_tasks = 2; } message RetryWorkflowResponse {} message ResetWorkflowCallbacksRequest { string workflow_id = 1; } message ResetWorkflowCallbacksResponse {} message TerminateWorkflowRequest { string workflow_id = 1; string reason = 2; } message TerminateWorkflowResponse {} message WorkflowSummarySearchResult { int64 total_hits = 1; repeated conductor.proto.WorkflowSummary results = 2; } message WorkflowSearchResult { int64 total_hits = 1; repeated conductor.proto.Workflow results = 2; } ================================================ FILE: grpc/src/main/proto/model/dynamicforkjointask.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "DynamicForkJoinTaskPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message DynamicForkJoinTask { string task_name = 1; string workflow_name = 2; string reference_name = 3; map input = 4; string type = 5; } ================================================ FILE: grpc/src/main/proto/model/dynamicforkjointasklist.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/dynamicforkjointask.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "DynamicForkJoinTaskListPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message DynamicForkJoinTaskList { repeated DynamicForkJoinTask dynamic_tasks = 1; } ================================================ FILE: grpc/src/main/proto/model/eventexecution.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/eventhandler.proto"; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "EventExecutionPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message EventExecution { enum Status { IN_PROGRESS = 0; COMPLETED = 1; FAILED = 2; SKIPPED = 3; } string id = 1; string message_id = 2; string name = 3; string event = 4; int64 created = 5; EventExecution.Status status = 6; EventHandler.Action.Type action = 7; map output = 8; } ================================================ FILE: grpc/src/main/proto/model/eventhandler.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "EventHandlerPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message EventHandler { message StartWorkflow { string name = 1; int32 version = 2; string correlation_id = 3; map input = 4; google.protobuf.Any input_message = 5; map task_to_domain = 6; } message TaskDetails { string workflow_id = 1; string task_ref_name = 2; map output = 3; google.protobuf.Any output_message = 4; string task_id = 5; } message Action { enum Type { START_WORKFLOW = 0; COMPLETE_TASK = 1; FAIL_TASK = 2; } EventHandler.Action.Type action = 1; EventHandler.StartWorkflow start_workflow = 2; EventHandler.TaskDetails complete_task = 3; EventHandler.TaskDetails fail_task = 4; bool expand_inline_json = 5; } string name = 1; string event = 2; string condition = 3; repeated EventHandler.Action actions = 4; bool active = 5; string evaluator_type = 6; } ================================================ FILE: grpc/src/main/proto/model/polldata.proto ================================================ syntax = "proto3"; package conductor.proto; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "PollDataPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message PollData { string queue_name = 1; string domain = 2; string worker_id = 3; int64 last_poll_time = 4; } ================================================ FILE: grpc/src/main/proto/model/rerunworkflowrequest.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "RerunWorkflowRequestPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message RerunWorkflowRequest { string re_run_from_workflow_id = 1; map workflow_input = 2; string re_run_from_task_id = 3; map task_input = 4; string correlation_id = 5; } ================================================ FILE: grpc/src/main/proto/model/skiptaskrequest.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "SkipTaskRequestPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message SkipTaskRequest { map task_input = 1; map task_output = 2; google.protobuf.Any task_input_message = 3; google.protobuf.Any task_output_message = 4; } ================================================ FILE: grpc/src/main/proto/model/startworkflowrequest.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/workflowdef.proto"; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "StartWorkflowRequestPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message StartWorkflowRequest { string name = 1; int32 version = 2; string correlation_id = 3; map input = 4; map task_to_domain = 5; WorkflowDef workflow_def = 6; string external_input_payload_storage_path = 7; int32 priority = 8; } ================================================ FILE: grpc/src/main/proto/model/subworkflowparams.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "SubWorkflowParamsPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message SubWorkflowParams { string name = 1; int32 version = 2; map task_to_domain = 3; google.protobuf.Value workflow_definition = 4; } ================================================ FILE: grpc/src/main/proto/model/task.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/workflowtask.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "TaskPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message Task { enum Status { IN_PROGRESS = 0; CANCELED = 1; FAILED = 2; FAILED_WITH_TERMINAL_ERROR = 3; COMPLETED = 4; COMPLETED_WITH_ERRORS = 5; SCHEDULED = 6; TIMED_OUT = 7; SKIPPED = 8; } string task_type = 1; Task.Status status = 2; map input_data = 3; string reference_task_name = 4; int32 retry_count = 5; int32 seq = 6; string correlation_id = 7; int32 poll_count = 8; string task_def_name = 9; int64 scheduled_time = 10; int64 start_time = 11; int64 end_time = 12; int64 update_time = 13; int32 start_delay_in_seconds = 14; string retried_task_id = 15; bool retried = 16; bool executed = 17; bool callback_from_worker = 18; int64 response_timeout_seconds = 19; string workflow_instance_id = 20; string workflow_type = 21; string task_id = 22; string reason_for_incompletion = 23; int64 callback_after_seconds = 24; string worker_id = 25; map output_data = 26; WorkflowTask workflow_task = 27; string domain = 28; google.protobuf.Any input_message = 29; google.protobuf.Any output_message = 30; int32 rate_limit_per_frequency = 32; int32 rate_limit_frequency_in_seconds = 33; string external_input_payload_storage_path = 34; string external_output_payload_storage_path = 35; int32 workflow_priority = 36; string execution_name_space = 37; string isolation_group_id = 38; int32 iteration = 40; string sub_workflow_id = 41; bool subworkflow_changed = 42; } ================================================ FILE: grpc/src/main/proto/model/taskdef.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "TaskDefPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message TaskDef { enum TimeoutPolicy { RETRY = 0; TIME_OUT_WF = 1; ALERT_ONLY = 2; } enum RetryLogic { FIXED = 0; EXPONENTIAL_BACKOFF = 1; LINEAR_BACKOFF = 2; } string name = 1; string description = 2; int32 retry_count = 3; int64 timeout_seconds = 4; repeated string input_keys = 5; repeated string output_keys = 6; TaskDef.TimeoutPolicy timeout_policy = 7; TaskDef.RetryLogic retry_logic = 8; int32 retry_delay_seconds = 9; int64 response_timeout_seconds = 10; int32 concurrent_exec_limit = 11; map input_template = 12; int32 rate_limit_per_frequency = 14; int32 rate_limit_frequency_in_seconds = 15; string isolation_group_id = 16; string execution_name_space = 17; string owner_email = 18; int32 poll_timeout_seconds = 19; int32 backoff_scale_factor = 20; } ================================================ FILE: grpc/src/main/proto/model/taskexeclog.proto ================================================ syntax = "proto3"; package conductor.proto; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "TaskExecLogPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message TaskExecLog { string log = 1; string task_id = 2; int64 created_time = 3; } ================================================ FILE: grpc/src/main/proto/model/taskresult.proto ================================================ syntax = "proto3"; package conductor.proto; import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "TaskResultPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message TaskResult { enum Status { IN_PROGRESS = 0; FAILED = 1; FAILED_WITH_TERMINAL_ERROR = 2; COMPLETED = 3; } string workflow_instance_id = 1; string task_id = 2; string reason_for_incompletion = 3; int64 callback_after_seconds = 4; string worker_id = 5; TaskResult.Status status = 6; map output_data = 7; google.protobuf.Any output_message = 8; } ================================================ FILE: grpc/src/main/proto/model/tasksummary.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/task.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "TaskSummaryPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message TaskSummary { string workflow_id = 1; string workflow_type = 2; string correlation_id = 3; string scheduled_time = 4; string start_time = 5; string update_time = 6; string end_time = 7; Task.Status status = 8; string reason_for_incompletion = 9; int64 execution_time = 10; int64 queue_wait_time = 11; string task_def_name = 12; string task_type = 13; string input = 14; string output = 15; string task_id = 16; string external_input_payload_storage_path = 17; string external_output_payload_storage_path = 18; int32 workflow_priority = 19; string domain = 20; } ================================================ FILE: grpc/src/main/proto/model/workflow.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/workflowdef.proto"; import "model/task.proto"; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "WorkflowPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message Workflow { enum WorkflowStatus { RUNNING = 0; COMPLETED = 1; FAILED = 2; TIMED_OUT = 3; TERMINATED = 4; PAUSED = 5; } Workflow.WorkflowStatus status = 1; int64 end_time = 2; string workflow_id = 3; string parent_workflow_id = 4; string parent_workflow_task_id = 5; repeated Task tasks = 6; map input = 8; map output = 9; string correlation_id = 12; string re_run_from_workflow_id = 13; string reason_for_incompletion = 14; string event = 16; map task_to_domain = 17; repeated string failed_reference_task_names = 18; WorkflowDef workflow_definition = 19; string external_input_payload_storage_path = 20; string external_output_payload_storage_path = 21; int32 priority = 22; map variables = 23; int64 last_retried_time = 24; repeated string failed_task_names = 25; } ================================================ FILE: grpc/src/main/proto/model/workflowdef.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/workflowtask.proto"; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "WorkflowDefPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message WorkflowDef { enum TimeoutPolicy { TIME_OUT_WF = 0; ALERT_ONLY = 1; } string name = 1; string description = 2; int32 version = 3; repeated WorkflowTask tasks = 4; repeated string input_parameters = 5; map output_parameters = 6; string failure_workflow = 7; int32 schema_version = 8; bool restartable = 9; bool workflow_status_listener_enabled = 10; string owner_email = 11; WorkflowDef.TimeoutPolicy timeout_policy = 12; int64 timeout_seconds = 13; map variables = 14; map input_template = 15; } ================================================ FILE: grpc/src/main/proto/model/workflowdefsummary.proto ================================================ syntax = "proto3"; package conductor.proto; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "WorkflowDefSummaryPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message WorkflowDefSummary { string name = 1; int32 version = 2; int64 create_time = 3; } ================================================ FILE: grpc/src/main/proto/model/workflowsummary.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/workflow.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "WorkflowSummaryPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message WorkflowSummary { string workflow_type = 1; int32 version = 2; string workflow_id = 3; string correlation_id = 4; string start_time = 5; string update_time = 6; string end_time = 7; Workflow.WorkflowStatus status = 8; string input = 9; string output = 10; string reason_for_incompletion = 11; int64 execution_time = 12; string event = 13; string failed_reference_task_names = 14; string external_input_payload_storage_path = 15; string external_output_payload_storage_path = 16; int32 priority = 17; repeated string failed_task_names = 18; } ================================================ FILE: grpc/src/main/proto/model/workflowtask.proto ================================================ syntax = "proto3"; package conductor.proto; import "model/taskdef.proto"; import "model/subworkflowparams.proto"; import "google/protobuf/struct.proto"; option java_package = "com.netflix.conductor.proto"; option java_outer_classname = "WorkflowTaskPb"; option go_package = "github.com/netflix/conductor/client/gogrpc/conductor/model"; message WorkflowTask { message WorkflowTaskList { repeated WorkflowTask tasks = 1; } string name = 1; string task_reference_name = 2; string description = 3; map input_parameters = 4; string type = 5; string dynamic_task_name_param = 6; string case_value_param = 7; string case_expression = 8; string script_expression = 22; map decision_cases = 9; string dynamic_fork_tasks_param = 10; string dynamic_fork_tasks_input_param_name = 11; repeated WorkflowTask default_case = 12; repeated WorkflowTask.WorkflowTaskList fork_tasks = 13; int32 start_delay = 14; SubWorkflowParams sub_workflow_param = 15; repeated string join_on = 16; string sink = 17; bool optional = 18; TaskDef task_definition = 19; bool rate_limited = 20; repeated string default_exclusive_join_task = 21; bool async_complete = 23; string loop_condition = 24; repeated WorkflowTask loop_over = 25; int32 retry_count = 26; string evaluator_type = 27; string expression = 28; } ================================================ FILE: grpc/src/test/java/com/netflix/conductor/grpc/TestProtoMapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.proto.WorkflowTaskPb; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; public class TestProtoMapper { private final ProtoMapper mapper = ProtoMapper.INSTANCE; @Test public void workflowTaskToProto() { final WorkflowTask taskWithDefaultRetryCount = new WorkflowTask(); final WorkflowTask taskWith1RetryCount = new WorkflowTask(); taskWith1RetryCount.setRetryCount(1); final WorkflowTask taskWithNoRetryCount = new WorkflowTask(); taskWithNoRetryCount.setRetryCount(0); assertEquals(-1, mapper.toProto(taskWithDefaultRetryCount).getRetryCount()); assertEquals(1, mapper.toProto(taskWith1RetryCount).getRetryCount()); assertEquals(0, mapper.toProto(taskWithNoRetryCount).getRetryCount()); } @Test public void workflowTaskFromProto() { final WorkflowTaskPb.WorkflowTask taskWithDefaultRetryCount = WorkflowTaskPb.WorkflowTask.newBuilder().build(); final WorkflowTaskPb.WorkflowTask taskWith1RetryCount = WorkflowTaskPb.WorkflowTask.newBuilder().setRetryCount(1).build(); final WorkflowTaskPb.WorkflowTask taskWithNoRetryCount = WorkflowTaskPb.WorkflowTask.newBuilder().setRetryCount(-1).build(); assertEquals(new Integer(0), mapper.fromProto(taskWithDefaultRetryCount).getRetryCount()); assertEquals(1, mapper.fromProto(taskWith1RetryCount).getRetryCount().intValue()); assertNull(mapper.fromProto(taskWithNoRetryCount).getRetryCount()); } } ================================================ FILE: grpc-client/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-grpc') implementation "io.grpc:grpc-netty:${revGrpc}" implementation "io.grpc:grpc-protobuf:${revGrpc}" implementation "io.grpc:grpc-stub:${revGrpc}" implementation "com.google.protobuf:protobuf-java:${revProtoBuf}" implementation "org.slf4j:slf4j-api" implementation "org.apache.commons:commons-lang3" implementation "com.google.guava:guava:${revGuava}" } ================================================ FILE: grpc-client/src/main/java/com/netflix/conductor/client/grpc/ClientBase.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.grpc.SearchPb; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; abstract class ClientBase { private static final Logger LOGGER = LoggerFactory.getLogger(ClientBase.class); protected static ProtoMapper protoMapper = ProtoMapper.INSTANCE; protected final ManagedChannel channel; public ClientBase(String address, int port) { this(ManagedChannelBuilder.forAddress(address, port).usePlaintext()); } public ClientBase(ManagedChannelBuilder builder) { channel = builder.build(); } public void shutdown() throws InterruptedException { channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); } SearchPb.Request createSearchRequest( @Nullable Integer start, @Nullable Integer size, @Nullable String sort, @Nullable String freeText, @Nullable String query) { SearchPb.Request.Builder request = SearchPb.Request.newBuilder(); if (start != null) request.setStart(start); if (size != null) request.setSize(size); if (sort != null) request.setSort(sort); if (freeText != null) request.setFreeText(freeText); if (query != null) request.setQuery(query); return request.build(); } } ================================================ FILE: grpc-client/src/main/java/com/netflix/conductor/client/grpc/EventClient.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.Iterator; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.grpc.EventServiceGrpc; import com.netflix.conductor.grpc.EventServicePb; import com.netflix.conductor.proto.EventHandlerPb; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import io.grpc.ManagedChannelBuilder; public class EventClient extends ClientBase { private final EventServiceGrpc.EventServiceBlockingStub stub; public EventClient(String address, int port) { super(address, port); this.stub = EventServiceGrpc.newBlockingStub(this.channel); } public EventClient(ManagedChannelBuilder builder) { super(builder); this.stub = EventServiceGrpc.newBlockingStub(this.channel); } /** * Register an event handler with the server * * @param eventHandler the event handler definition */ public void registerEventHandler(EventHandler eventHandler) { Preconditions.checkNotNull(eventHandler, "Event handler definition cannot be null"); stub.addEventHandler( EventServicePb.AddEventHandlerRequest.newBuilder() .setHandler(protoMapper.toProto(eventHandler)) .build()); } /** * Updates an existing event handler * * @param eventHandler the event handler to be updated */ public void updateEventHandler(EventHandler eventHandler) { Preconditions.checkNotNull(eventHandler, "Event handler definition cannot be null"); stub.updateEventHandler( EventServicePb.UpdateEventHandlerRequest.newBuilder() .setHandler(protoMapper.toProto(eventHandler)) .build()); } /** * @param event name of the event * @param activeOnly if true, returns only the active handlers * @return Returns the list of all the event handlers for a given event */ public Iterator getEventHandlers(String event, boolean activeOnly) { Preconditions.checkArgument(StringUtils.isNotBlank(event), "Event cannot be blank"); EventServicePb.GetEventHandlersForEventRequest.Builder request = EventServicePb.GetEventHandlersForEventRequest.newBuilder() .setEvent(event) .setActiveOnly(activeOnly); Iterator it = stub.getEventHandlersForEvent(request.build()); return Iterators.transform(it, protoMapper::fromProto); } /** * Removes the event handler from the conductor server * * @param name the name of the event handler */ public void unregisterEventHandler(String name) { Preconditions.checkArgument(StringUtils.isNotBlank(name), "Name cannot be blank"); stub.removeEventHandler( EventServicePb.RemoveEventHandlerRequest.newBuilder().setName(name).build()); } } ================================================ FILE: grpc-client/src/main/java/com/netflix/conductor/client/grpc/MetadataClient.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.List; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.grpc.MetadataServiceGrpc; import com.netflix.conductor.grpc.MetadataServicePb; import com.google.common.base.Preconditions; import io.grpc.ManagedChannelBuilder; public class MetadataClient extends ClientBase { private final MetadataServiceGrpc.MetadataServiceBlockingStub stub; public MetadataClient(String address, int port) { super(address, port); this.stub = MetadataServiceGrpc.newBlockingStub(this.channel); } public MetadataClient(ManagedChannelBuilder builder) { super(builder); this.stub = MetadataServiceGrpc.newBlockingStub(this.channel); } /** * Register a workflow definition with the server * * @param workflowDef the workflow definition */ public void registerWorkflowDef(WorkflowDef workflowDef) { Preconditions.checkNotNull(workflowDef, "Worfklow definition cannot be null"); stub.createWorkflow( MetadataServicePb.CreateWorkflowRequest.newBuilder() .setWorkflow(protoMapper.toProto(workflowDef)) .build()); } /** * Updates a list of existing workflow definitions * * @param workflowDefs List of workflow definitions to be updated */ public void updateWorkflowDefs(List workflowDefs) { Preconditions.checkNotNull(workflowDefs, "Workflow defs list cannot be null"); stub.updateWorkflows( MetadataServicePb.UpdateWorkflowsRequest.newBuilder() .addAllDefs(workflowDefs.stream().map(protoMapper::toProto)::iterator) .build()); } /** * Retrieve the workflow definition * * @param name the name of the workflow * @param version the version of the workflow def * @return Workflow definition for the given workflow and version */ public WorkflowDef getWorkflowDef(String name, @Nullable Integer version) { Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be blank"); MetadataServicePb.GetWorkflowRequest.Builder request = MetadataServicePb.GetWorkflowRequest.newBuilder().setName(name); if (version != null) { request.setVersion(version); } return protoMapper.fromProto(stub.getWorkflow(request.build()).getWorkflow()); } /** * Registers a list of task types with the conductor server * * @param taskDefs List of task types to be registered. */ public void registerTaskDefs(List taskDefs) { Preconditions.checkNotNull(taskDefs, "Task defs list cannot be null"); stub.createTasks( MetadataServicePb.CreateTasksRequest.newBuilder() .addAllDefs(taskDefs.stream().map(protoMapper::toProto)::iterator) .build()); } /** * Updates an existing task definition * * @param taskDef the task definition to be updated */ public void updateTaskDef(TaskDef taskDef) { Preconditions.checkNotNull(taskDef, "Task definition cannot be null"); stub.updateTask( MetadataServicePb.UpdateTaskRequest.newBuilder() .setTask(protoMapper.toProto(taskDef)) .build()); } /** * Retrieve the task definition of a given task type * * @param taskType type of task for which to retrieve the definition * @return Task Definition for the given task type */ public TaskDef getTaskDef(String taskType) { Preconditions.checkArgument(StringUtils.isNotBlank(taskType), "Task type cannot be blank"); return protoMapper.fromProto( stub.getTask( MetadataServicePb.GetTaskRequest.newBuilder() .setTaskType(taskType) .build()) .getTask()); } /** * Removes the task definition of a task type from the conductor server. Use with caution. * * @param taskType Task type to be unregistered. */ public void unregisterTaskDef(String taskType) { Preconditions.checkArgument(StringUtils.isNotBlank(taskType), "Task type cannot be blank"); stub.deleteTask( MetadataServicePb.DeleteTaskRequest.newBuilder().setTaskType(taskType).build()); } } ================================================ FILE: grpc-client/src/main/java/com/netflix/conductor/client/grpc/TaskClient.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.TaskServiceGrpc; import com.netflix.conductor.grpc.TaskServicePb; import com.netflix.conductor.proto.TaskPb; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import io.grpc.ManagedChannelBuilder; public class TaskClient extends ClientBase { private final TaskServiceGrpc.TaskServiceBlockingStub stub; public TaskClient(String address, int port) { super(address, port); this.stub = TaskServiceGrpc.newBlockingStub(this.channel); } public TaskClient(ManagedChannelBuilder builder) { super(builder); this.stub = TaskServiceGrpc.newBlockingStub(this.channel); } /** * Perform a poll for a task of a specific task type. * * @param taskType The taskType to poll for * @param domain The domain of the task type * @param workerId Name of the client worker. Used for logging. * @return Task waiting to be executed. */ public Task pollTask(String taskType, String workerId, String domain) { Preconditions.checkArgument(StringUtils.isNotBlank(taskType), "Task type cannot be blank"); Preconditions.checkArgument(StringUtils.isNotBlank(domain), "Domain cannot be blank"); Preconditions.checkArgument(StringUtils.isNotBlank(workerId), "Worker id cannot be blank"); TaskServicePb.PollResponse response = stub.poll( TaskServicePb.PollRequest.newBuilder() .setTaskType(taskType) .setWorkerId(workerId) .setDomain(domain) .build()); return protoMapper.fromProto(response.getTask()); } /** * Perform a batch poll for tasks by task type. Batch size is configurable by count. * * @param taskType Type of task to poll for * @param workerId Name of the client worker. Used for logging. * @param count Maximum number of tasks to be returned. Actual number of tasks returned can be * less than this number. * @param timeoutInMillisecond Long poll wait timeout. * @return List of tasks awaiting to be executed. */ public List batchPollTasksByTaskType( String taskType, String workerId, int count, int timeoutInMillisecond) { return Lists.newArrayList( batchPollTasksByTaskTypeAsync(taskType, workerId, count, timeoutInMillisecond)); } /** * Perform a batch poll for tasks by task type. Batch size is configurable by count. Returns an * iterator that streams tasks as they become available through GRPC. * * @param taskType Type of task to poll for * @param workerId Name of the client worker. Used for logging. * @param count Maximum number of tasks to be returned. Actual number of tasks returned can be * less than this number. * @param timeoutInMillisecond Long poll wait timeout. * @return Iterator of tasks awaiting to be executed. */ public Iterator batchPollTasksByTaskTypeAsync( String taskType, String workerId, int count, int timeoutInMillisecond) { Preconditions.checkArgument(StringUtils.isNotBlank(taskType), "Task type cannot be blank"); Preconditions.checkArgument(StringUtils.isNotBlank(workerId), "Worker id cannot be blank"); Preconditions.checkArgument(count > 0, "Count must be greater than 0"); Iterator it = stub.batchPoll( TaskServicePb.BatchPollRequest.newBuilder() .setTaskType(taskType) .setWorkerId(workerId) .setCount(count) .setTimeout(timeoutInMillisecond) .build()); return Iterators.transform(it, protoMapper::fromProto); } /** * Updates the result of a task execution. * * @param taskResult TaskResults to be updated. */ public void updateTask(TaskResult taskResult) { Preconditions.checkNotNull(taskResult, "Task result cannot be null"); stub.updateTask( TaskServicePb.UpdateTaskRequest.newBuilder() .setResult(protoMapper.toProto(taskResult)) .build()); } /** * Log execution messages for a task. * * @param taskId id of the task * @param logMessage the message to be logged */ public void logMessageForTask(String taskId, String logMessage) { Preconditions.checkArgument(StringUtils.isNotBlank(taskId), "Task id cannot be blank"); stub.addLog( TaskServicePb.AddLogRequest.newBuilder() .setTaskId(taskId) .setLog(logMessage) .build()); } /** * Fetch execution logs for a task. * * @param taskId id of the task. */ public List getTaskLogs(String taskId) { Preconditions.checkArgument(StringUtils.isNotBlank(taskId), "Task id cannot be blank"); return stub .getTaskLogs( TaskServicePb.GetTaskLogsRequest.newBuilder().setTaskId(taskId).build()) .getLogsList() .stream() .map(protoMapper::fromProto) .collect(Collectors.toList()); } /** * Retrieve information about the task * * @param taskId ID of the task * @return Task details */ public Task getTaskDetails(String taskId) { Preconditions.checkArgument(StringUtils.isNotBlank(taskId), "Task id cannot be blank"); return protoMapper.fromProto( stub.getTask(TaskServicePb.GetTaskRequest.newBuilder().setTaskId(taskId).build()) .getTask()); } public int getQueueSizeForTask(String taskType) { Preconditions.checkArgument(StringUtils.isNotBlank(taskType), "Task type cannot be blank"); TaskServicePb.QueueSizesResponse sizes = stub.getQueueSizesForTasks( TaskServicePb.QueueSizesRequest.newBuilder() .addTaskTypes(taskType) .build()); return sizes.getQueueForTaskOrDefault(taskType, 0); } public SearchResult search(String query) { return search(null, null, null, null, query); } public SearchResult searchV2(String query) { return searchV2(null, null, null, null, query); } public SearchResult search( @Nullable Integer start, @Nullable Integer size, @Nullable String sort, @Nullable String freeText, @Nullable String query) { SearchPb.Request searchRequest = createSearchRequest(start, size, sort, freeText, query); TaskServicePb.TaskSummarySearchResult result = stub.search(searchRequest); return new SearchResult<>( result.getTotalHits(), result.getResultsList().stream() .map(protoMapper::fromProto) .collect(Collectors.toList())); } public SearchResult searchV2( @Nullable Integer start, @Nullable Integer size, @Nullable String sort, @Nullable String freeText, @Nullable String query) { SearchPb.Request searchRequest = createSearchRequest(start, size, sort, freeText, query); TaskServicePb.TaskSearchResult result = stub.searchV2(searchRequest); return new SearchResult<>( result.getTotalHits(), result.getResultsList().stream() .map(protoMapper::fromProto) .collect(Collectors.toList())); } } ================================================ FILE: grpc-client/src/main/java/com/netflix/conductor/client/grpc/WorkflowClient.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.WorkflowServiceGrpc; import com.netflix.conductor.grpc.WorkflowServicePb; import com.netflix.conductor.proto.WorkflowPb; import com.google.common.base.Preconditions; import io.grpc.ManagedChannelBuilder; public class WorkflowClient extends ClientBase { private final WorkflowServiceGrpc.WorkflowServiceBlockingStub stub; public WorkflowClient(String address, int port) { super(address, port); this.stub = WorkflowServiceGrpc.newBlockingStub(this.channel); } public WorkflowClient(ManagedChannelBuilder builder) { super(builder); this.stub = WorkflowServiceGrpc.newBlockingStub(this.channel); } /** * Starts a workflow * * @param startWorkflowRequest the {@link StartWorkflowRequest} object to start the workflow * @return the id of the workflow instance that can be used for tracking */ public String startWorkflow(StartWorkflowRequest startWorkflowRequest) { Preconditions.checkNotNull(startWorkflowRequest, "StartWorkflowRequest cannot be null"); return stub.startWorkflow(protoMapper.toProto(startWorkflowRequest)).getWorkflowId(); } /** * Retrieve a workflow by workflow id * * @param workflowId the id of the workflow * @param includeTasks specify if the tasks in the workflow need to be returned * @return the requested workflow */ public Workflow getWorkflow(String workflowId, boolean includeTasks) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); WorkflowPb.Workflow workflow = stub.getWorkflowStatus( WorkflowServicePb.GetWorkflowStatusRequest.newBuilder() .setWorkflowId(workflowId) .setIncludeTasks(includeTasks) .build()); return protoMapper.fromProto(workflow); } /** * Retrieve all workflows for a given correlation id and name * * @param name the name of the workflow * @param correlationId the correlation id * @param includeClosed specify if all workflows are to be returned or only running workflows * @param includeTasks specify if the tasks in the workflow need to be returned * @return list of workflows for the given correlation id and name */ public List getWorkflows( String name, String correlationId, boolean includeClosed, boolean includeTasks) { Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be blank"); Preconditions.checkArgument( StringUtils.isNotBlank(correlationId), "correlationId cannot be blank"); WorkflowServicePb.GetWorkflowsResponse workflows = stub.getWorkflows( WorkflowServicePb.GetWorkflowsRequest.newBuilder() .setName(name) .addCorrelationId(correlationId) .setIncludeClosed(includeClosed) .setIncludeTasks(includeTasks) .build()); if (!workflows.containsWorkflowsById(correlationId)) { return Collections.emptyList(); } return workflows.getWorkflowsByIdOrThrow(correlationId).getWorkflowsList().stream() .map(protoMapper::fromProto) .collect(Collectors.toList()); } /** * Removes a workflow from the system * * @param workflowId the id of the workflow to be deleted * @param archiveWorkflow flag to indicate if the workflow should be archived before deletion */ public void deleteWorkflow(String workflowId, boolean archiveWorkflow) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "Workflow id cannot be blank"); stub.removeWorkflow( WorkflowServicePb.RemoveWorkflowRequest.newBuilder() .setWorkflodId(workflowId) .setArchiveWorkflow(archiveWorkflow) .build()); } /* * Retrieve all running workflow instances for a given name and version * * @param workflowName the name of the workflow * @param version the version of the wokflow definition. Defaults to 1. * @return the list of running workflow instances */ public List getRunningWorkflow(String workflowName, @Nullable Integer version) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name cannot be blank"); WorkflowServicePb.GetRunningWorkflowsResponse workflows = stub.getRunningWorkflows( WorkflowServicePb.GetRunningWorkflowsRequest.newBuilder() .setName(workflowName) .setVersion(version == null ? 1 : version) .build()); return workflows.getWorkflowIdsList(); } /** * Retrieve all workflow instances for a given workflow name between a specific time period * * @param workflowName the name of the workflow * @param version the version of the workflow definition. Defaults to 1. * @param startTime the start time of the period * @param endTime the end time of the period * @return returns a list of workflows created during the specified during the time period */ public List getWorkflowsByTimePeriod( String workflowName, int version, Long startTime, Long endTime) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name cannot be blank"); Preconditions.checkNotNull(startTime, "Start time cannot be null"); Preconditions.checkNotNull(endTime, "End time cannot be null"); // TODO return null; } /* * Starts the decision task for the given workflow instance * * @param workflowId the id of the workflow instance */ public void runDecider(String workflowId) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.decideWorkflow( WorkflowServicePb.DecideWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .build()); } /** * Pause a workflow by workflow id * * @param workflowId the workflow id of the workflow to be paused */ public void pauseWorkflow(String workflowId) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.pauseWorkflow( WorkflowServicePb.PauseWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .build()); } /** * Resume a paused workflow by workflow id * * @param workflowId the workflow id of the paused workflow */ public void resumeWorkflow(String workflowId) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.resumeWorkflow( WorkflowServicePb.ResumeWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .build()); } /** * Skips a given task from a current RUNNING workflow * * @param workflowId the id of the workflow instance * @param taskReferenceName the reference name of the task to be skipped */ public void skipTaskFromWorkflow(String workflowId, String taskReferenceName) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); Preconditions.checkArgument( StringUtils.isNotBlank(taskReferenceName), "Task reference name cannot be blank"); stub.skipTaskFromWorkflow( WorkflowServicePb.SkipTaskRequest.newBuilder() .setWorkflowId(workflowId) .setTaskReferenceName(taskReferenceName) .build()); } /** * Reruns the workflow from a specific task * * @param rerunWorkflowRequest the request containing the task to rerun from * @return the id of the workflow */ public String rerunWorkflow(RerunWorkflowRequest rerunWorkflowRequest) { Preconditions.checkNotNull(rerunWorkflowRequest, "RerunWorkflowRequest cannot be null"); return stub.rerunWorkflow(protoMapper.toProto(rerunWorkflowRequest)).getWorkflowId(); } /** * Restart a completed workflow * * @param workflowId the workflow id of the workflow to be restarted */ public void restart(String workflowId, boolean useLatestDefinitions) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.restartWorkflow( WorkflowServicePb.RestartWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .setUseLatestDefinitions(useLatestDefinitions) .build()); } /** * Retries the last failed task in a workflow * * @param workflowId the workflow id of the workflow with the failed task */ public void retryLastFailedTask(String workflowId, boolean resumeSubworkflowTasks) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.retryWorkflow( WorkflowServicePb.RetryWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .setResumeSubworkflowTasks(resumeSubworkflowTasks) .build()); } /** * Resets the callback times of all IN PROGRESS tasks to 0 for the given workflow * * @param workflowId the id of the workflow */ public void resetCallbacksForInProgressTasks(String workflowId) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.resetWorkflowCallbacks( WorkflowServicePb.ResetWorkflowCallbacksRequest.newBuilder() .setWorkflowId(workflowId) .build()); } /** * Terminates the execution of the given workflow instance * * @param workflowId the id of the workflow to be terminated * @param reason the reason to be logged and displayed */ public void terminateWorkflow(String workflowId, String reason) { Preconditions.checkArgument( StringUtils.isNotBlank(workflowId), "workflow id cannot be blank"); stub.terminateWorkflow( WorkflowServicePb.TerminateWorkflowRequest.newBuilder() .setWorkflowId(workflowId) .setReason(reason) .build()); } /** * Search for workflows based on payload * * @param query the search query * @return the {@link SearchResult} containing the {@link WorkflowSummary} that match the query */ public SearchResult search(String query) { return search(null, null, null, null, query); } /** * Search for workflows based on payload * * @param query the search query * @return the {@link SearchResult} containing the {@link Workflow} that match the query */ public SearchResult searchV2(String query) { return searchV2(null, null, null, null, query); } /** * Paginated search for workflows based on payload * * @param start start value of page * @param size number of workflows to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link WorkflowSummary} that match the query */ public SearchResult search( @Nullable Integer start, @Nullable Integer size, @Nullable String sort, @Nullable String freeText, @Nullable String query) { SearchPb.Request searchRequest = createSearchRequest(start, size, sort, freeText, query); WorkflowServicePb.WorkflowSummarySearchResult result = stub.search(searchRequest); return new SearchResult<>( result.getTotalHits(), result.getResultsList().stream() .map(protoMapper::fromProto) .collect(Collectors.toList())); } /** * Paginated search for workflows based on payload * * @param start start value of page * @param size number of workflows to be returned * @param sort sort order * @param freeText additional free text query * @param query the search query * @return the {@link SearchResult} containing the {@link Workflow} that match the query */ public SearchResult searchV2( @Nullable Integer start, @Nullable Integer size, @Nullable String sort, @Nullable String freeText, @Nullable String query) { SearchPb.Request searchRequest = createSearchRequest(start, size, sort, freeText, query); WorkflowServicePb.WorkflowSearchResult result = stub.searchV2(searchRequest); return new SearchResult<>( result.getTotalHits(), result.getResultsList().stream() .map(protoMapper::fromProto) .collect(Collectors.toList())); } } ================================================ FILE: grpc-client/src/test/java/com/netflix/conductor/client/grpc/EventClientTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.util.ReflectionTestUtils; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.grpc.EventServiceGrpc; import com.netflix.conductor.grpc.EventServicePb; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.proto.EventHandlerPb; import static junit.framework.TestCase.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(SpringRunner.class) public class EventClientTest { @Mock ProtoMapper mockedProtoMapper; @Mock EventServiceGrpc.EventServiceBlockingStub mockedStub; EventClient eventClient; @Before public void init() { eventClient = new EventClient("test", 0); ReflectionTestUtils.setField(eventClient, "stub", mockedStub); ReflectionTestUtils.setField(eventClient, "protoMapper", mockedProtoMapper); } @Test public void testRegisterEventHandler() { EventHandler eventHandler = mock(EventHandler.class); EventHandlerPb.EventHandler eventHandlerPB = mock(EventHandlerPb.EventHandler.class); when(mockedProtoMapper.toProto(eventHandler)).thenReturn(eventHandlerPB); EventServicePb.AddEventHandlerRequest request = EventServicePb.AddEventHandlerRequest.newBuilder() .setHandler(eventHandlerPB) .build(); eventClient.registerEventHandler(eventHandler); verify(mockedStub, times(1)).addEventHandler(request); } @Test public void testUpdateEventHandler() { EventHandler eventHandler = mock(EventHandler.class); EventHandlerPb.EventHandler eventHandlerPB = mock(EventHandlerPb.EventHandler.class); when(mockedProtoMapper.toProto(eventHandler)).thenReturn(eventHandlerPB); EventServicePb.UpdateEventHandlerRequest request = EventServicePb.UpdateEventHandlerRequest.newBuilder() .setHandler(eventHandlerPB) .build(); eventClient.updateEventHandler(eventHandler); verify(mockedStub, times(1)).updateEventHandler(request); } @Test public void testGetEventHandlers() { EventHandler eventHandler = mock(EventHandler.class); EventHandlerPb.EventHandler eventHandlerPB = mock(EventHandlerPb.EventHandler.class); when(mockedProtoMapper.fromProto(eventHandlerPB)).thenReturn(eventHandler); EventServicePb.GetEventHandlersForEventRequest request = EventServicePb.GetEventHandlersForEventRequest.newBuilder() .setEvent("test") .setActiveOnly(true) .build(); List result = new ArrayList<>(); result.add(eventHandlerPB); when(mockedStub.getEventHandlersForEvent(request)).thenReturn(result.iterator()); Iterator response = eventClient.getEventHandlers("test", true); verify(mockedStub, times(1)).getEventHandlersForEvent(request); assertEquals(response.next(), eventHandler); } @Test public void testUnregisterEventHandler() { EventClient eventClient = createClientWithManagedChannel(); EventServicePb.RemoveEventHandlerRequest request = EventServicePb.RemoveEventHandlerRequest.newBuilder().setName("test").build(); eventClient.unregisterEventHandler("test"); verify(mockedStub, times(1)).removeEventHandler(request); } @Test public void testUnregisterEventHandlerWithManagedChannel() { EventServicePb.RemoveEventHandlerRequest request = EventServicePb.RemoveEventHandlerRequest.newBuilder().setName("test").build(); eventClient.unregisterEventHandler("test"); verify(mockedStub, times(1)).removeEventHandler(request); } public EventClient createClientWithManagedChannel() { EventClient eventClient = new EventClient("test", 0); ReflectionTestUtils.setField(eventClient, "stub", mockedStub); ReflectionTestUtils.setField(eventClient, "protoMapper", mockedProtoMapper); return eventClient; } } ================================================ FILE: grpc-client/src/test/java/com/netflix/conductor/client/grpc/TaskClientTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.util.ReflectionTestUtils; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.TaskServiceGrpc; import com.netflix.conductor.grpc.TaskServicePb; import com.netflix.conductor.proto.TaskPb; import com.netflix.conductor.proto.TaskSummaryPb; import io.grpc.ManagedChannelBuilder; import static junit.framework.TestCase.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(SpringRunner.class) public class TaskClientTest { @Mock ProtoMapper mockedProtoMapper; @Mock TaskServiceGrpc.TaskServiceBlockingStub mockedStub; TaskClient taskClient; @Before public void init() { taskClient = new TaskClient("test", 0); ReflectionTestUtils.setField(taskClient, "stub", mockedStub); ReflectionTestUtils.setField(taskClient, "protoMapper", mockedProtoMapper); } @Test public void testSearch() { TaskSummary taskSummary = mock(TaskSummary.class); TaskSummaryPb.TaskSummary taskSummaryPB = mock(TaskSummaryPb.TaskSummary.class); when(mockedProtoMapper.fromProto(taskSummaryPB)).thenReturn(taskSummary); TaskServicePb.TaskSummarySearchResult result = TaskServicePb.TaskSummarySearchResult.newBuilder() .addResults(taskSummaryPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder().setQuery("test query").build(); when(mockedStub.search(searchRequest)).thenReturn(result); SearchResult searchResult = taskClient.search("test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(taskSummary, searchResult.getResults().get(0)); } @Test public void testSearchV2() { Task task = mock(Task.class); TaskPb.Task taskPB = mock(TaskPb.Task.class); when(mockedProtoMapper.fromProto(taskPB)).thenReturn(task); TaskServicePb.TaskSearchResult result = TaskServicePb.TaskSearchResult.newBuilder() .addResults(taskPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder().setQuery("test query").build(); when(mockedStub.searchV2(searchRequest)).thenReturn(result); SearchResult searchResult = taskClient.searchV2("test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(task, searchResult.getResults().get(0)); } @Test public void testSearchWithParams() { TaskSummary taskSummary = mock(TaskSummary.class); TaskSummaryPb.TaskSummary taskSummaryPB = mock(TaskSummaryPb.TaskSummary.class); when(mockedProtoMapper.fromProto(taskSummaryPB)).thenReturn(taskSummary); TaskServicePb.TaskSummarySearchResult result = TaskServicePb.TaskSummarySearchResult.newBuilder() .addResults(taskSummaryPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder() .setStart(1) .setSize(5) .setSort("*") .setFreeText("*") .setQuery("test query") .build(); when(mockedStub.search(searchRequest)).thenReturn(result); SearchResult searchResult = taskClient.search(1, 5, "*", "*", "test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(taskSummary, searchResult.getResults().get(0)); } @Test public void testSearchV2WithParams() { Task task = mock(Task.class); TaskPb.Task taskPB = mock(TaskPb.Task.class); when(mockedProtoMapper.fromProto(taskPB)).thenReturn(task); TaskServicePb.TaskSearchResult result = TaskServicePb.TaskSearchResult.newBuilder() .addResults(taskPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder() .setStart(1) .setSize(5) .setSort("*") .setFreeText("*") .setQuery("test query") .build(); when(mockedStub.searchV2(searchRequest)).thenReturn(result); SearchResult searchResult = taskClient.searchV2(1, 5, "*", "*", "test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(task, searchResult.getResults().get(0)); } @Test public void testSearchWithChannelBuilder() { TaskClient taskClient = createClientWithManagedChannel(); TaskSummary taskSummary = mock(TaskSummary.class); TaskSummaryPb.TaskSummary taskSummaryPB = mock(TaskSummaryPb.TaskSummary.class); when(mockedProtoMapper.fromProto(taskSummaryPB)).thenReturn(taskSummary); TaskServicePb.TaskSummarySearchResult result = TaskServicePb.TaskSummarySearchResult.newBuilder() .addResults(taskSummaryPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder().setQuery("test query").build(); when(mockedStub.search(searchRequest)).thenReturn(result); SearchResult searchResult = taskClient.search("test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(taskSummary, searchResult.getResults().get(0)); } private TaskClient createClientWithManagedChannel() { TaskClient taskClient = new TaskClient(ManagedChannelBuilder.forAddress("test", 0)); ReflectionTestUtils.setField(taskClient, "stub", mockedStub); ReflectionTestUtils.setField(taskClient, "protoMapper", mockedProtoMapper); return taskClient; } } ================================================ FILE: grpc-client/src/test/java/com/netflix/conductor/client/grpc/WorkflowClientTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.client.grpc; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.util.ReflectionTestUtils; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.WorkflowServiceGrpc; import com.netflix.conductor.grpc.WorkflowServicePb; import com.netflix.conductor.proto.WorkflowPb; import com.netflix.conductor.proto.WorkflowSummaryPb; import io.grpc.ManagedChannelBuilder; import static junit.framework.TestCase.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(SpringRunner.class) public class WorkflowClientTest { @Mock ProtoMapper mockedProtoMapper; @Mock WorkflowServiceGrpc.WorkflowServiceBlockingStub mockedStub; WorkflowClient workflowClient; @Before public void init() { workflowClient = new WorkflowClient("test", 0); ReflectionTestUtils.setField(workflowClient, "stub", mockedStub); ReflectionTestUtils.setField(workflowClient, "protoMapper", mockedProtoMapper); } @Test public void testSearch() { WorkflowSummary workflow = mock(WorkflowSummary.class); WorkflowSummaryPb.WorkflowSummary workflowPB = mock(WorkflowSummaryPb.WorkflowSummary.class); when(mockedProtoMapper.fromProto(workflowPB)).thenReturn(workflow); WorkflowServicePb.WorkflowSummarySearchResult result = WorkflowServicePb.WorkflowSummarySearchResult.newBuilder() .addResults(workflowPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder().setQuery("test query").build(); when(mockedStub.search(searchRequest)).thenReturn(result); SearchResult searchResult = workflowClient.search("test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow, searchResult.getResults().get(0)); } @Test public void testSearchV2() { Workflow workflow = mock(Workflow.class); WorkflowPb.Workflow workflowPB = mock(WorkflowPb.Workflow.class); when(mockedProtoMapper.fromProto(workflowPB)).thenReturn(workflow); WorkflowServicePb.WorkflowSearchResult result = WorkflowServicePb.WorkflowSearchResult.newBuilder() .addResults(workflowPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder().setQuery("test query").build(); when(mockedStub.searchV2(searchRequest)).thenReturn(result); SearchResult searchResult = workflowClient.searchV2("test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow, searchResult.getResults().get(0)); } @Test public void testSearchWithParams() { WorkflowSummary workflow = mock(WorkflowSummary.class); WorkflowSummaryPb.WorkflowSummary workflowPB = mock(WorkflowSummaryPb.WorkflowSummary.class); when(mockedProtoMapper.fromProto(workflowPB)).thenReturn(workflow); WorkflowServicePb.WorkflowSummarySearchResult result = WorkflowServicePb.WorkflowSummarySearchResult.newBuilder() .addResults(workflowPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder() .setStart(1) .setSize(5) .setSort("*") .setFreeText("*") .setQuery("test query") .build(); when(mockedStub.search(searchRequest)).thenReturn(result); SearchResult searchResult = workflowClient.search(1, 5, "*", "*", "test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow, searchResult.getResults().get(0)); } @Test public void testSearchV2WithParams() { Workflow workflow = mock(Workflow.class); WorkflowPb.Workflow workflowPB = mock(WorkflowPb.Workflow.class); when(mockedProtoMapper.fromProto(workflowPB)).thenReturn(workflow); WorkflowServicePb.WorkflowSearchResult result = WorkflowServicePb.WorkflowSearchResult.newBuilder() .addResults(workflowPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder() .setStart(1) .setSize(5) .setSort("*") .setFreeText("*") .setQuery("test query") .build(); when(mockedStub.searchV2(searchRequest)).thenReturn(result); SearchResult searchResult = workflowClient.searchV2(1, 5, "*", "*", "test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow, searchResult.getResults().get(0)); } @Test public void testSearchV2WithParamsWithManagedChannel() { WorkflowClient workflowClient = createClientWithManagedChannel(); Workflow workflow = mock(Workflow.class); WorkflowPb.Workflow workflowPB = mock(WorkflowPb.Workflow.class); when(mockedProtoMapper.fromProto(workflowPB)).thenReturn(workflow); WorkflowServicePb.WorkflowSearchResult result = WorkflowServicePb.WorkflowSearchResult.newBuilder() .addResults(workflowPB) .setTotalHits(1) .build(); SearchPb.Request searchRequest = SearchPb.Request.newBuilder() .setStart(1) .setSize(5) .setSort("*") .setFreeText("*") .setQuery("test query") .build(); when(mockedStub.searchV2(searchRequest)).thenReturn(result); SearchResult searchResult = workflowClient.searchV2(1, 5, "*", "*", "test query"); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow, searchResult.getResults().get(0)); } public WorkflowClient createClientWithManagedChannel() { WorkflowClient workflowClient = new WorkflowClient(ManagedChannelBuilder.forAddress("test", 0)); ReflectionTestUtils.setField(workflowClient, "stub", mockedStub); ReflectionTestUtils.setField(workflowClient, "protoMapper", mockedProtoMapper); return workflowClient; } } ================================================ FILE: grpc-client/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker ================================================ mock-maker-inline ================================================ FILE: grpc-server/build.gradle ================================================ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') implementation project(':conductor-grpc') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "io.grpc:grpc-netty:${revGrpc}" implementation "io.grpc:grpc-services:${revGrpc}" implementation "io.grpc:grpc-protobuf:${revGrpc}" implementation "org.apache.commons:commons-lang3" testImplementation "io.grpc:grpc-testing:${revGrpc}" testImplementation "io.grpc:grpc-protobuf:${revGrpc}" testImplementation "org.testinfected.hamcrest-matchers:all-matchers:${revHamcrestAllMatchers}" } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/GRPCServer.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server; import java.io.IOException; import java.util.List; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.grpc.BindableService; import io.grpc.Server; import io.grpc.ServerBuilder; public class GRPCServer { private static final Logger LOGGER = LoggerFactory.getLogger(GRPCServer.class); private final Server server; public GRPCServer(int port, List services) { ServerBuilder builder = ServerBuilder.forPort(port); services.forEach(builder::addService); server = builder.build(); } @PostConstruct public void start() throws IOException { server.start(); LOGGER.info("grpc: Server started, listening on " + server.getPort()); } @PreDestroy public void stop() { if (server != null) { LOGGER.info("grpc: server shutting down"); server.shutdown(); } } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/GRPCServerProperties.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("conductor.grpc-server") public class GRPCServerProperties { /** The port at which the gRPC server will serve requests */ private int port = 8090; /** Enables the reflection service for Protobuf services */ private boolean reflectionEnabled = true; public int getPort() { return port; } public void setPort(int port) { this.port = port; } public boolean isReflectionEnabled() { return reflectionEnabled; } public void setReflectionEnabled(boolean reflectionEnabled) { this.reflectionEnabled = reflectionEnabled; } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/GrpcConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server; import java.util.List; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import io.grpc.BindableService; import io.grpc.protobuf.services.ProtoReflectionService; @Configuration @ConditionalOnProperty(name = "conductor.grpc-server.enabled", havingValue = "true") @EnableConfigurationProperties(GRPCServerProperties.class) public class GrpcConfiguration { @Bean public GRPCServer grpcServer( List bindableServices, // all gRPC service implementations GRPCServerProperties grpcServerProperties) { if (grpcServerProperties.isReflectionEnabled()) { bindableServices.add(ProtoReflectionService.newInstance()); } return new GRPCServer(grpcServerProperties.getPort(), bindableServices); } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/EventServiceImpl.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.grpc.EventServiceGrpc; import com.netflix.conductor.grpc.EventServicePb; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.proto.EventHandlerPb; import com.netflix.conductor.service.MetadataService; import io.grpc.stub.StreamObserver; @Service("grpcEventService") public class EventServiceImpl extends EventServiceGrpc.EventServiceImplBase { private static final Logger LOGGER = LoggerFactory.getLogger(EventServiceImpl.class); private static final ProtoMapper PROTO_MAPPER = ProtoMapper.INSTANCE; private final MetadataService metadataService; public EventServiceImpl(MetadataService metadataService) { this.metadataService = metadataService; } @Override public void addEventHandler( EventServicePb.AddEventHandlerRequest req, StreamObserver response) { metadataService.addEventHandler(PROTO_MAPPER.fromProto(req.getHandler())); response.onNext(EventServicePb.AddEventHandlerResponse.getDefaultInstance()); response.onCompleted(); } @Override public void updateEventHandler( EventServicePb.UpdateEventHandlerRequest req, StreamObserver response) { metadataService.updateEventHandler(PROTO_MAPPER.fromProto(req.getHandler())); response.onNext(EventServicePb.UpdateEventHandlerResponse.getDefaultInstance()); response.onCompleted(); } @Override public void removeEventHandler( EventServicePb.RemoveEventHandlerRequest req, StreamObserver response) { metadataService.removeEventHandlerStatus(req.getName()); response.onNext(EventServicePb.RemoveEventHandlerResponse.getDefaultInstance()); response.onCompleted(); } @Override public void getEventHandlers( EventServicePb.GetEventHandlersRequest req, StreamObserver response) { metadataService.getAllEventHandlers().stream() .map(PROTO_MAPPER::toProto) .forEach(response::onNext); response.onCompleted(); } @Override public void getEventHandlersForEvent( EventServicePb.GetEventHandlersForEventRequest req, StreamObserver response) { metadataService.getEventHandlersForEvent(req.getEvent(), req.getActiveOnly()).stream() .map(PROTO_MAPPER::toProto) .forEach(response::onNext); response.onCompleted(); } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/GRPCHelper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.Arrays; import javax.annotation.Nonnull; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import com.google.rpc.DebugInfo; import io.grpc.Metadata; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.protobuf.lite.ProtoLiteUtils; import io.grpc.stub.StreamObserver; public class GRPCHelper { private final Logger logger; private static final Metadata.Key STATUS_DETAILS_KEY = Metadata.Key.of( "grpc-status-details-bin", ProtoLiteUtils.metadataMarshaller(DebugInfo.getDefaultInstance())); public GRPCHelper(Logger log) { this.logger = log; } /** * Converts an internal exception thrown by Conductor into an StatusException that uses modern * "Status" metadata for GRPC. * *

    Note that this is trickier than it ought to be because the GRPC APIs have not been * upgraded yet. Here's a quick breakdown of how this works in practice: * *

    Reporting a "status" result back to a client with GRPC is pretty straightforward. GRPC * implementations simply serialize the status into several HTTP/2 trailer headers that are sent * back to the client before shutting down the HTTP/2 stream. * *

    - 'grpc-status', which is a string representation of a {@link com.google.rpc.Code} - * 'grpc-message', which is the description of the returned status - 'grpc-status-details-bin' * (optional), which is an arbitrary payload with a serialized ProtoBuf object, containing an * accurate description of the error in case the status is not successful. * *

    By convention, Google provides a default set of ProtoBuf messages for the most common * error cases. Here, we'll be using {@link DebugInfo}, as we're reporting an internal Java * exception which we couldn't properly handle. * *

    Now, how do we go about sending all those headers _and_ the {@link DebugInfo} payload * using the Java GRPC API? * *

    The only way we can return an error with the Java API is by passing an instance of {@link * io.grpc.StatusException} or {@link io.grpc.StatusRuntimeException} to {@link * StreamObserver#onError(Throwable)}. The easiest way to create either of these exceptions is * by using the {@link Status} class and one of its predefined code identifiers (in this case, * {@link Status#INTERNAL} because we're reporting an internal exception). The {@link Status} * class has setters to set its most relevant attributes, namely those that will be * automatically serialized into the 'grpc-status' and 'grpc-message' trailers in the response. * There is, however, no setter to pass an arbitrary ProtoBuf message to be serialized into a * `grpc-status-details-bin` trailer. This feature exists in the other language implementations * but it hasn't been brought to Java yet. * *

    Fortunately, {@link Status#asException(Metadata)} exists, allowing us to pass any amount * of arbitrary trailers before we close the response. So we're using this API to manually craft * the 'grpc-status-detail-bin' trailer, in the same way that the GRPC server implementations * for Go and C++ craft and serialize the header. This will allow us to access the metadata * cleanly from Go and C++ clients by using the 'details' method which _has_ been implemented in * those two clients. * * @param t The exception to convert * @return an instance of {@link StatusException} which will properly serialize all its headers * into the response. */ private StatusException throwableToStatusException(Throwable t) { String[] frames = ExceptionUtils.getStackFrames(t); Metadata metadata = new Metadata(); metadata.put( STATUS_DETAILS_KEY, DebugInfo.newBuilder() .addAllStackEntries(Arrays.asList(frames)) .setDetail(ExceptionUtils.getMessage(t)) .build()); return Status.INTERNAL.withDescription(t.getMessage()).withCause(t).asException(metadata); } void onError(StreamObserver response, Throwable t) { logger.error("internal exception during GRPC request", t); response.onError(throwableToStatusException(t)); } /** * Convert a non-null String instance to a possibly null String instance based on ProtoBuf's * rules for optional arguments. * *

    This helper converts an String instance from a ProtoBuf object into a possibly null * String. In ProtoBuf objects, String fields are not nullable, but an empty String field is * considered to be "missing". * *

    The internal Conductor APIs expect missing arguments to be passed as null values, so this * helper performs such conversion. * * @param str a string from a ProtoBuf object * @return the original string, or null */ String optional(@Nonnull String str) { return str.isEmpty() ? null : str; } /** * Check if a given non-null String instance is "missing" according to ProtoBuf's missing field * rules. If the String is missing, the given default value will be returned. Otherwise, the * string itself will be returned. * * @param str the input String * @param defaults the default value for the string * @return 'str' if it is not empty according to ProtoBuf rules; 'defaults' otherwise */ String optionalOr(@Nonnull String str, String defaults) { return str.isEmpty() ? defaults : str; } /** * Convert a non-null Integer instance to a possibly null Integer instance based on ProtoBuf's * rules for optional arguments. * *

    This helper converts an Integer instance from a ProtoBuf object into a possibly null * Integer. In ProtoBuf objects, Integer fields are not nullable, but a zero-value Integer field * is considered to be "missing". * *

    The internal Conductor APIs expect missing arguments to be passed as null values, so this * helper performs such conversion. * * @param i an Integer from a ProtoBuf object * @return the original Integer, or null */ Integer optional(@Nonnull Integer i) { return i == 0 ? null : i; } /** * Check if a given non-null Integer instance is "missing" according to ProtoBuf's missing field * rules. If the Integer is missing (i.e. if it has a zero-value), the given default value will * be returned. Otherwise, the Integer itself will be returned. * * @param i the input Integer * @param defaults the default value for the Integer * @return 'i' if it is not a zero-value according to ProtoBuf rules; 'defaults' otherwise */ Integer optionalOr(@Nonnull Integer i, int defaults) { return i == 0 ? defaults : i; } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/HealthServiceImpl.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import org.springframework.stereotype.Service; import io.grpc.health.v1.HealthCheckRequest; import io.grpc.health.v1.HealthCheckResponse; import io.grpc.health.v1.HealthGrpc; import io.grpc.stub.StreamObserver; @Service("grpcHealthService") public class HealthServiceImpl extends HealthGrpc.HealthImplBase { // SBMTODO: Move this Spring boot health check @Override public void check( HealthCheckRequest request, StreamObserver responseObserver) { responseObserver.onNext( HealthCheckResponse.newBuilder() .setStatus(HealthCheckResponse.ServingStatus.SERVING) .build()); responseObserver.onCompleted(); } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/MetadataServiceImpl.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.List; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.grpc.MetadataServiceGrpc; import com.netflix.conductor.grpc.MetadataServicePb; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.proto.TaskDefPb; import com.netflix.conductor.proto.WorkflowDefPb; import com.netflix.conductor.service.MetadataService; import io.grpc.Status; import io.grpc.stub.StreamObserver; @Service("grpcMetadataService") public class MetadataServiceImpl extends MetadataServiceGrpc.MetadataServiceImplBase { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataServiceImpl.class); private static final ProtoMapper PROTO_MAPPER = ProtoMapper.INSTANCE; private static final GRPCHelper GRPC_HELPER = new GRPCHelper(LOGGER); private final MetadataService service; public MetadataServiceImpl(MetadataService service) { this.service = service; } @Override public void createWorkflow( MetadataServicePb.CreateWorkflowRequest req, StreamObserver response) { WorkflowDef workflow = PROTO_MAPPER.fromProto(req.getWorkflow()); service.registerWorkflowDef(workflow); response.onNext(MetadataServicePb.CreateWorkflowResponse.getDefaultInstance()); response.onCompleted(); } @Override public void validateWorkflow( MetadataServicePb.ValidateWorkflowRequest req, StreamObserver response) { WorkflowDef workflow = PROTO_MAPPER.fromProto(req.getWorkflow()); service.validateWorkflowDef(workflow); response.onNext(MetadataServicePb.ValidateWorkflowResponse.getDefaultInstance()); response.onCompleted(); } @Override public void updateWorkflows( MetadataServicePb.UpdateWorkflowsRequest req, StreamObserver response) { List workflows = req.getDefsList().stream() .map(PROTO_MAPPER::fromProto) .collect(Collectors.toList()); service.updateWorkflowDef(workflows); response.onNext(MetadataServicePb.UpdateWorkflowsResponse.getDefaultInstance()); response.onCompleted(); } @Override public void getWorkflow( MetadataServicePb.GetWorkflowRequest req, StreamObserver response) { try { WorkflowDef workflowDef = service.getWorkflowDef(req.getName(), GRPC_HELPER.optional(req.getVersion())); WorkflowDefPb.WorkflowDef workflow = PROTO_MAPPER.toProto(workflowDef); response.onNext( MetadataServicePb.GetWorkflowResponse.newBuilder() .setWorkflow(workflow) .build()); response.onCompleted(); } catch (NotFoundException e) { // TODO replace this with gRPC exception interceptor. response.onError( Status.NOT_FOUND .withDescription("No such workflow found by name=" + req.getName()) .asRuntimeException()); } } @Override public void createTasks( MetadataServicePb.CreateTasksRequest req, StreamObserver response) { service.registerTaskDef( req.getDefsList().stream() .map(PROTO_MAPPER::fromProto) .collect(Collectors.toList())); response.onNext(MetadataServicePb.CreateTasksResponse.getDefaultInstance()); response.onCompleted(); } @Override public void updateTask( MetadataServicePb.UpdateTaskRequest req, StreamObserver response) { TaskDef task = PROTO_MAPPER.fromProto(req.getTask()); service.updateTaskDef(task); response.onNext(MetadataServicePb.UpdateTaskResponse.getDefaultInstance()); response.onCompleted(); } @Override public void getTask( MetadataServicePb.GetTaskRequest req, StreamObserver response) { TaskDef def = service.getTaskDef(req.getTaskType()); if (def != null) { TaskDefPb.TaskDef task = PROTO_MAPPER.toProto(def); response.onNext(MetadataServicePb.GetTaskResponse.newBuilder().setTask(task).build()); response.onCompleted(); } else { response.onError( Status.NOT_FOUND .withDescription( "No such TaskDef found by taskType=" + req.getTaskType()) .asRuntimeException()); } } @Override public void deleteTask( MetadataServicePb.DeleteTaskRequest req, StreamObserver response) { service.unregisterTaskDef(req.getTaskType()); response.onNext(MetadataServicePb.DeleteTaskResponse.getDefaultInstance()); response.onCompleted(); } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/TaskServiceImpl.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.TaskServiceGrpc; import com.netflix.conductor.grpc.TaskServicePb; import com.netflix.conductor.proto.TaskPb; import com.netflix.conductor.service.ExecutionService; import com.netflix.conductor.service.TaskService; import io.grpc.Status; import io.grpc.stub.StreamObserver; @Service("grpcTaskService") public class TaskServiceImpl extends TaskServiceGrpc.TaskServiceImplBase { private static final Logger LOGGER = LoggerFactory.getLogger(TaskServiceImpl.class); private static final ProtoMapper PROTO_MAPPER = ProtoMapper.INSTANCE; private static final GRPCHelper GRPC_HELPER = new GRPCHelper(LOGGER); private static final int POLL_TIMEOUT_MS = 100; private static final int MAX_POLL_TIMEOUT_MS = 5000; private final TaskService taskService; private final int maxSearchSize; private final ExecutionService executionService; public TaskServiceImpl( ExecutionService executionService, TaskService taskService, @Value("${workflow.max.search.size:5000}") int maxSearchSize) { this.executionService = executionService; this.taskService = taskService; this.maxSearchSize = maxSearchSize; } @Override public void poll( TaskServicePb.PollRequest req, StreamObserver response) { try { List tasks = executionService.poll( req.getTaskType(), req.getWorkerId(), GRPC_HELPER.optional(req.getDomain()), 1, POLL_TIMEOUT_MS); if (!tasks.isEmpty()) { TaskPb.Task t = PROTO_MAPPER.toProto(tasks.get(0)); response.onNext(TaskServicePb.PollResponse.newBuilder().setTask(t).build()); } response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void batchPoll( TaskServicePb.BatchPollRequest req, StreamObserver response) { final int count = GRPC_HELPER.optionalOr(req.getCount(), 1); final int timeout = GRPC_HELPER.optionalOr(req.getTimeout(), POLL_TIMEOUT_MS); if (timeout > MAX_POLL_TIMEOUT_MS) { response.onError( Status.INVALID_ARGUMENT .withDescription( "longpoll timeout cannot be longer than " + MAX_POLL_TIMEOUT_MS + "ms") .asRuntimeException()); return; } try { List polledTasks = taskService.batchPoll( req.getTaskType(), req.getWorkerId(), GRPC_HELPER.optional(req.getDomain()), count, timeout); LOGGER.info("polled tasks: " + polledTasks); polledTasks.stream().map(PROTO_MAPPER::toProto).forEach(response::onNext); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void updateTask( TaskServicePb.UpdateTaskRequest req, StreamObserver response) { try { TaskResult task = PROTO_MAPPER.fromProto(req.getResult()); taskService.updateTask(task); response.onNext( TaskServicePb.UpdateTaskResponse.newBuilder() .setTaskId(task.getTaskId()) .build()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void addLog( TaskServicePb.AddLogRequest req, StreamObserver response) { taskService.log(req.getTaskId(), req.getLog()); response.onNext(TaskServicePb.AddLogResponse.getDefaultInstance()); response.onCompleted(); } @Override public void getTaskLogs( TaskServicePb.GetTaskLogsRequest req, StreamObserver response) { List logs = taskService.getTaskLogs(req.getTaskId()); response.onNext( TaskServicePb.GetTaskLogsResponse.newBuilder() .addAllLogs(logs.stream().map(PROTO_MAPPER::toProto)::iterator) .build()); response.onCompleted(); } @Override public void getTask( TaskServicePb.GetTaskRequest req, StreamObserver response) { try { Task task = taskService.getTask(req.getTaskId()); if (task == null) { response.onError( Status.NOT_FOUND .withDescription("No such task found by id=" + req.getTaskId()) .asRuntimeException()); } else { response.onNext( TaskServicePb.GetTaskResponse.newBuilder() .setTask(PROTO_MAPPER.toProto(task)) .build()); response.onCompleted(); } } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void getQueueSizesForTasks( TaskServicePb.QueueSizesRequest req, StreamObserver response) { Map sizes = taskService.getTaskQueueSizes(req.getTaskTypesList()); response.onNext( TaskServicePb.QueueSizesResponse.newBuilder().putAllQueueForTask(sizes).build()); response.onCompleted(); } @Override public void getQueueInfo( TaskServicePb.QueueInfoRequest req, StreamObserver response) { Map queueInfo = taskService.getAllQueueDetails(); response.onNext( TaskServicePb.QueueInfoResponse.newBuilder().putAllQueues(queueInfo).build()); response.onCompleted(); } @Override public void getQueueAllInfo( TaskServicePb.QueueAllInfoRequest req, StreamObserver response) { Map>> info = taskService.allVerbose(); TaskServicePb.QueueAllInfoResponse.Builder queuesBuilder = TaskServicePb.QueueAllInfoResponse.newBuilder(); for (Map.Entry>> queue : info.entrySet()) { final String queueName = queue.getKey(); final Map> queueShards = queue.getValue(); TaskServicePb.QueueAllInfoResponse.QueueInfo.Builder queueInfoBuilder = TaskServicePb.QueueAllInfoResponse.QueueInfo.newBuilder(); for (Map.Entry> shard : queueShards.entrySet()) { final String shardName = shard.getKey(); final Map shardInfo = shard.getValue(); // FIXME: make shardInfo an actual type // shardInfo is an immutable map with predefined keys, so we can always // access 'size' and 'uacked'. It would be better if shardInfo // were actually a POJO. queueInfoBuilder.putShards( shardName, TaskServicePb.QueueAllInfoResponse.ShardInfo.newBuilder() .setSize(shardInfo.get("size")) .setUacked(shardInfo.get("uacked")) .build()); } queuesBuilder.putQueues(queueName, queueInfoBuilder.build()); } response.onNext(queuesBuilder.build()); response.onCompleted(); } @Override public void search( SearchPb.Request req, StreamObserver response) { final int start = req.getStart(); final int size = GRPC_HELPER.optionalOr(req.getSize(), maxSearchSize); final String sort = req.getSort(); final String freeText = GRPC_HELPER.optionalOr(req.getFreeText(), "*"); final String query = req.getQuery(); if (size > maxSearchSize) { response.onError( Status.INVALID_ARGUMENT .withDescription( "Cannot return more than " + maxSearchSize + " results") .asRuntimeException()); return; } SearchResult searchResult = taskService.search(start, size, sort, freeText, query); response.onNext( TaskServicePb.TaskSummarySearchResult.newBuilder() .setTotalHits(searchResult.getTotalHits()) .addAllResults( searchResult.getResults().stream().map(PROTO_MAPPER::toProto) ::iterator) .build()); response.onCompleted(); } @Override public void searchV2( SearchPb.Request req, StreamObserver response) { final int start = req.getStart(); final int size = GRPC_HELPER.optionalOr(req.getSize(), maxSearchSize); final String sort = req.getSort(); final String freeText = GRPC_HELPER.optionalOr(req.getFreeText(), "*"); final String query = req.getQuery(); if (size > maxSearchSize) { response.onError( Status.INVALID_ARGUMENT .withDescription( "Cannot return more than " + maxSearchSize + " results") .asRuntimeException()); return; } SearchResult searchResult = taskService.searchV2(start, size, sort, freeText, query); response.onNext( TaskServicePb.TaskSearchResult.newBuilder() .setTotalHits(searchResult.getTotalHits()) .addAllResults( searchResult.getResults().stream().map(PROTO_MAPPER::toProto) ::iterator) .build()); response.onCompleted(); } } ================================================ FILE: grpc-server/src/main/java/com/netflix/conductor/grpc/server/service/WorkflowServiceImpl.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.grpc.ProtoMapper; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.WorkflowServiceGrpc; import com.netflix.conductor.grpc.WorkflowServicePb; import com.netflix.conductor.proto.RerunWorkflowRequestPb; import com.netflix.conductor.proto.StartWorkflowRequestPb; import com.netflix.conductor.proto.WorkflowPb; import com.netflix.conductor.service.WorkflowService; import io.grpc.Status; import io.grpc.stub.StreamObserver; @Service("grpcWorkflowService") public class WorkflowServiceImpl extends WorkflowServiceGrpc.WorkflowServiceImplBase { private static final Logger LOGGER = LoggerFactory.getLogger(TaskServiceImpl.class); private static final ProtoMapper PROTO_MAPPER = ProtoMapper.INSTANCE; private static final GRPCHelper GRPC_HELPER = new GRPCHelper(LOGGER); private final WorkflowService workflowService; private final int maxSearchSize; public WorkflowServiceImpl( WorkflowService workflowService, @Value("${workflow.max.search.size:5000}") int maxSearchSize) { this.workflowService = workflowService; this.maxSearchSize = maxSearchSize; } @Override public void startWorkflow( StartWorkflowRequestPb.StartWorkflowRequest pbRequest, StreamObserver response) { // TODO: better handling of optional 'version' final StartWorkflowRequest request = PROTO_MAPPER.fromProto(pbRequest); try { String id = workflowService.startWorkflow( pbRequest.getName(), GRPC_HELPER.optional(request.getVersion()), request.getCorrelationId(), request.getPriority(), request.getInput(), request.getExternalInputPayloadStoragePath(), request.getTaskToDomain(), request.getWorkflowDef()); response.onNext( WorkflowServicePb.StartWorkflowResponse.newBuilder().setWorkflowId(id).build()); response.onCompleted(); } catch (NotFoundException nfe) { response.onError( Status.NOT_FOUND .withDescription("No such workflow found by name=" + request.getName()) .asRuntimeException()); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void getWorkflows( WorkflowServicePb.GetWorkflowsRequest req, StreamObserver response) { final String name = req.getName(); final boolean includeClosed = req.getIncludeClosed(); final boolean includeTasks = req.getIncludeTasks(); WorkflowServicePb.GetWorkflowsResponse.Builder builder = WorkflowServicePb.GetWorkflowsResponse.newBuilder(); for (String correlationId : req.getCorrelationIdList()) { List workflows = workflowService.getWorkflows(name, correlationId, includeClosed, includeTasks); builder.putWorkflowsById( correlationId, WorkflowServicePb.GetWorkflowsResponse.Workflows.newBuilder() .addAllWorkflows( workflows.stream().map(PROTO_MAPPER::toProto)::iterator) .build()); } response.onNext(builder.build()); response.onCompleted(); } @Override public void getWorkflowStatus( WorkflowServicePb.GetWorkflowStatusRequest req, StreamObserver response) { try { Workflow workflow = workflowService.getExecutionStatus(req.getWorkflowId(), req.getIncludeTasks()); response.onNext(PROTO_MAPPER.toProto(workflow)); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void removeWorkflow( WorkflowServicePb.RemoveWorkflowRequest req, StreamObserver response) { try { workflowService.deleteWorkflow(req.getWorkflodId(), req.getArchiveWorkflow()); response.onNext(WorkflowServicePb.RemoveWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void getRunningWorkflows( WorkflowServicePb.GetRunningWorkflowsRequest req, StreamObserver response) { try { List workflowIds = workflowService.getRunningWorkflows( req.getName(), req.getVersion(), req.getStartTime(), req.getEndTime()); response.onNext( WorkflowServicePb.GetRunningWorkflowsResponse.newBuilder() .addAllWorkflowIds(workflowIds) .build()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void decideWorkflow( WorkflowServicePb.DecideWorkflowRequest req, StreamObserver response) { try { workflowService.decideWorkflow(req.getWorkflowId()); response.onNext(WorkflowServicePb.DecideWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void pauseWorkflow( WorkflowServicePb.PauseWorkflowRequest req, StreamObserver response) { try { workflowService.pauseWorkflow(req.getWorkflowId()); response.onNext(WorkflowServicePb.PauseWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void resumeWorkflow( WorkflowServicePb.ResumeWorkflowRequest req, StreamObserver response) { try { workflowService.resumeWorkflow(req.getWorkflowId()); response.onNext(WorkflowServicePb.ResumeWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void skipTaskFromWorkflow( WorkflowServicePb.SkipTaskRequest req, StreamObserver response) { try { SkipTaskRequest skipTask = PROTO_MAPPER.fromProto(req.getRequest()); workflowService.skipTaskFromWorkflow( req.getWorkflowId(), req.getTaskReferenceName(), skipTask); response.onNext(WorkflowServicePb.SkipTaskResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void rerunWorkflow( RerunWorkflowRequestPb.RerunWorkflowRequest req, StreamObserver response) { try { String id = workflowService.rerunWorkflow( req.getReRunFromWorkflowId(), PROTO_MAPPER.fromProto(req)); response.onNext( WorkflowServicePb.RerunWorkflowResponse.newBuilder().setWorkflowId(id).build()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void restartWorkflow( WorkflowServicePb.RestartWorkflowRequest req, StreamObserver response) { try { workflowService.restartWorkflow(req.getWorkflowId(), req.getUseLatestDefinitions()); response.onNext(WorkflowServicePb.RestartWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void retryWorkflow( WorkflowServicePb.RetryWorkflowRequest req, StreamObserver response) { try { workflowService.retryWorkflow(req.getWorkflowId(), req.getResumeSubworkflowTasks()); response.onNext(WorkflowServicePb.RetryWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void resetWorkflowCallbacks( WorkflowServicePb.ResetWorkflowCallbacksRequest req, StreamObserver response) { try { workflowService.resetWorkflow(req.getWorkflowId()); response.onNext(WorkflowServicePb.ResetWorkflowCallbacksResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } @Override public void terminateWorkflow( WorkflowServicePb.TerminateWorkflowRequest req, StreamObserver response) { try { workflowService.terminateWorkflow(req.getWorkflowId(), req.getReason()); response.onNext(WorkflowServicePb.TerminateWorkflowResponse.getDefaultInstance()); response.onCompleted(); } catch (Exception e) { GRPC_HELPER.onError(response, e); } } private void doSearch( boolean searchByTask, SearchPb.Request req, StreamObserver response) { final int start = req.getStart(); final int size = GRPC_HELPER.optionalOr(req.getSize(), maxSearchSize); final List sort = convertSort(req.getSort()); final String freeText = GRPC_HELPER.optionalOr(req.getFreeText(), "*"); final String query = req.getQuery(); if (size > maxSearchSize) { response.onError( Status.INVALID_ARGUMENT .withDescription( "Cannot return more than " + maxSearchSize + " results") .asRuntimeException()); return; } SearchResult search; if (searchByTask) { search = workflowService.searchWorkflowsByTasks(start, size, sort, freeText, query); } else { search = workflowService.searchWorkflows(start, size, sort, freeText, query); } response.onNext( WorkflowServicePb.WorkflowSummarySearchResult.newBuilder() .setTotalHits(search.getTotalHits()) .addAllResults( search.getResults().stream().map(PROTO_MAPPER::toProto)::iterator) .build()); response.onCompleted(); } private void doSearchV2( boolean searchByTask, SearchPb.Request req, StreamObserver response) { final int start = req.getStart(); final int size = GRPC_HELPER.optionalOr(req.getSize(), maxSearchSize); final List sort = convertSort(req.getSort()); final String freeText = GRPC_HELPER.optionalOr(req.getFreeText(), "*"); final String query = req.getQuery(); if (size > maxSearchSize) { response.onError( Status.INVALID_ARGUMENT .withDescription( "Cannot return more than " + maxSearchSize + " results") .asRuntimeException()); return; } SearchResult search; if (searchByTask) { search = workflowService.searchWorkflowsByTasksV2(start, size, sort, freeText, query); } else { search = workflowService.searchWorkflowsV2(start, size, sort, freeText, query); } response.onNext( WorkflowServicePb.WorkflowSearchResult.newBuilder() .setTotalHits(search.getTotalHits()) .addAllResults( search.getResults().stream().map(PROTO_MAPPER::toProto)::iterator) .build()); response.onCompleted(); } private List convertSort(String sortStr) { List list = new ArrayList<>(); if (sortStr != null && sortStr.length() != 0) { list = Arrays.asList(sortStr.split("\\|")); } return list; } @Override public void search( SearchPb.Request request, StreamObserver responseObserver) { doSearch(false, request, responseObserver); } @Override public void searchByTasks( SearchPb.Request request, StreamObserver responseObserver) { doSearch(true, request, responseObserver); } @Override public void searchV2( SearchPb.Request request, StreamObserver responseObserver) { doSearchV2(false, request, responseObserver); } @Override public void searchByTasksV2( SearchPb.Request request, StreamObserver responseObserver) { doSearchV2(true, request, responseObserver); } } ================================================ FILE: grpc-server/src/test/java/com/netflix/conductor/grpc/server/service/HealthServiceImplTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; public class HealthServiceImplTest { // SBMTODO: Move this Spring boot health check // @Rule // public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); // // @Rule // public ExpectedException thrown = ExpectedException.none(); // // @Test // public void healthServing() throws Exception { // // Generate a unique in-process server name. // String serverName = InProcessServerBuilder.generateName(); // HealthCheckAggregator hca = mock(HealthCheckAggregator.class); // CompletableFuture hcsf = mock(CompletableFuture.class); // HealthCheckStatus hcs = mock(HealthCheckStatus.class); // when(hcs.isHealthy()).thenReturn(true); // when(hcsf.get()).thenReturn(hcs); // when(hca.check()).thenReturn(hcsf); // HealthServiceImpl healthyService = new HealthServiceImpl(hca); // // addService(serverName, healthyService); // HealthGrpc.HealthBlockingStub blockingStub = HealthGrpc.newBlockingStub( // // Create a client channel and register for automatic graceful shutdown. // // grpcCleanup.register(InProcessChannelBuilder.forName(serverName).directExecutor().build())); // // // HealthCheckResponse reply = // blockingStub.check(HealthCheckRequest.newBuilder().build()); // // assertEquals(HealthCheckResponse.ServingStatus.SERVING, reply.getStatus()); // } // // @Test // public void healthNotServing() throws Exception { // // Generate a unique in-process server name. // String serverName = InProcessServerBuilder.generateName(); // HealthCheckAggregator hca = mock(HealthCheckAggregator.class); // CompletableFuture hcsf = mock(CompletableFuture.class); // HealthCheckStatus hcs = mock(HealthCheckStatus.class); // when(hcs.isHealthy()).thenReturn(false); // when(hcsf.get()).thenReturn(hcs); // when(hca.check()).thenReturn(hcsf); // HealthServiceImpl healthyService = new HealthServiceImpl(hca); // // addService(serverName, healthyService); // HealthGrpc.HealthBlockingStub blockingStub = HealthGrpc.newBlockingStub( // // Create a client channel and register for automatic graceful shutdown. // // grpcCleanup.register(InProcessChannelBuilder.forName(serverName).directExecutor().build())); // // // HealthCheckResponse reply = // blockingStub.check(HealthCheckRequest.newBuilder().build()); // // assertEquals(HealthCheckResponse.ServingStatus.NOT_SERVING, reply.getStatus()); // } // // @Test // public void healthException() throws Exception { // // Generate a unique in-process server name. // String serverName = InProcessServerBuilder.generateName(); // HealthCheckAggregator hca = mock(HealthCheckAggregator.class); // CompletableFuture hcsf = mock(CompletableFuture.class); // when(hcsf.get()).thenThrow(InterruptedException.class); // when(hca.check()).thenReturn(hcsf); // HealthServiceImpl healthyService = new HealthServiceImpl(hca); // // addService(serverName, healthyService); // HealthGrpc.HealthBlockingStub blockingStub = HealthGrpc.newBlockingStub( // // Create a client channel and register for automatic graceful shutdown. // // grpcCleanup.register(InProcessChannelBuilder.forName(serverName).directExecutor().build())); // // thrown.expect(StatusRuntimeException.class); // thrown.expect(hasProperty("status", is(Status.INTERNAL))); // blockingStub.check(HealthCheckRequest.newBuilder().build()); // // } // // private void addService(String name, BindableService service) throws Exception { // // Create a server, add service, start, and register for automatic graceful shutdown. // grpcCleanup.register(InProcessServerBuilder // .forName(name).directExecutor().addService(service).build().start()); // } } ================================================ FILE: grpc-server/src/test/java/com/netflix/conductor/grpc/server/service/TaskServiceImplTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.TaskServicePb; import com.netflix.conductor.proto.TaskPb; import com.netflix.conductor.proto.TaskSummaryPb; import com.netflix.conductor.service.ExecutionService; import com.netflix.conductor.service.TaskService; import io.grpc.stub.StreamObserver; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; public class TaskServiceImplTest { @Mock private TaskService taskService; @Mock private ExecutionService executionService; private TaskServiceImpl taskServiceImpl; @Before public void init() { initMocks(this); taskServiceImpl = new TaskServiceImpl(executionService, taskService, 5000); } @Test public void searchExceptionTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference throwable = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(50000) .setSort("strings") .setQuery("") .setFreeText("*") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(TaskServicePb.TaskSummarySearchResult value) {} @Override public void onError(Throwable t) { throwable.set(t); streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; taskServiceImpl.search(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); assertEquals( "INVALID_ARGUMENT: Cannot return more than 5000 results", throwable.get().getMessage()); } @Test public void searchV2ExceptionTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference throwable = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(50000) .setSort("strings") .setQuery("") .setFreeText("*") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(TaskServicePb.TaskSearchResult value) {} @Override public void onError(Throwable t) { throwable.set(t); streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; taskServiceImpl.searchV2(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); assertEquals( "INVALID_ARGUMENT: Cannot return more than 5000 results", throwable.get().getMessage()); } @Test public void searchTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("*") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(TaskServicePb.TaskSummarySearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; TaskSummary taskSummary = new TaskSummary(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(taskSummary)); when(taskService.search(1, 1, "strings", "*", "")).thenReturn(searchResult); taskServiceImpl.search(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); TaskServicePb.TaskSummarySearchResult taskSummarySearchResult = result.get(); assertEquals(1, taskSummarySearchResult.getTotalHits()); assertEquals( TaskSummaryPb.TaskSummary.newBuilder().build(), taskSummarySearchResult.getResultsList().get(0)); } @Test public void searchV2Test() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("*") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(TaskServicePb.TaskSearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; Task task = new Task(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(task)); when(taskService.searchV2(1, 1, "strings", "*", "")).thenReturn(searchResult); taskServiceImpl.searchV2(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); TaskServicePb.TaskSearchResult taskSearchResult = result.get(); assertEquals(1, taskSearchResult.getTotalHits()); assertEquals( TaskPb.Task.newBuilder().setCallbackFromWorker(true).build(), taskSearchResult.getResultsList().get(0)); } } ================================================ FILE: grpc-server/src/test/java/com/netflix/conductor/grpc/server/service/WorkflowServiceImplTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.grpc.server.service; import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.grpc.SearchPb; import com.netflix.conductor.grpc.WorkflowServicePb; import com.netflix.conductor.proto.WorkflowPb; import com.netflix.conductor.proto.WorkflowSummaryPb; import com.netflix.conductor.service.WorkflowService; import io.grpc.stub.StreamObserver; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; public class WorkflowServiceImplTest { private static final String WORKFLOW_ID = "anyWorkflowId"; private static final Boolean RESUME_SUBWORKFLOW_TASKS = true; @Mock private WorkflowService workflowService; private WorkflowServiceImpl workflowServiceImpl; @Before public void init() { initMocks(this); workflowServiceImpl = new WorkflowServiceImpl(workflowService, 5000); } @SuppressWarnings("unchecked") @Test public void givenWorkflowIdWhenRetryWorkflowThenRetriedSuccessfully() { // Given WorkflowServicePb.RetryWorkflowRequest req = WorkflowServicePb.RetryWorkflowRequest.newBuilder() .setWorkflowId(WORKFLOW_ID) .setResumeSubworkflowTasks(true) .build(); // When workflowServiceImpl.retryWorkflow(req, mock(StreamObserver.class)); // Then verify(workflowService).retryWorkflow(WORKFLOW_ID, RESUME_SUBWORKFLOW_TASKS); } @Test public void searchExceptionTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference throwable = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(50000) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSummarySearchResult value) {} @Override public void onError(Throwable t) { throwable.set(t); streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; workflowServiceImpl.search(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); assertEquals( "INVALID_ARGUMENT: Cannot return more than 5000 results", throwable.get().getMessage()); } @Test public void searchV2ExceptionTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference throwable = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(50000) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSearchResult value) {} @Override public void onError(Throwable t) { throwable.set(t); streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; workflowServiceImpl.searchV2(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); assertEquals( "INVALID_ARGUMENT: Cannot return more than 5000 results", throwable.get().getMessage()); } @Test public void searchTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSummarySearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; WorkflowSummary workflow = new WorkflowSummary(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(workflow)); when(workflowService.searchWorkflows( anyInt(), anyInt(), anyList(), anyString(), anyString())) .thenReturn(searchResult); workflowServiceImpl.search(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); WorkflowServicePb.WorkflowSummarySearchResult workflowSearchResult = result.get(); assertEquals(1, workflowSearchResult.getTotalHits()); assertEquals( WorkflowSummaryPb.WorkflowSummary.newBuilder().build(), workflowSearchResult.getResultsList().get(0)); } @Test public void searchByTasksTest() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSummarySearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; WorkflowSummary workflow = new WorkflowSummary(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(workflow)); when(workflowService.searchWorkflowsByTasks( anyInt(), anyInt(), anyList(), anyString(), anyString())) .thenReturn(searchResult); workflowServiceImpl.searchByTasks(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); WorkflowServicePb.WorkflowSummarySearchResult workflowSearchResult = result.get(); assertEquals(1, workflowSearchResult.getTotalHits()); assertEquals( WorkflowSummaryPb.WorkflowSummary.newBuilder().build(), workflowSearchResult.getResultsList().get(0)); } @Test public void searchV2Test() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; Workflow workflow = new Workflow(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(workflow)); when(workflowService.searchWorkflowsV2(1, 1, Collections.singletonList("strings"), "*", "")) .thenReturn(searchResult); workflowServiceImpl.searchV2(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); WorkflowServicePb.WorkflowSearchResult workflowSearchResult = result.get(); assertEquals(1, workflowSearchResult.getTotalHits()); assertEquals( WorkflowPb.Workflow.newBuilder().build(), workflowSearchResult.getResultsList().get(0)); } @Test public void searchByTasksV2Test() throws InterruptedException { CountDownLatch streamAlive = new CountDownLatch(1); AtomicReference result = new AtomicReference<>(); SearchPb.Request req = SearchPb.Request.newBuilder() .setStart(1) .setSize(1) .setSort("strings") .setQuery("") .setFreeText("") .build(); StreamObserver streamObserver = new StreamObserver<>() { @Override public void onNext(WorkflowServicePb.WorkflowSearchResult value) { result.set(value); } @Override public void onError(Throwable t) { streamAlive.countDown(); } @Override public void onCompleted() { streamAlive.countDown(); } }; Workflow workflow = new Workflow(); SearchResult searchResult = new SearchResult<>(); searchResult.setTotalHits(1); searchResult.setResults(Collections.singletonList(workflow)); when(workflowService.searchWorkflowsByTasksV2( 1, 1, Collections.singletonList("strings"), "*", "")) .thenReturn(searchResult); workflowServiceImpl.searchByTasksV2(req, streamObserver); streamAlive.await(10, TimeUnit.MILLISECONDS); WorkflowServicePb.WorkflowSearchResult workflowSearchResult = result.get(); assertEquals(1, workflowSearchResult.getTotalHits()); assertEquals( WorkflowPb.Workflow.newBuilder().build(), workflowSearchResult.getResultsList().get(0)); } } ================================================ FILE: grpc-server/src/test/resources/log4j.properties ================================================ # # Copyright 2019 Netflix, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set root logger level to WARN and its only appender to A1. log4j.rootLogger=WARN, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n ================================================ FILE: http-task/build.gradle ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.boot:spring-boot-starter-web' implementation "javax.ws.rs:jsr311-api:${revJsr311Api}" testImplementation 'org.springframework.boot:spring-boot-starter-web' testImplementation "org.testcontainers:mockserver:${revTestContainer}" testImplementation "org.mock-server:mockserver-client-java:${revMockServerClient}" testImplementation "org.bouncycastle:bcprov-jdk15on:1.70" testImplementation "org.bouncycastle:bcpkix-jdk15on:1.70" } ================================================ FILE: http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.http; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.tasks.http.providers.RestTemplateProvider; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HTTP; /** Task that enables calling another HTTP endpoint as part of its execution */ @Component(TASK_TYPE_HTTP) public class HttpTask extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(HttpTask.class); public static final String REQUEST_PARAMETER_NAME = "http_request"; static final String MISSING_REQUEST = "Missing HTTP request. Task input MUST have a '" + REQUEST_PARAMETER_NAME + "' key with HttpTask.Input as value. See documentation for HttpTask for required input parameters"; private final TypeReference> mapOfObj = new TypeReference>() {}; private final TypeReference> listOfObj = new TypeReference>() {}; protected ObjectMapper objectMapper; protected RestTemplateProvider restTemplateProvider; private final String requestParameter; @Autowired public HttpTask(RestTemplateProvider restTemplateProvider, ObjectMapper objectMapper) { this(TASK_TYPE_HTTP, restTemplateProvider, objectMapper); } public HttpTask( String name, RestTemplateProvider restTemplateProvider, ObjectMapper objectMapper) { super(name); this.restTemplateProvider = restTemplateProvider; this.objectMapper = objectMapper; this.requestParameter = REQUEST_PARAMETER_NAME; LOGGER.info("{} initialized...", getTaskType()); } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { Object request = task.getInputData().get(requestParameter); task.setWorkerId(Utils.getServerId()); if (request == null) { task.setReasonForIncompletion(MISSING_REQUEST); task.setStatus(TaskModel.Status.FAILED); return; } Input input = objectMapper.convertValue(request, Input.class); if (input.getUri() == null) { String reason = "Missing HTTP URI. See documentation for HttpTask for required input parameters"; task.setReasonForIncompletion(reason); task.setStatus(TaskModel.Status.FAILED); return; } if (input.getMethod() == null) { String reason = "No HTTP method specified"; task.setReasonForIncompletion(reason); task.setStatus(TaskModel.Status.FAILED); return; } try { HttpResponse response = httpCall(input); LOGGER.debug( "Response: {}, {}, task:{}", response.statusCode, response.body, task.getTaskId()); if (response.statusCode > 199 && response.statusCode < 300) { if (isAsyncComplete(task)) { task.setStatus(TaskModel.Status.IN_PROGRESS); } else { task.setStatus(TaskModel.Status.COMPLETED); } } else { if (response.body != null) { task.setReasonForIncompletion(response.body.toString()); } else { task.setReasonForIncompletion("No response from the remote service"); } task.setStatus(TaskModel.Status.FAILED); } //noinspection ConstantConditions if (response != null) { task.addOutput("response", response.asMap()); } } catch (Exception e) { LOGGER.error( "Failed to invoke {} task: {} - uri: {}, vipAddress: {} in workflow: {}", getTaskType(), task.getTaskId(), input.getUri(), input.getVipAddress(), task.getWorkflowInstanceId(), e); task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion( "Failed to invoke " + getTaskType() + " task due to: " + e); task.addOutput("response", e.toString()); } } /** * @param input HTTP Request * @return Response of the http call * @throws Exception If there was an error making http call Note: protected access is so that * tasks extended from this task can re-use this to make http calls */ protected HttpResponse httpCall(Input input) throws Exception { RestTemplate restTemplate = restTemplateProvider.getRestTemplate(input); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.valueOf(input.getContentType())); headers.setAccept(Collections.singletonList(MediaType.valueOf(input.getAccept()))); input.headers.forEach( (key, value) -> { if (value != null) { headers.add(key, value.toString()); } }); HttpEntity request = new HttpEntity<>(input.getBody(), headers); HttpResponse response = new HttpResponse(); try { ResponseEntity responseEntity = restTemplate.exchange(input.getUri(), input.getMethod(), request, String.class); if (responseEntity.getStatusCode().is2xxSuccessful() && responseEntity.hasBody()) { response.body = extractBody(responseEntity.getBody()); } response.statusCode = responseEntity.getStatusCodeValue(); response.reasonPhrase = responseEntity.getStatusCode().getReasonPhrase(); response.headers = responseEntity.getHeaders(); return response; } catch (RestClientException ex) { LOGGER.error( String.format( "Got unexpected http response - uri: %s, vipAddress: %s", input.getUri(), input.getVipAddress()), ex); String reason = ex.getLocalizedMessage(); LOGGER.error(reason, ex); throw new Exception(reason); } } private Object extractBody(String responseBody) { try { JsonNode node = objectMapper.readTree(responseBody); if (node.isArray()) { return objectMapper.convertValue(node, listOfObj); } else if (node.isObject()) { return objectMapper.convertValue(node, mapOfObj); } else if (node.isNumber()) { return objectMapper.convertValue(node, Double.class); } else { return node.asText(); } } catch (IOException jpe) { LOGGER.error("Error extracting response body", jpe); return responseBody; } } @Override public boolean execute(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { return false; } @Override public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { task.setStatus(TaskModel.Status.CANCELED); } @Override public boolean isAsync() { return true; } public static class HttpResponse { public Object body; public MultiValueMap headers; public int statusCode; public String reasonPhrase; @Override public String toString() { return "HttpResponse [body=" + body + ", headers=" + headers + ", statusCode=" + statusCode + ", reasonPhrase=" + reasonPhrase + "]"; } public Map asMap() { Map map = new HashMap<>(); map.put("body", body); map.put("headers", headers); map.put("statusCode", statusCode); map.put("reasonPhrase", reasonPhrase); return map; } } public static class Input { private HttpMethod method; // PUT, POST, GET, DELETE, OPTIONS, HEAD private String vipAddress; private String appName; private Map headers = new HashMap<>(); private String uri; private Object body; private String accept = MediaType.APPLICATION_JSON_VALUE; private String contentType = MediaType.APPLICATION_JSON_VALUE; private Integer connectionTimeOut; private Integer readTimeOut; /** * @return the method */ public HttpMethod getMethod() { return method; } /** * @param method the method to set */ public void setMethod(String method) { this.method = HttpMethod.valueOf(method); } /** * @return the headers */ public Map getHeaders() { return headers; } /** * @param headers the headers to set */ public void setHeaders(Map headers) { this.headers = headers; } /** * @return the body */ public Object getBody() { return body; } /** * @param body the body to set */ public void setBody(Object body) { this.body = body; } /** * @return the uri */ public String getUri() { return uri; } /** * @param uri the uri to set */ public void setUri(String uri) { this.uri = uri; } /** * @return the vipAddress */ public String getVipAddress() { return vipAddress; } /** * @param vipAddress the vipAddress to set */ public void setVipAddress(String vipAddress) { this.vipAddress = vipAddress; } /** * @return the accept */ public String getAccept() { return accept; } /** * @param accept the accept to set */ public void setAccept(String accept) { this.accept = accept; } /** * @return the MIME content type to use for the request */ public String getContentType() { return contentType; } /** * @param contentType the MIME content type to set */ public void setContentType(String contentType) { this.contentType = contentType; } public String getAppName() { return appName; } public void setAppName(String appName) { this.appName = appName; } /** * @return the connectionTimeOut */ public Integer getConnectionTimeOut() { return connectionTimeOut; } /** * @return the readTimeOut */ public Integer getReadTimeOut() { return readTimeOut; } public void setConnectionTimeOut(Integer connectionTimeOut) { this.connectionTimeOut = connectionTimeOut; } public void setReadTimeOut(Integer readTimeOut) { this.readTimeOut = readTimeOut; } } } ================================================ FILE: http-task/src/main/java/com/netflix/conductor/tasks/http/providers/DefaultRestTemplateProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.http.providers; import java.time.Duration; import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; import com.netflix.conductor.tasks.http.HttpTask; /** * Provider for a customized RestTemplateBuilder. This class provides a default {@link * RestTemplateBuilder} which can be configured or extended as needed. */ @Component public class DefaultRestTemplateProvider implements RestTemplateProvider { private final ThreadLocal threadLocalRestTemplate; private final int defaultReadTimeout; private final int defaultConnectTimeout; @Autowired public DefaultRestTemplateProvider( @Value("${conductor.tasks.http.readTimeout:150ms}") Duration readTimeout, @Value("${conductor.tasks.http.connectTimeout:100ms}") Duration connectTimeout) { this.threadLocalRestTemplate = ThreadLocal.withInitial(RestTemplate::new); this.defaultReadTimeout = (int) readTimeout.toMillis(); this.defaultConnectTimeout = (int) connectTimeout.toMillis(); } @Override public @NonNull RestTemplate getRestTemplate(@NonNull HttpTask.Input input) { RestTemplate restTemplate = threadLocalRestTemplate.get(); HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(); requestFactory.setConnectTimeout( Optional.ofNullable(input.getConnectionTimeOut()).orElse(defaultConnectTimeout)); requestFactory.setReadTimeout( Optional.ofNullable(input.getReadTimeOut()).orElse(defaultReadTimeout)); restTemplate.setRequestFactory(requestFactory); return restTemplate; } } ================================================ FILE: http-task/src/main/java/com/netflix/conductor/tasks/http/providers/RestTemplateProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.http.providers; import org.springframework.lang.NonNull; import org.springframework.web.client.RestTemplate; import com.netflix.conductor.tasks.http.HttpTask; @FunctionalInterface public interface RestTemplateProvider { RestTemplate getRestTemplate(@NonNull HttpTask.Input input); } ================================================ FILE: http-task/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.tasks.http.readTimeout", "type": "java.lang.Integer", "description": "The read timeout of the underlying HttpClient used by the HTTP task." }, { "name": "conductor.tasks.http.connectTimeout", "type": "java.lang.Integer", "description": "The connection timeout of the underlying HttpClient used by the HTTP task." } ] } ================================================ FILE: http-task/src/test/java/com/netflix/conductor/tasks/http/HttpTaskTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.http; import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.mockserver.client.MockServerClient; import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; import org.testcontainers.containers.MockServerContainer; import org.testcontainers.utility.DockerImageName; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.core.execution.DeciderService; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.utils.ExternalPayloadStorageUtils; import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.tasks.http.providers.DefaultRestTemplateProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.*; import static org.mockito.Mockito.mock; @SuppressWarnings("unchecked") public class HttpTaskTest { private static final String ERROR_RESPONSE = "Something went wrong!"; private static final String TEXT_RESPONSE = "Text Response"; private static final double NUM_RESPONSE = 42.42d; private HttpTask httpTask; private WorkflowExecutor workflowExecutor; private final WorkflowModel workflow = new WorkflowModel(); private static final ObjectMapper objectMapper = new ObjectMapper(); private static String JSON_RESPONSE; @ClassRule public static MockServerContainer mockServer = new MockServerContainer( DockerImageName.parse("mockserver/mockserver").withTag("mockserver-5.12.0")); @BeforeClass public static void init() throws Exception { Map map = new HashMap<>(); map.put("key", "value1"); map.put("num", 42); map.put("SomeKey", null); JSON_RESPONSE = objectMapper.writeValueAsString(map); final TypeReference> mapOfObj = new TypeReference<>() {}; MockServerClient client = new MockServerClient(mockServer.getHost(), mockServer.getServerPort()); client.when(HttpRequest.request().withPath("/post").withMethod("POST")) .respond( request -> { Map reqBody = objectMapper.readValue(request.getBody().toString(), mapOfObj); Set keys = reqBody.keySet(); Map respBody = new HashMap<>(); keys.forEach(k -> respBody.put(k, k)); return HttpResponse.response() .withContentType(MediaType.APPLICATION_JSON) .withBody(objectMapper.writeValueAsString(respBody)); }); client.when(HttpRequest.request().withPath("/post2").withMethod("POST")) .respond(HttpResponse.response().withStatusCode(204)); client.when(HttpRequest.request().withPath("/failure").withMethod("GET")) .respond( HttpResponse.response() .withStatusCode(500) .withContentType(MediaType.TEXT_PLAIN) .withBody(ERROR_RESPONSE)); client.when(HttpRequest.request().withPath("/text").withMethod("GET")) .respond(HttpResponse.response().withBody(TEXT_RESPONSE)); client.when(HttpRequest.request().withPath("/numeric").withMethod("GET")) .respond(HttpResponse.response().withBody(String.valueOf(NUM_RESPONSE))); client.when(HttpRequest.request().withPath("/json").withMethod("GET")) .respond( HttpResponse.response() .withContentType(MediaType.APPLICATION_JSON) .withBody(JSON_RESPONSE)); } @Before public void setup() { workflowExecutor = mock(WorkflowExecutor.class); DefaultRestTemplateProvider defaultRestTemplateProvider = new DefaultRestTemplateProvider(Duration.ofMillis(150), Duration.ofMillis(100)); httpTask = new HttpTask(defaultRestTemplateProvider, objectMapper); } @Test public void testPost() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/post"); Map body = new HashMap<>(); body.put("input_key1", "value1"); body.put("input_key2", 45.3d); body.put("someKey", null); input.setBody(body); input.setMethod("POST"); input.setReadTimeOut(1000); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); assertEquals(task.getReasonForIncompletion(), TaskModel.Status.COMPLETED, task.getStatus()); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertTrue("response is: " + response, response instanceof Map); Map map = (Map) response; Set inputKeys = body.keySet(); Set responseKeys = map.keySet(); inputKeys.containsAll(responseKeys); responseKeys.containsAll(inputKeys); } @Test public void testPostNoContent() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri( "http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/post2"); Map body = new HashMap<>(); body.put("input_key1", "value1"); body.put("input_key2", 45.3d); input.setBody(body); input.setMethod("POST"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); assertEquals(task.getReasonForIncompletion(), TaskModel.Status.COMPLETED, task.getStatus()); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertNull("response is: " + response, response); } @Test public void testFailure() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri( "http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/failure"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); assertEquals( "Task output: " + task.getOutputData(), TaskModel.Status.FAILED, task.getStatus()); assertTrue(task.getReasonForIncompletion().contains(ERROR_RESPONSE)); task.setStatus(TaskModel.Status.SCHEDULED); task.getInputData().remove(HttpTask.REQUEST_PARAMETER_NAME); httpTask.start(workflow, task, workflowExecutor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); assertEquals(HttpTask.MISSING_REQUEST, task.getReasonForIncompletion()); } @Test public void testPostAsyncComplete() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/post"); Map body = new HashMap<>(); body.put("input_key1", "value1"); body.put("input_key2", 45.3d); input.setBody(body); input.setMethod("POST"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); task.getInputData().put("asyncComplete", true); httpTask.start(workflow, task, workflowExecutor); assertEquals( task.getReasonForIncompletion(), TaskModel.Status.IN_PROGRESS, task.getStatus()); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.IN_PROGRESS, task.getStatus()); assertTrue("response is: " + response, response instanceof Map); Map map = (Map) response; Set inputKeys = body.keySet(); Set responseKeys = map.keySet(); inputKeys.containsAll(responseKeys); responseKeys.containsAll(inputKeys); } @Test public void testTextGET() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/text"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(TEXT_RESPONSE, response); } @Test public void testNumberGET() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri( "http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/numeric"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertEquals(NUM_RESPONSE, response); assertTrue(response instanceof Number); } @Test public void testJsonGET() throws JsonProcessingException { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/json"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); Map hr = (Map) task.getOutputData().get("response"); Object response = hr.get("body"); assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); assertTrue(response instanceof Map); Map map = (Map) response; assertEquals(JSON_RESPONSE, objectMapper.writeValueAsString(map)); } @Test public void testExecute() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/json"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); task.setStatus(TaskModel.Status.SCHEDULED); task.setScheduledTime(0); boolean executed = httpTask.execute(workflow, task, workflowExecutor); assertFalse(executed); } @Test public void testHTTPGetConnectionTimeOut() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); Instant start = Instant.now(); input.setConnectionTimeOut(110); input.setMethod("GET"); input.setUri("http://10.255.14.15"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); task.setStatus(TaskModel.Status.SCHEDULED); task.setScheduledTime(0); httpTask.start(workflow, task, workflowExecutor); Instant end = Instant.now(); long diff = end.toEpochMilli() - start.toEpochMilli(); assertEquals(task.getStatus(), TaskModel.Status.FAILED); assertTrue(diff >= 110L); } @Test public void testHTTPGETReadTimeOut() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setReadTimeOut(-1); input.setMethod("GET"); input.setUri("http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/json"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); task.setStatus(TaskModel.Status.SCHEDULED); task.setScheduledTime(0); httpTask.start(workflow, task, workflowExecutor); assertEquals(task.getStatus(), TaskModel.Status.FAILED); } @Test public void testOptional() { TaskModel task = new TaskModel(); HttpTask.Input input = new HttpTask.Input(); input.setUri( "http://" + mockServer.getHost() + ":" + mockServer.getServerPort() + "/failure"); input.setMethod("GET"); task.getInputData().put(HttpTask.REQUEST_PARAMETER_NAME, input); httpTask.start(workflow, task, workflowExecutor); assertEquals( "Task output: " + task.getOutputData(), TaskModel.Status.FAILED, task.getStatus()); assertTrue(task.getReasonForIncompletion().contains(ERROR_RESPONSE)); assertFalse(task.getStatus().isSuccessful()); task.setStatus(TaskModel.Status.SCHEDULED); task.getInputData().remove(HttpTask.REQUEST_PARAMETER_NAME); task.setReferenceTaskName("t1"); httpTask.start(workflow, task, workflowExecutor); assertEquals(TaskModel.Status.FAILED, task.getStatus()); assertEquals(HttpTask.MISSING_REQUEST, task.getReasonForIncompletion()); assertFalse(task.getStatus().isSuccessful()); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setOptional(true); workflowTask.setName("HTTP"); workflowTask.setWorkflowTaskType(TaskType.USER_DEFINED); workflowTask.setTaskReferenceName("t1"); WorkflowDef def = new WorkflowDef(); def.getTasks().add(workflowTask); WorkflowModel workflow = new WorkflowModel(); workflow.setWorkflowDefinition(def); workflow.getTasks().add(task); MetadataDAO metadataDAO = mock(MetadataDAO.class); ExternalPayloadStorageUtils externalPayloadStorageUtils = mock(ExternalPayloadStorageUtils.class); ParametersUtils parametersUtils = mock(ParametersUtils.class); SystemTaskRegistry systemTaskRegistry = mock(SystemTaskRegistry.class); new DeciderService( new IDGenerator(), parametersUtils, metadataDAO, externalPayloadStorageUtils, systemTaskRegistry, Collections.emptyMap(), Duration.ofMinutes(60)) .decide(workflow); } } ================================================ FILE: http-task/src/test/java/com/netflix/conductor/tasks/http/providers/DefaultRestTemplateProviderTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.http.providers; import java.time.Duration; import org.junit.Test; import org.springframework.web.client.RestTemplate; import com.netflix.conductor.tasks.http.HttpTask; import static org.junit.Assert.*; public class DefaultRestTemplateProviderTest { @Test public void differentObjectsForDifferentThreads() throws InterruptedException { DefaultRestTemplateProvider defaultRestTemplateProvider = new DefaultRestTemplateProvider(Duration.ofMillis(150), Duration.ofMillis(100)); final RestTemplate restTemplate = defaultRestTemplateProvider.getRestTemplate(new HttpTask.Input()); final StringBuilder result = new StringBuilder(); Thread t1 = new Thread( () -> { RestTemplate restTemplate1 = defaultRestTemplateProvider.getRestTemplate( new HttpTask.Input()); if (restTemplate1 != restTemplate) { result.append("different"); } }); t1.start(); t1.join(); assertEquals(result.toString(), "different"); } @Test public void sameObjectForSameThread() { DefaultRestTemplateProvider defaultRestTemplateProvider = new DefaultRestTemplateProvider(Duration.ofMillis(150), Duration.ofMillis(100)); RestTemplate client1 = defaultRestTemplateProvider.getRestTemplate(new HttpTask.Input()); RestTemplate client2 = defaultRestTemplateProvider.getRestTemplate(new HttpTask.Input()); assertSame(client1, client2); assertNotNull(client1); } } ================================================ FILE: java-sdk/README.md ================================================ # SDK for Conductor Conductor SDK allows developers to create, test and execute workflows using code. There are three main features of the SDK: 1. [Create and run workflows using code](workflow_sdk.md) 2. [Create and run strongly typed workers](worker_sdk.md) 3. [Unit Testing framework for workflows and workers](testing_framework.md) ================================================ FILE: java-sdk/build.gradle ================================================ apply plugin: 'groovy' dependencies { implementation project(':conductor-common') implementation project(':conductor-client') implementation "com.fasterxml.jackson.core:jackson-databind:${revFasterXml}" implementation "com.google.guava:guava:${revGuava}" implementation "cglib:cglib:3.3.0" implementation "com.sun.jersey:jersey-client:${revJersey}" implementation "javax.ws.rs:javax.ws.rs-api:${revJAXRS}" implementation "org.glassfish.jersey.core:jersey-common:${revJerseyCommon}" implementation "org.openjdk.nashorn:nashorn-core:15.4" testImplementation "org.springframework:spring-web" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" testImplementation "com.fasterxml.jackson.core:jackson-core:${revFasterXml}" testImplementation "org.apache.commons:commons-lang3" testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" } test { testLogging { exceptionFormat = 'full' } } sourceSets.main.java.srcDirs += ['example/java', 'example/resources'] ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/Order.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; import java.math.BigDecimal; public class Order { public enum ShippingMethod { GROUND, NEXT_DAY_AIR, SAME_DAY } private String orderNumber; private String sku; private int quantity; private BigDecimal unitPrice; private String zipCode; private String countryCode; private ShippingMethod shippingMethod; public Order(String orderNumber, String sku, int quantity, BigDecimal unitPrice) { this.orderNumber = orderNumber; this.sku = sku; this.quantity = quantity; this.unitPrice = unitPrice; } public Order() {} public String getOrderNumber() { return orderNumber; } public void setOrderNumber(String orderNumber) { this.orderNumber = orderNumber; } public String getSku() { return sku; } public void setSku(String sku) { this.sku = sku; } public int getQuantity() { return quantity; } public void setQuantity(int quantity) { this.quantity = quantity; } public BigDecimal getUnitPrice() { return unitPrice; } public void setUnitPrice(BigDecimal unitPrice) { this.unitPrice = unitPrice; } public String getZipCode() { return zipCode; } public void setZipCode(String zipCode) { this.zipCode = zipCode; } public String getCountryCode() { return countryCode; } public void setCountryCode(String countryCode) { this.countryCode = countryCode; } public ShippingMethod getShippingMethod() { return shippingMethod; } public void setShippingMethod(ShippingMethod shippingMethod) { this.shippingMethod = shippingMethod; } } ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/Shipment.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; public class Shipment { private String userId; private String orderNo; public Shipment(String userId, String orderNo) { this.userId = userId; this.orderNo = orderNo; } public Shipment() {} public String getUserId() { return userId; } public void setUserId(String userId) { this.userId = userId; } public String getOrderNo() { return orderNo; } public void setOrderNo(String orderNo) { this.orderNo = orderNo; } } ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/ShipmentState.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; public class ShipmentState { private boolean paymentCompleted; private boolean emailSent; private boolean shipped; private String trackingNumber; public boolean isPaymentCompleted() { return paymentCompleted; } public void setPaymentCompleted(boolean paymentCompleted) { this.paymentCompleted = paymentCompleted; } public boolean isEmailSent() { return emailSent; } public void setEmailSent(boolean emailSent) { this.emailSent = emailSent; } public boolean isShipped() { return shipped; } public void setShipped(boolean shipped) { this.shipped = shipped; } public String getTrackingNumber() { return trackingNumber; } public void setTrackingNumber(String trackingNumber) { this.trackingNumber = trackingNumber; } } ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/ShipmentWorkers.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; import java.math.BigDecimal; import java.util.*; import com.netflix.conductor.sdk.workflow.def.tasks.DynamicForkInput; import com.netflix.conductor.sdk.workflow.def.tasks.SubWorkflow; import com.netflix.conductor.sdk.workflow.def.tasks.Task; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.task.WorkerTask; public class ShipmentWorkers { @WorkerTask(value = "generateDynamicFork", threadCount = 3) public DynamicForkInput generateDynamicFork( @InputParam("orderDetails") List orderDetails, @InputParam("userDetails") User userDetails) { DynamicForkInput input = new DynamicForkInput(); List> tasks = new ArrayList<>(); Map inputs = new HashMap<>(); for (int i = 0; i < orderDetails.size(); i++) { Order detail = orderDetails.get(i); String referenceName = "order_flow_sub_" + i; tasks.add( new SubWorkflow(referenceName, "order_flow", null) .input("orderDetail", detail) .input("userDetails", userDetails)); inputs.put(referenceName, new HashMap<>()); } input.setInputs(inputs); input.setTasks(tasks); return input; } @WorkerTask(value = "get_order_details", threadCount = 5) public List getOrderDetails(@InputParam("orderNo") String orderNo) { int lineItemCount = new Random().nextInt(10); List orderDetails = new ArrayList<>(); for (int i = 0; i < lineItemCount; i++) { Order orderDetail = new Order(orderNo, "sku_" + i, 2, BigDecimal.valueOf(20.5)); orderDetail.setOrderNumber(UUID.randomUUID().toString()); orderDetail.setCountryCode(i % 2 == 0 ? "US" : "CA"); if (i % 3 == 0) { orderDetail.setCountryCode("UK"); } if (orderDetail.getCountryCode().equals("US")) orderDetail.setShippingMethod(Order.ShippingMethod.SAME_DAY); else if (orderDetail.getCountryCode().equals("CA")) orderDetail.setShippingMethod(Order.ShippingMethod.NEXT_DAY_AIR); else orderDetail.setShippingMethod(Order.ShippingMethod.GROUND); orderDetails.add(orderDetail); } return orderDetails; } @WorkerTask("get_user_details") public User getUserDetails(@InputParam("userId") String userId) { User user = new User( "User Name", userId + "@example.com", "1234 forline street", "mountain view", "95030", "US", "Paypal", "biling_001"); return user; } @WorkerTask("calculate_tax_and_total") public @OutputParam("total_amount") BigDecimal calculateTax( @InputParam("orderDetail") Order orderDetails) { BigDecimal preTaxAmount = orderDetails.getUnitPrice().multiply(new BigDecimal(orderDetails.getQuantity())); BigDecimal tax = BigDecimal.valueOf(0.2).multiply(preTaxAmount); if (!"US".equals(orderDetails.getCountryCode())) { tax = BigDecimal.ZERO; } return preTaxAmount.add(tax); } @WorkerTask("ground_shipping_label") public @OutputParam("reference_number") String prepareGroundShipping( @InputParam("name") String name, @InputParam("address") String address, @InputParam("orderNo") String orderNo) { return "Ground_" + orderNo; } @WorkerTask("air_shipping_label") public @OutputParam("reference_number") String prepareAirShipping( @InputParam("name") String name, @InputParam("address") String address, @InputParam("orderNo") String orderNo) { return "Air_" + orderNo; } @WorkerTask("same_day_shipping_label") public @OutputParam("reference_number") String prepareSameDayShipping( @InputParam("name") String name, @InputParam("address") String address, @InputParam("orderNo") String orderNo) { return "SameDay_" + orderNo; } @WorkerTask("charge_payment") public @OutputParam("reference") String chargePayment( @InputParam("amount") BigDecimal amount, @InputParam("billingId") String billingId, @InputParam("billingType") String billingType) { return UUID.randomUUID().toString(); } @WorkerTask("send_email") public void sendEmail( @InputParam("name") String name, @InputParam("email") String email, @InputParam("orderNo") String orderNo) {} } ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/ShipmentWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; import com.netflix.conductor.sdk.workflow.def.WorkflowBuilder; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; public class ShipmentWorkflow { private final WorkflowExecutor executor; public ShipmentWorkflow(WorkflowExecutor executor) { this.executor = executor; this.executor.initWorkers(ShipmentWorkflow.class.getPackageName()); } public ConductorWorkflow createOrderFlow() { WorkflowBuilder builder = new WorkflowBuilder<>(executor); builder.name("order_flow") .version(1) .ownerEmail("user@example.com") .timeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF, 60) // 1 day max .description("Workflow to track shipment") .add( new SimpleTask("calculate_tax_and_total", "calculate_tax_and_total") .input("orderDetail", ConductorWorkflow.input.get("orderDetail"))) .add( new SimpleTask("charge_payment", "charge_payment") .input( "billingId", ConductorWorkflow.input .map("userDetails") .get("billingId"), "billingType", ConductorWorkflow.input .map("userDetails") .get("billingType"), "amount", "${calculate_tax_and_total.output.total_amount}")) .add( new Switch("shipping_label", "${workflow.input.orderDetail.shippingMethod}") .switchCase( Order.ShippingMethod.GROUND.toString(), new SimpleTask( "ground_shipping_label", "ground_shipping_label") .input( "name", ConductorWorkflow.input .map("userDetails") .get("name"), "address", ConductorWorkflow.input .map("userDetails") .get("addressLine"), "orderNo", ConductorWorkflow.input .map("orderDetail") .get("orderNumber"))) .switchCase( Order.ShippingMethod.NEXT_DAY_AIR.toString(), new SimpleTask("air_shipping_label", "air_shipping_label") .input( "name", ConductorWorkflow.input .map("userDetails") .get("name"), "address", ConductorWorkflow.input .map("userDetails") .get("addressLine"), "orderNo", ConductorWorkflow.input .map("orderDetail") .get("orderNumber"))) .switchCase( Order.ShippingMethod.SAME_DAY.toString(), new SimpleTask( "same_day_shipping_label", "same_day_shipping_label") .input( "name", ConductorWorkflow.input .map("userDetails") .get("name"), "address", ConductorWorkflow.input .map("userDetails") .get("addressLine"), "orderNo", ConductorWorkflow.input .map("orderDetail") .get("orderNumber"))) .defaultCase( new Terminate( "unsupported_shipping_type", Workflow.WorkflowStatus.FAILED, "Unsupported Shipping Method"))) .add( new SimpleTask("send_email", "send_email") .input( "name", ConductorWorkflow.input .map("userDetails") .get("name"), "email", ConductorWorkflow.input .map("userDetails") .get("email"), "orderNo", ConductorWorkflow.input .map("orderDetail") .get("orderNumber"))); ConductorWorkflow conductorWorkflow = builder.build(); conductorWorkflow.registerWorkflow(true, true); return conductorWorkflow; } public ConductorWorkflow createShipmentWorkflow() { WorkflowBuilder builder = new WorkflowBuilder<>(executor); SimpleTask getOrderDetails = new SimpleTask("get_order_details", "get_order_details") .input("orderNo", ConductorWorkflow.input.get("orderNo")); SimpleTask getUserDetails = new SimpleTask("get_user_details", "get_user_details") .input("userId", ConductorWorkflow.input.get("userId")); ConductorWorkflow conductorWorkflow = builder.name("shipment_workflow") .version(1) .ownerEmail("user@example.com") .variables(new ShipmentState()) .timeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF, 60) // 30 days .description("Workflow to track shipment") .add( new ForkJoin( "get_in_parallel", new Task[] {getOrderDetails}, new Task[] {getUserDetails})) // For all the line items in the order, run in parallel: // (calculate tax, charge payment, set state, prepare shipment, send // shipment, set state) .add( new DynamicFork( "process_order", new SimpleTask("generateDynamicFork", "generateDynamicFork") .input( "orderDetails", getOrderDetails.taskOutput.get("result")) .input("userDetails", getUserDetails.taskOutput))) // Update the workflow state with shipped = true .add(new SetVariable("update_state").input("shipped", true)) .build(); conductorWorkflow.registerWorkflow(true, true); return conductorWorkflow; } public static void main(String[] args) { String conductorServerURL = "http://localhost:8080/api/"; // Change this to your Conductor server WorkflowExecutor executor = new WorkflowExecutor(conductorServerURL); // Create the new shipment workflow ShipmentWorkflow shipmentWorkflow = new ShipmentWorkflow(executor); // Create two workflows // 1. Order flow that ships an individual order // 2. Shipment Workflow that tracks multiple orders in a shipment shipmentWorkflow.createOrderFlow(); ConductorWorkflow workflow = shipmentWorkflow.createShipmentWorkflow(); // Execute the workflow and wait for it to complete try { Shipment workflowInput = new Shipment("userA", "order123"); // Execute returns a completable future. CompletableFuture executionFuture = workflow.execute(workflowInput); // Wait for a maximum of a minute for the workflow to complete. Workflow run = executionFuture.get(1, TimeUnit.MINUTES); System.out.println("Workflow Id: " + run); System.out.println("Workflow Status: " + run.getStatus()); System.out.println("Workflow Output: " + run.getOutput()); } catch (Exception e) { e.printStackTrace(); } finally { System.exit(0); } System.out.println("Done"); } } ================================================ FILE: java-sdk/example/java/com/netflix/conductor/sdk/example/shipment/User.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.example.shipment; public class User { private String name; private String email; private String addressLine; private String city; private String zipCode; private String countryCode; private String billingType; private String billingId; public User( String name, String email, String addressLine, String city, String zipCode, String countryCode, String billingType, String billingId) { this.name = name; this.email = email; this.addressLine = addressLine; this.city = city; this.zipCode = zipCode; this.countryCode = countryCode; this.billingType = billingType; this.billingId = billingId; } public User() {} public String getName() { return name; } public void setName(String name) { this.name = name; } public String getEmail() { return email; } public void setEmail(String email) { this.email = email; } public String getAddressLine() { return addressLine; } public void setAddressLine(String addressLine) { this.addressLine = addressLine; } public String getCity() { return city; } public void setCity(String city) { this.city = city; } public String getZipCode() { return zipCode; } public void setZipCode(String zipCode) { this.zipCode = zipCode; } public String getCountryCode() { return countryCode; } public void setCountryCode(String countryCode) { this.countryCode = countryCode; } public String getBillingType() { return billingType; } public void setBillingType(String billingType) { this.billingType = billingType; } public String getBillingId() { return billingId; } public void setBillingId(String billingId) { this.billingId = billingId; } } ================================================ FILE: java-sdk/example/resources/script.js ================================================ function e() { if ($.value > 1){ return { "key": "value", "key2": 42 }; } else { return {}; } } e(); ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/healthcheck/HealthCheckClient.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.healthcheck; import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.URL; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; public class HealthCheckClient { private final String healthCheckURL; private final ObjectMapper objectMapper; public HealthCheckClient(String healthCheckURL) { this.healthCheckURL = healthCheckURL; this.objectMapper = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } public boolean isServerRunning() { try { BufferedReader in = new BufferedReader(new InputStreamReader(new URL(healthCheckURL).openStream())); StringBuilder response = new StringBuilder(); String inputLine; while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); HealthCheckResults healthCheckResults = objectMapper.readValue(response.toString(), HealthCheckResults.class); return healthCheckResults.healthy; } catch (Throwable t) { return false; } } private static final class HealthCheckResults { private boolean healthy; public boolean isHealthy() { return healthy; } public void setHealthy(boolean healthy) { this.healthy = healthy; } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/testing/LocalServerRunner.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.testing; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.sdk.healthcheck.HealthCheckClient; import com.google.common.util.concurrent.Uninterruptibles; public class LocalServerRunner { private static final Logger LOGGER = LoggerFactory.getLogger(LocalServerRunner.class); private final HealthCheckClient healthCheck; private Process serverProcess; private final ScheduledExecutorService healthCheckExecutor = Executors.newSingleThreadScheduledExecutor(); private final CountDownLatch serverProcessLatch = new CountDownLatch(1); private final int port; private final String conductorVersion; private final String serverURL; private static Map serverInstances = new HashMap<>(); public LocalServerRunner(int port, String conductorVersion) { this.port = port; this.conductorVersion = conductorVersion; this.serverURL = "http://localhost:" + port + "/"; healthCheck = new HealthCheckClient(serverURL + "health"); } public String getServerAPIUrl() { return this.serverURL + "api/"; } /** * Starts the local server. Downloads the latest conductor build from the maven repo If you want * to start the server from a specific download location, set `repositoryURL` system property * with the link to the actual downloadable server boot jar file. * *

    System Properties that can be set conductorVersion: when specified, uses this * version of conductor to run tests (and downloads from maven repo) repositoryURL: full url * where the server boot jar can be downloaded from. This can be a public repo or internal * repository, allowing full control over the location and version of the conductor server */ public void startLocalServer() { synchronized (serverInstances) { if (serverInstances.get(port) != null) { throw new IllegalStateException( "Another server has already been started at port " + port); } serverInstances.put(port, this); } try { String downloadURL = "https://repo1.maven.org/maven2/com/netflix/conductor/conductor-server/" + conductorVersion + "/conductor-server-" + conductorVersion + "-boot.jar"; String repositoryURL = Optional.ofNullable(System.getProperty("repositoryURL")).orElse(downloadURL); LOGGER.info( "Running conductor with version {} from repo url {}", conductorVersion, repositoryURL); Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown)); installAndStartServer(repositoryURL, port); healthCheckExecutor.scheduleAtFixedRate( () -> { try { if (serverProcessLatch.getCount() > 0) { boolean isRunning = healthCheck.isServerRunning(); if (isRunning) { serverProcessLatch.countDown(); } } } catch (Exception e) { LOGGER.warn( "Caught an exception while polling for server running status {}", e.getMessage()); } }, 100, 100, TimeUnit.MILLISECONDS); Uninterruptibles.awaitUninterruptibly(serverProcessLatch, 1, TimeUnit.MINUTES); if (serverProcessLatch.getCount() > 0) { throw new RuntimeException("Server not healthy"); } healthCheckExecutor.shutdownNow(); } catch (IOException e) { throw new Error(e); } } public void shutdown() { if (serverProcess != null) { serverProcess.destroyForcibly(); serverInstances.remove(port); } } private synchronized void installAndStartServer(String repositoryURL, int localServerPort) throws IOException { if (serverProcess != null) { return; } String configFile = LocalServerRunner.class.getResource("/test-server.properties").getFile(); String tempDir = System.getProperty("java.io.tmpdir"); Path serverFile = Paths.get(tempDir, "conductor-server.jar"); if (!Files.exists(serverFile)) { Files.copy(new URL(repositoryURL).openStream(), serverFile); } String command = "java -Dserver.port=" + localServerPort + " -DCONDUCTOR_CONFIG_FILE=" + configFile + " -jar " + serverFile; LOGGER.info("Running command {}", command); serverProcess = Runtime.getRuntime().exec(command); BufferedReader error = new BufferedReader(new InputStreamReader(serverProcess.getErrorStream())); BufferedReader op = new BufferedReader(new InputStreamReader(serverProcess.getInputStream())); // This captures the stream and copies to a visible log for tracking errors asynchronously // using a separate thread Executors.newSingleThreadScheduledExecutor() .execute( () -> { String line = null; while (true) { try { if ((line = error.readLine()) == null) break; } catch (IOException e) { LOGGER.error("Exception reading input stream:", e); } // copy to standard error LOGGER.error("Server error stream - {}", line); } }); // This captures the stream and copies to a visible log for tracking errors asynchronously // using a separate thread Executors.newSingleThreadScheduledExecutor() .execute( () -> { String line = null; while (true) { try { if ((line = op.readLine()) == null) break; } catch (IOException e) { LOGGER.error("Exception reading input stream:", e); } // copy to standard out LOGGER.trace("Server input stream - {}", line); } }); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/testing/WorkflowTestRunner.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.testing; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; public class WorkflowTestRunner { private LocalServerRunner localServerRunner; private final AnnotatedWorkerExecutor annotatedWorkerExecutor; private final WorkflowExecutor workflowExecutor; public WorkflowTestRunner(String serverApiUrl) { TaskClient taskClient = new TaskClient(); taskClient.setRootURI(serverApiUrl); this.annotatedWorkerExecutor = new AnnotatedWorkerExecutor(taskClient); this.workflowExecutor = new WorkflowExecutor(serverApiUrl); } public WorkflowTestRunner(int port, String conductorVersion) { localServerRunner = new LocalServerRunner(port, conductorVersion); TaskClient taskClient = new TaskClient(); taskClient.setRootURI(localServerRunner.getServerAPIUrl()); this.annotatedWorkerExecutor = new AnnotatedWorkerExecutor(taskClient); this.workflowExecutor = new WorkflowExecutor(localServerRunner.getServerAPIUrl()); } public WorkflowExecutor getWorkflowExecutor() { return workflowExecutor; } public void init(String basePackages) { if (localServerRunner != null) { localServerRunner.startLocalServer(); } annotatedWorkerExecutor.initWorkers(basePackages); } public void shutdown() { localServerRunner.shutdown(); annotatedWorkerExecutor.shutdown(); workflowExecutor.shutdown(); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/ConductorWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; import java.util.*; import java.util.concurrent.CompletableFuture; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.workflow.def.tasks.Task; import com.netflix.conductor.sdk.workflow.def.tasks.TaskRegistry; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import com.netflix.conductor.sdk.workflow.utils.InputOutputGetter; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; /** * @param Type of the workflow input */ public class ConductorWorkflow { public static final InputOutputGetter input = new InputOutputGetter("workflow", InputOutputGetter.Field.input); public static final InputOutputGetter output = new InputOutputGetter("workflow", InputOutputGetter.Field.output); private String name; private String description; private int version; private String failureWorkflow; private String ownerEmail; private WorkflowDef.TimeoutPolicy timeoutPolicy; private Map workflowOutput; private long timeoutSeconds; private boolean restartable = true; private T defaultInput; private Map variables; private List tasks = new ArrayList<>(); private final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); private final WorkflowExecutor workflowExecutor; public ConductorWorkflow(WorkflowExecutor workflowExecutor) { this.workflowOutput = new HashMap<>(); this.workflowExecutor = workflowExecutor; this.restartable = true; } public void setName(String name) { this.name = name; } public void setVersion(int version) { this.version = version; } public void setDescription(String description) { this.description = description; } public void setFailureWorkflow(String failureWorkflow) { this.failureWorkflow = failureWorkflow; } public void add(Task task) { this.tasks.add(task); } public String getName() { return name; } public String getDescription() { return description; } public int getVersion() { return version; } public String getFailureWorkflow() { return failureWorkflow; } public String getOwnerEmail() { return ownerEmail; } public void setOwnerEmail(String ownerEmail) { this.ownerEmail = ownerEmail; } public WorkflowDef.TimeoutPolicy getTimeoutPolicy() { return timeoutPolicy; } public void setTimeoutPolicy(WorkflowDef.TimeoutPolicy timeoutPolicy) { this.timeoutPolicy = timeoutPolicy; } public long getTimeoutSeconds() { return timeoutSeconds; } public void setTimeoutSeconds(long timeoutSeconds) { this.timeoutSeconds = timeoutSeconds; } public boolean isRestartable() { return restartable; } public void setRestartable(boolean restartable) { this.restartable = restartable; } public T getDefaultInput() { return defaultInput; } public void setDefaultInput(T defaultInput) { this.defaultInput = defaultInput; } public Map getWorkflowOutput() { return workflowOutput; } public void setWorkflowOutput(Map workflowOutput) { this.workflowOutput = workflowOutput; } public Object getVariables() { return variables; } public void setVariables(Map variables) { this.variables = variables; } /** * Execute a dynamic workflow without creating a definition in metadata store. * *


    * Note: Use this with caution - as this does not promote re-usability of the workflows * * @param input Workflow Input - The input object is converted a JSON doc as an input to the * workflow * @return */ public CompletableFuture executeDynamic(T input) { return workflowExecutor.executeWorkflow(this, input); } /** * Executes the workflow using registered metadata definitions * * @see #registerWorkflow() * @param input * @return */ public CompletableFuture execute(T input) { return workflowExecutor.executeWorkflow(this.getName(), this.getVersion(), input); } /** * Registers a new workflow in the server. * * @return true if the workflow is successfully registered. False if the workflow cannot be * registered and the workflow definition already exists on the server with given name + * version The call will throw a runtime exception if any of the tasks are missing * definitions on the server. */ public boolean registerWorkflow() { return registerWorkflow(false, false); } /** * @param overwrite set to true if the workflow should be overwritten if the definition already * exists with the given name and version. Use with caution * @return true if success, false otherwise. */ public boolean registerWorkflow(boolean overwrite) { return registerWorkflow(overwrite, false); } /** * @param overwrite set to true if the workflow should be overwritten if the definition already * exists with the given name and version. Use with caution * @param registerTasks if set to true, missing task definitions are registered with the default * configuration. * @return true if success, false otherwise. */ public boolean registerWorkflow(boolean overwrite, boolean registerTasks) { WorkflowDef workflowDef = toWorkflowDef(); List missing = getMissingTasks(workflowDef); if (!missing.isEmpty()) { if (!registerTasks) { throw new RuntimeException( "Workflow cannot be registered. The following tasks do not have definitions. " + "Please register these tasks before creating the workflow. Missing Tasks = " + missing); } else { String ownerEmail = this.ownerEmail; missing.stream().forEach(taskName -> registerTaskDef(taskName, ownerEmail)); } } return workflowExecutor.registerWorkflow(workflowDef, overwrite); } /** * @return Convert to the WorkflowDef model used by the Metadata APIs */ public WorkflowDef toWorkflowDef() { WorkflowDef def = new WorkflowDef(); def.setName(name); def.setDescription(description); def.setVersion(version); def.setFailureWorkflow(failureWorkflow); def.setOwnerEmail(ownerEmail); def.setTimeoutPolicy(timeoutPolicy); def.setTimeoutSeconds(timeoutSeconds); def.setRestartable(restartable); def.setOutputParameters(workflowOutput); def.setVariables(variables); def.setInputTemplate(objectMapper.convertValue(defaultInput, Map.class)); for (Task task : tasks) { def.getTasks().addAll(task.getWorkflowDefTasks()); } return def; } /** * Generate ConductorWorkflow based on the workflow metadata definition * * @param def * @return */ public static ConductorWorkflow fromWorkflowDef(WorkflowDef def) { ConductorWorkflow workflow = new ConductorWorkflow<>(null); fromWorkflowDef(workflow, def); return workflow; } public ConductorWorkflow from(String workflowName, Integer workflowVersion) { WorkflowDef def = workflowExecutor.getMetadataClient().getWorkflowDef(workflowName, workflowVersion); fromWorkflowDef(this, def); return this; } private static void fromWorkflowDef(ConductorWorkflow workflow, WorkflowDef def) { workflow.setName(def.getName()); workflow.setVersion(def.getVersion()); workflow.setFailureWorkflow(def.getFailureWorkflow()); workflow.setRestartable(def.isRestartable()); workflow.setVariables(def.getVariables()); workflow.setDefaultInput((T) def.getInputTemplate()); workflow.setWorkflowOutput(def.getOutputParameters()); workflow.setOwnerEmail(def.getOwnerEmail()); workflow.setDescription(def.getDescription()); workflow.setTimeoutSeconds(def.getTimeoutSeconds()); workflow.setTimeoutPolicy(def.getTimeoutPolicy()); List workflowTasks = def.getTasks(); for (WorkflowTask workflowTask : workflowTasks) { Task task = TaskRegistry.getTask(workflowTask); workflow.tasks.add(task); } } private List getMissingTasks(WorkflowDef workflowDef) { List missing = new ArrayList<>(); workflowDef.collectTasks().stream() .filter(workflowTask -> workflowTask.getType().equals(TaskType.TASK_TYPE_SIMPLE)) .map(WorkflowTask::getName) .distinct() .parallel() .forEach( taskName -> { try { TaskDef taskDef = workflowExecutor.getMetadataClient().getTaskDef(taskName); } catch (ConductorClientException cce) { if (cce.getStatus() == 404) { missing.add(taskName); } else { throw cce; } } }); return missing; } private void registerTaskDef(String taskName, String ownerEmail) { TaskDef taskDef = new TaskDef(); taskDef.setName(taskName); taskDef.setOwnerEmail(ownerEmail); workflowExecutor.getMetadataClient().registerTaskDefs(Arrays.asList(taskDef)); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ConductorWorkflow workflow = (ConductorWorkflow) o; return version == workflow.version && Objects.equals(name, workflow.name); } @Override public int hashCode() { return Objects.hash(name, version); } @Override public String toString() { try { return objectMapper.writeValueAsString(toWorkflowDef()); } catch (JsonProcessingException e) { throw new RuntimeException(e); } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/ValidationError.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; public class ValidationError extends RuntimeException { public ValidationError(String message) { super(message); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/WorkflowBuilder.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; import java.util.*; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import com.netflix.conductor.sdk.workflow.utils.InputOutputGetter; import com.netflix.conductor.sdk.workflow.utils.MapBuilder; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.databind.ObjectMapper; /** * @param Input type for the workflow */ public class WorkflowBuilder { private String name; private String description; private int version; private String failureWorkflow; private String ownerEmail; private WorkflowDef.TimeoutPolicy timeoutPolicy; private long timeoutSeconds; private boolean restartable = true; private T defaultInput; private Map output = new HashMap<>(); private Map state; protected List> tasks = new ArrayList<>(); private WorkflowExecutor workflowExecutor; public final InputOutputGetter input = new InputOutputGetter("workflow", InputOutputGetter.Field.input); private final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); public WorkflowBuilder(WorkflowExecutor workflowExecutor) { this.workflowExecutor = workflowExecutor; this.tasks = new ArrayList<>(); } public WorkflowBuilder name(String name) { this.name = name; return this; } public WorkflowBuilder version(int version) { this.version = version; return this; } public WorkflowBuilder description(String description) { this.description = description; return this; } public WorkflowBuilder failureWorkflow(String failureWorkflow) { this.failureWorkflow = failureWorkflow; return this; } public WorkflowBuilder ownerEmail(String ownerEmail) { this.ownerEmail = ownerEmail; return this; } public WorkflowBuilder timeoutPolicy( WorkflowDef.TimeoutPolicy timeoutPolicy, long timeoutSeconds) { this.timeoutPolicy = timeoutPolicy; this.timeoutSeconds = timeoutSeconds; return this; } public WorkflowBuilder add(Task... tasks) { Collections.addAll(this.tasks, tasks); return this; } public WorkflowBuilder defaultInput(T defaultInput) { this.defaultInput = defaultInput; return this; } public WorkflowBuilder restartable(boolean restartable) { this.restartable = restartable; return this; } public WorkflowBuilder variables(Object variables) { try { this.state = objectMapper.convertValue(variables, Map.class); } catch (Exception e) { throw new IllegalArgumentException( "Workflow Variables cannot be converted to Map. Supplied: " + variables.getClass().getName()); } return this; } public WorkflowBuilder output(String key, boolean value) { output.put(key, value); return this; } public WorkflowBuilder output(String key, String value) { output.put(key, value); return this; } public WorkflowBuilder output(String key, Number value) { output.put(key, value); return this; } public WorkflowBuilder output(String key, Object value) { output.put(key, value); return this; } public WorkflowBuilder output(MapBuilder mapBuilder) { output.putAll(mapBuilder.build()); return this; } public ConductorWorkflow build() throws ValidationError { validate(); ConductorWorkflow workflow = new ConductorWorkflow(workflowExecutor); if (description != null) { workflow.setDescription(description); } workflow.setName(name); workflow.setVersion(version); workflow.setDescription(description); workflow.setFailureWorkflow(failureWorkflow); workflow.setOwnerEmail(ownerEmail); workflow.setTimeoutPolicy(timeoutPolicy); workflow.setTimeoutSeconds(timeoutSeconds); workflow.setRestartable(restartable); workflow.setDefaultInput(defaultInput); workflow.setWorkflowOutput(output); workflow.setVariables(state); for (Task task : tasks) { workflow.add(task); } return workflow; } /** * Validate: 1. There are no tasks with duplicate reference names 2. Each of the task is * consistent with its definition 3. */ private void validate() throws ValidationError { List allTasks = new ArrayList<>(); for (Task task : tasks) { List workflowDefTasks = task.getWorkflowDefTasks(); for (WorkflowTask workflowDefTask : workflowDefTasks) { allTasks.addAll(workflowDefTask.collectTasks()); } } Map taskMap = new HashMap<>(); Set duplicateTasks = new HashSet<>(); for (WorkflowTask task : allTasks) { if (taskMap.containsKey(task.getTaskReferenceName())) { duplicateTasks.add(task.getTaskReferenceName()); } else { taskMap.put(task.getTaskReferenceName(), task); } } if (!duplicateTasks.isEmpty()) { throw new ValidationError( "Task Reference Names MUST be unique across all the tasks in the workkflow. " + "Please update/change reference names to be unique for the following tasks: " + duplicateTasks); } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/DoWhile.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.ArrayList; import java.util.Collections; import java.util.List; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; public class DoWhile extends Task { private String loopCondition; private List> loopTasks = new ArrayList<>(); /** * Execute tasks in a loop determined by the condition set using condition parameter. The loop * will continue till the condition is true * * @param taskReferenceName * @param condition Javascript that evaluates to a boolean value * @param tasks */ public DoWhile(String taskReferenceName, String condition, Task... tasks) { super(taskReferenceName, TaskType.DO_WHILE); Collections.addAll(this.loopTasks, tasks); this.loopCondition = condition; } /** * Similar to a for loop, run tasks for N times * * @param taskReferenceName * @param loopCount * @param tasks */ public DoWhile(String taskReferenceName, int loopCount, Task... tasks) { super(taskReferenceName, TaskType.DO_WHILE); Collections.addAll(this.loopTasks, tasks); this.loopCondition = getForLoopCondition(loopCount); } DoWhile(WorkflowTask workflowTask) { super(workflowTask); this.loopCondition = workflowTask.getLoopCondition(); for (WorkflowTask task : workflowTask.getLoopOver()) { Task loopTask = TaskRegistry.getTask(task); this.loopTasks.add(loopTask); } } public DoWhile loopOver(Task... tasks) { for (Task task : tasks) { this.loopTasks.add(task); } return this; } private String getForLoopCondition(int loopCount) { return "if ( $." + getTaskReferenceName() + "['iteration'] < " + loopCount + ") { true; } else { false; }"; } public String getLoopCondition() { return loopCondition; } public List getLoopTasks() { return loopTasks; } @Override public void updateWorkflowTask(WorkflowTask workflowTask) { workflowTask.setLoopCondition(loopCondition); List loopWorkflowTasks = new ArrayList<>(); for (Task task : this.loopTasks) { loopWorkflowTasks.addAll(task.getWorkflowDefTasks()); } workflowTask.setLoopOver(loopWorkflowTasks); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Dynamic.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.google.common.base.Strings; /** Wait task */ public class Dynamic extends Task { public static final String TASK_NAME_INPUT_PARAM = "taskToExecute"; public Dynamic(String taskReferenceName, String dynamicTaskNameValue) { super(taskReferenceName, TaskType.DYNAMIC); if (Strings.isNullOrEmpty(dynamicTaskNameValue)) { throw new AssertionError("Null/Empty dynamicTaskNameValue"); } super.input(TASK_NAME_INPUT_PARAM, dynamicTaskNameValue); } Dynamic(WorkflowTask workflowTask) { super(workflowTask); } @Override public void updateWorkflowTask(WorkflowTask task) { task.setDynamicTaskNameParam(TASK_NAME_INPUT_PARAM); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/DynamicFork.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.ArrayList; import java.util.List; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; public class DynamicFork extends Task { public static final String FORK_TASK_PARAM = "forkedTasks"; public static final String FORK_TASK_INPUT_PARAM = "forkedTasksInputs"; private String forkTasksParameter; private String forkTasksInputsParameter; private Join join; private SimpleTask forkPrepareTask; /** * Dynamic fork task that executes a set of tasks in parallel which are determined at run time. * Use cases: Based on the input, you want to fork N number of processes in parallel to be * executed. The number N is not pre-determined at the definition time and so a regular ForkJoin * cannot be used. * * @param taskReferenceName */ public DynamicFork( String taskReferenceName, String forkTasksParameter, String forkTasksInputsParameter) { super(taskReferenceName, TaskType.FORK_JOIN_DYNAMIC); this.join = new Join(taskReferenceName + "_join"); this.forkTasksParameter = forkTasksParameter; this.forkTasksInputsParameter = forkTasksInputsParameter; super.input(FORK_TASK_PARAM, forkTasksParameter); super.input(FORK_TASK_INPUT_PARAM, forkTasksInputsParameter); } /** * Dynamic fork task that executes a set of tasks in parallel which are determined at run time. * Use cases: Based on the input, you want to fork N number of processes in parallel to be * executed. The number N is not pre-determined at the definition time and so a regular ForkJoin * cannot be used. * * @param taskReferenceName * @param forkPrepareTask A Task that produces the output as {@link DynamicForkInput} to specify * which tasks to fork. */ public DynamicFork(String taskReferenceName, SimpleTask forkPrepareTask) { super(taskReferenceName, TaskType.FORK_JOIN_DYNAMIC); this.forkPrepareTask = forkPrepareTask; this.join = new Join(taskReferenceName + "_join"); this.forkTasksParameter = forkPrepareTask.taskOutput.get(FORK_TASK_PARAM); this.forkTasksInputsParameter = forkPrepareTask.taskOutput.get(FORK_TASK_INPUT_PARAM); super.input(FORK_TASK_PARAM, forkTasksParameter); super.input(FORK_TASK_INPUT_PARAM, forkTasksInputsParameter); } DynamicFork(WorkflowTask workflowTask) { super(workflowTask); String nameOfParamForForkTask = workflowTask.getDynamicForkTasksParam(); String nameOfParamForForkTaskInput = workflowTask.getDynamicForkTasksInputParamName(); this.forkTasksParameter = (String) workflowTask.getInputParameters().get(nameOfParamForForkTask); this.forkTasksInputsParameter = (String) workflowTask.getInputParameters().get(nameOfParamForForkTaskInput); } public Join getJoin() { return join; } public String getForkTasksParameter() { return forkTasksParameter; } public String getForkTasksInputsParameter() { return forkTasksInputsParameter; } @Override public void updateWorkflowTask(WorkflowTask task) { task.setDynamicForkTasksParam("forkedTasks"); task.setDynamicForkTasksInputParamName("forkedTasksInputs"); } @Override protected List getChildrenTasks() { List tasks = new ArrayList<>(); tasks.addAll(join.getWorkflowDefTasks()); return tasks; } @Override protected List getParentTasks() { if (forkPrepareTask != null) { return List.of(forkPrepareTask.toWorkflowTask()); } return List.of(); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/DynamicForkInput.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.List; import java.util.Map; public class DynamicForkInput { /** List of tasks to execute in parallel */ private List> tasks; /** * Input to the tasks. Key is the reference name of the task and value is an Object that is sent * as input to the task */ private Map inputs; public DynamicForkInput(List> tasks, Map inputs) { this.tasks = tasks; this.inputs = inputs; } public DynamicForkInput() {} public List> getTasks() { return tasks; } public void setTasks(List> tasks) { this.tasks = tasks; } public Map getInputs() { return inputs; } public void setInputs(Map inputs) { this.inputs = inputs; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Event.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.google.common.base.Strings; /** Task to publish Events to external queuing systems like SQS, NATS, AMQP etc. */ public class Event extends Task { private static final String SINK_PARAMETER = "sink"; /** * @param taskReferenceName Unique reference name within the workflow * @param eventSink qualified name of the event sink where the message is published. Using the * format sink_type:location e.g. sqs:sqs_queue_name, amqp_queue:queue_name, * amqp_exchange:queue_name, nats:queue_name */ public Event(String taskReferenceName, String eventSink) { super(taskReferenceName, TaskType.EVENT); if (Strings.isNullOrEmpty(eventSink)) { throw new AssertionError("Null/Empty eventSink"); } super.input(SINK_PARAMETER, eventSink); } Event(WorkflowTask workflowTask) { super(workflowTask); } public String getSink() { return (String) getInput().get(SINK_PARAMETER); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/ForkJoin.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; /** ForkJoin task */ public class ForkJoin extends Task { private Join join; private Task[][] forkedTasks; /** * execute task specified in the forkedTasks parameter in parallel. * *

    forkedTask is a two-dimensional list that executes the outermost list in parallel and list * within that is executed sequentially. * *

    e.g. [[task1, task2],[task3, task4],[task5]] are executed as: * * 

         *                    ---------------
     *                    |     fork    |
     *                    ---------------
     *                    |       |     |
     *                    |       |     |
     *                  task1  task3  task5
     *                  task2  task4    |
     *                    |      |      |
     *                 ---------------------
     *                 |       join        |
     *                 ---------------------
     *
    * *

    This method automatically adds a join that waits for all the *last* tasks in the fork * (e.g. task2, task4 and task5 in the above example) to be completed.* * *

    Use join method @see {@link ForkJoin#joinOn(String...)} to override this behavior (note: * not a common scenario) * * @param taskReferenceName unique task reference name * @param forkedTasks List of tasks to be executed in parallel */ public ForkJoin(String taskReferenceName, Task[]... forkedTasks) { super(taskReferenceName, TaskType.FORK_JOIN); this.forkedTasks = forkedTasks; } ForkJoin(WorkflowTask workflowTask) { super(workflowTask); int size = workflowTask.getForkTasks().size(); this.forkedTasks = new Task[size][]; int i = 0; for (List forkTasks : workflowTask.getForkTasks()) { Task[] tasks = new Task[forkTasks.size()]; for (int j = 0; j < forkTasks.size(); j++) { WorkflowTask forkWorkflowTask = forkTasks.get(j); Task task = TaskRegistry.getTask(forkWorkflowTask); tasks[j] = task; } this.forkedTasks[i++] = tasks; } } public ForkJoin joinOn(String... joinOn) { this.join = new Join(getTaskReferenceName() + "_join", joinOn); return this; } @Override protected List getChildrenTasks() { WorkflowTask fork = toWorkflowTask(); WorkflowTask joinWorkflowTask = null; if (this.join != null) { List joinTasks = this.join.getWorkflowDefTasks(); joinWorkflowTask = joinTasks.get(0); } else { joinWorkflowTask = new WorkflowTask(); joinWorkflowTask.setWorkflowTaskType(TaskType.JOIN); joinWorkflowTask.setTaskReferenceName(getTaskReferenceName() + "_join"); joinWorkflowTask.setName(joinWorkflowTask.getTaskReferenceName()); joinWorkflowTask.setJoinOn(fork.getJoinOn()); } return Arrays.asList(joinWorkflowTask); } @Override public void updateWorkflowTask(WorkflowTask fork) { List joinOnTaskRefNames = new ArrayList<>(); List> forkTasks = new ArrayList<>(); for (Task[] forkedTaskList : forkedTasks) { List forkedWorkflowTasks = new ArrayList<>(); for (Task baseWorkflowTask : forkedTaskList) { forkedWorkflowTasks.addAll(baseWorkflowTask.getWorkflowDefTasks()); } forkTasks.add(forkedWorkflowTasks); joinOnTaskRefNames.add( forkedWorkflowTasks.get(forkedWorkflowTasks.size() - 1).getTaskReferenceName()); } if (this.join != null) { fork.setJoinOn(List.of(this.join.getJoinOn())); } else { fork.setJoinOn(joinOnTaskRefNames); } fork.setForkTasks(forkTasks); } public Task[][] getForkedTasks() { return forkedTasks; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Http.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.databind.ObjectMapper; /** Wait task */ public class Http extends Task { private static final Logger LOGGER = LoggerFactory.getLogger(Http.class); private static final String INPUT_PARAM = "http_request"; private ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); private Input httpRequest; public Http(String taskReferenceName) { super(taskReferenceName, TaskType.HTTP); this.httpRequest = new Input(); this.httpRequest.method = Input.HttpMethod.GET; super.input(INPUT_PARAM, httpRequest); } Http(WorkflowTask workflowTask) { super(workflowTask); Object inputRequest = workflowTask.getInputParameters().get(INPUT_PARAM); if (inputRequest != null) { try { this.httpRequest = objectMapper.convertValue(inputRequest, Input.class); } catch (Exception e) { LOGGER.error("Error while trying to convert input request " + e.getMessage(), e); } } } public Http input(Input httpRequest) { this.httpRequest = httpRequest; return this; } public Http url(String url) { this.httpRequest.setUri(url); return this; } public Http method(Input.HttpMethod method) { this.httpRequest.setMethod(method); return this; } public Http headers(Map headers) { this.httpRequest.setHeaders(headers); return this; } public Http body(Object body) { this.httpRequest.setBody(body); return this; } public Http readTimeout(int readTimeout) { this.httpRequest.setReadTimeOut(readTimeout); return this; } public Input getHttpRequest() { return httpRequest; } @Override protected void updateWorkflowTask(WorkflowTask workflowTask) { workflowTask.getInputParameters().put(INPUT_PARAM, httpRequest); } public static class Input { public enum HttpMethod { PUT, POST, GET, DELETE, OPTIONS, HEAD } private HttpMethod method; // PUT, POST, GET, DELETE, OPTIONS, HEAD private String vipAddress; private String appName; private Map headers = new HashMap<>(); private String uri; private Object body; private String accept = "application/json"; private String contentType = "application/json"; private Integer connectionTimeOut; private Integer readTimeOut; /** * @return the method */ public HttpMethod getMethod() { return method; } /** * @param method the method to set */ public void setMethod(HttpMethod method) { this.method = method; } /** * @return the headers */ public Map getHeaders() { return headers; } /** * @param headers the headers to set */ public void setHeaders(Map headers) { this.headers = headers; } /** * @return the body */ public Object getBody() { return body; } /** * @param body the body to set */ public void setBody(Object body) { this.body = body; } /** * @return the uri */ public String getUri() { return uri; } /** * @param uri the uri to set */ public void setUri(String uri) { this.uri = uri; } /** * @return the vipAddress */ public String getVipAddress() { return vipAddress; } /** * @param vipAddress the vipAddress to set */ public void setVipAddress(String vipAddress) { this.vipAddress = vipAddress; } /** * @return the accept */ public String getAccept() { return accept; } /** * @param accept the accept to set */ public void setAccept(String accept) { this.accept = accept; } /** * @return the MIME content type to use for the request */ public String getContentType() { return contentType; } /** * @param contentType the MIME content type to set */ public void setContentType(String contentType) { this.contentType = contentType; } public String getAppName() { return appName; } public void setAppName(String appName) { this.appName = appName; } /** * @return the connectionTimeOut */ public Integer getConnectionTimeOut() { return connectionTimeOut; } /** * @return the readTimeOut */ public Integer getReadTimeOut() { return readTimeOut; } public void setConnectionTimeOut(Integer connectionTimeOut) { this.connectionTimeOut = connectionTimeOut; } public void setReadTimeOut(Integer readTimeOut) { this.readTimeOut = readTimeOut; } @Override public String toString() { return "Input{" + "method=" + method + ", vipAddress='" + vipAddress + '\'' + ", appName='" + appName + '\'' + ", headers=" + headers + ", uri='" + uri + '\'' + ", body=" + body + ", accept='" + accept + '\'' + ", contentType='" + contentType + '\'' + ", connectionTimeOut=" + connectionTimeOut + ", readTimeOut=" + readTimeOut + '}'; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Input input = (Input) o; return method == input.method && Objects.equals(vipAddress, input.vipAddress) && Objects.equals(appName, input.appName) && Objects.equals(headers, input.headers) && Objects.equals(uri, input.uri) && Objects.equals(body, input.body) && Objects.equals(accept, input.accept) && Objects.equals(contentType, input.contentType) && Objects.equals(connectionTimeOut, input.connectionTimeOut) && Objects.equals(readTimeOut, input.readTimeOut); } @Override public int hashCode() { return Objects.hash( method, vipAddress, appName, headers, uri, body, accept, contentType, connectionTimeOut, readTimeOut); } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/JQ.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.google.common.base.Strings; /** * JQ Transformation task See https://stedolan.github.io/jq/ for how to form the queries to parse * JSON payloads */ public class JQ extends Task { private static final String QUERY_EXPRESSION_PARAMETER = "queryExpression"; public JQ(String taskReferenceName, String queryExpression) { super(taskReferenceName, TaskType.JSON_JQ_TRANSFORM); if (Strings.isNullOrEmpty(queryExpression)) { throw new AssertionError("Null/Empty queryExpression"); } super.input(QUERY_EXPRESSION_PARAMETER, queryExpression); } JQ(WorkflowTask workflowTask) { super(workflowTask); } public String getQueryExpression() { return (String) getInput().get(QUERY_EXPRESSION_PARAMETER); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Javascript.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.Map; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.ValidationError; import com.google.common.base.Strings; /** * JQ Transformation task See https://stedolan.github.io/jq/ for how to form the queries to parse * JSON payloads */ public class Javascript extends Task { private static final Logger LOGGER = LoggerFactory.getLogger(Javascript.class); private static final String EXPRESSION_PARAMETER = "expression"; private static final String EVALUATOR_TYPE_PARAMETER = "evaluatorType"; private static final String ENGINE = "nashorn"; /** * Javascript tasks are executed on the Conductor server without having to write worker code * *

    Use {@link Javascript#validate()} method to validate the javascript to ensure the script * is valid. * * @param taskReferenceName * @param script script to execute */ public Javascript(String taskReferenceName, String script) { super(taskReferenceName, TaskType.INLINE); if (Strings.isNullOrEmpty(script)) { throw new AssertionError("Null/Empty script"); } super.input(EVALUATOR_TYPE_PARAMETER, "javascript"); super.input(EXPRESSION_PARAMETER, script); } /** * Javascript tasks are executed on the Conductor server without having to write worker code * *

    Use {@link Javascript#validate()} method to validate the javascript to ensure the script * is valid. * * @param taskReferenceName * @param stream stream to load the script file from */ public Javascript(String taskReferenceName, InputStream stream) { super(taskReferenceName, TaskType.INLINE); if (stream == null) { throw new AssertionError("Stream is empty"); } super.input(EVALUATOR_TYPE_PARAMETER, "javascript"); try { String script = new String(stream.readAllBytes()); super.input(EXPRESSION_PARAMETER, script); } catch (IOException e) { throw new RuntimeException(e); } } Javascript(WorkflowTask workflowTask) { super(workflowTask); } public String getExpression() { return (String) getInput().get(EXPRESSION_PARAMETER); } /** * Validates the script. * * @return */ public Javascript validate() { ScriptEngine scriptEngine = new ScriptEngineManager().getEngineByName("Nashorn"); if (scriptEngine == null) { LOGGER.error("missing " + ENGINE + " engine. Ensure you are running supported JVM"); return this; } try { Bindings bindings = scriptEngine.createBindings(); bindings.put("$", new HashMap<>()); scriptEngine.eval(getExpression(), bindings); } catch (ScriptException e) { String message = e.getMessage(); throw new ValidationError(message); } return this; } /** * Helper method to unit test your javascript. The method is not used for creating or executing * workflow but is meant for testing only. * * @param input Input that against which the script will be executed * @return Output of the script */ public Object test(Map input) { ScriptEngine scriptEngine = new ScriptEngineManager().getEngineByName("Nashorn"); if (scriptEngine == null) { LOGGER.error("missing " + ENGINE + " engine. Ensure you are running supported JVM"); return this; } try { Bindings bindings = scriptEngine.createBindings(); bindings.put("$", input); return scriptEngine.eval(getExpression(), bindings); } catch (ScriptException e) { String message = e.getMessage(); throw new ValidationError(message); } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Join.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.Arrays; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; public class Join extends Task { private String[] joinOn; /** * @param taskReferenceName * @param joinOn List of task reference names to join on */ public Join(String taskReferenceName, String... joinOn) { super(taskReferenceName, TaskType.JOIN); this.joinOn = joinOn; } Join(WorkflowTask workflowTask) { super(workflowTask); this.joinOn = workflowTask.getJoinOn().toArray(new String[0]); } @Override protected void updateWorkflowTask(WorkflowTask workflowTask) { workflowTask.setJoinOn(Arrays.asList(joinOn)); } public String[] getJoinOn() { return joinOn; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SetVariable.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.WorkflowBuilder; public class SetVariable extends Task { /** * Sets the value of the variable in workflow. Used for workflow state management. Workflow * state is a Map that is initialized using @see {@link WorkflowBuilder#variables(Object)} * * @param taskReferenceName Use input methods to set the variable values */ public SetVariable(String taskReferenceName) { super(taskReferenceName, TaskType.SET_VARIABLE); } SetVariable(WorkflowTask workflowTask) { super(workflowTask); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SimpleTask.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; /** Workflow task executed by a worker */ public class SimpleTask extends Task { private TaskDef taskDef; public SimpleTask(String taskDefName, String taskReferenceName) { super(taskReferenceName, TaskType.SIMPLE); super.name(taskDefName); } SimpleTask(WorkflowTask workflowTask) { super(workflowTask); this.taskDef = workflowTask.getTaskDefinition(); } public TaskDef getTaskDef() { return taskDef; } public SimpleTask setTaskDef(TaskDef taskDef) { this.taskDef = taskDef; return this; } @Override protected void updateWorkflowTask(WorkflowTask workflowTask) { workflowTask.setTaskDefinition(taskDef); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SubWorkflow.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; public class SubWorkflow extends Task { private ConductorWorkflow conductorWorkflow; private String workflowName; private Integer workflowVersion; /** * Start a workflow as a sub-workflow * * @param taskReferenceName * @param workflowName * @param workflowVersion */ public SubWorkflow(String taskReferenceName, String workflowName, Integer workflowVersion) { super(taskReferenceName, TaskType.SUB_WORKFLOW); this.workflowName = workflowName; this.workflowVersion = workflowVersion; } /** * Start a workflow as a sub-workflow * * @param taskReferenceName * @param conductorWorkflow */ public SubWorkflow(String taskReferenceName, ConductorWorkflow conductorWorkflow) { super(taskReferenceName, TaskType.SUB_WORKFLOW); this.conductorWorkflow = conductorWorkflow; } SubWorkflow(WorkflowTask workflowTask) { super(workflowTask); SubWorkflowParams subworkflowParam = workflowTask.getSubWorkflowParam(); this.workflowName = subworkflowParam.getName(); this.workflowVersion = subworkflowParam.getVersion(); if (subworkflowParam.getWorkflowDef() != null) { this.conductorWorkflow = ConductorWorkflow.fromWorkflowDef(subworkflowParam.getWorkflowDef()); } } public ConductorWorkflow getConductorWorkflow() { return conductorWorkflow; } public String getWorkflowName() { return workflowName; } public int getWorkflowVersion() { return workflowVersion; } @Override protected void updateWorkflowTask(WorkflowTask workflowTask) { SubWorkflowParams subWorkflowParam = new SubWorkflowParams(); if (conductorWorkflow != null) { subWorkflowParam.setWorkflowDef(conductorWorkflow.toWorkflowDef()); } else { subWorkflowParam.setName(workflowName); subWorkflowParam.setVersion(workflowVersion); } workflowTask.setSubWorkflowParam(subWorkflowParam); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Switch.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.*; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; /** Switch Task */ public class Switch extends Task { public static final String VALUE_PARAM_NAME = "value-param"; public static final String JAVASCRIPT_NAME = "javascript"; private String caseExpression; private boolean useJavascript; private List> defaultTasks = new ArrayList<>(); private Map>> branches = new HashMap<>(); /** * Switch case (similar to if...then...else or switch in java language) * * @param taskReferenceName * @param caseExpression An expression that outputs a string value to be used as case branches. * Case expression can be a support value parameter e.g. ${workflow.input.key} or * ${task.output.key} or a Javascript statement. * @param useJavascript set to true if the caseExpression is a javascript statement */ public Switch(String taskReferenceName, String caseExpression, boolean useJavascript) { super(taskReferenceName, TaskType.SWITCH); this.caseExpression = caseExpression; this.useJavascript = useJavascript; } /** * Switch case (similar to if...then...else or switch in java language) * * @param taskReferenceName * @param caseExpression */ public Switch(String taskReferenceName, String caseExpression) { super(taskReferenceName, TaskType.SWITCH); this.caseExpression = caseExpression; this.useJavascript = false; } Switch(WorkflowTask workflowTask) { super(workflowTask); Map> decisions = workflowTask.getDecisionCases(); decisions.entrySet().stream() .forEach( branch -> { String branchName = branch.getKey(); List branchWorkflowTasks = branch.getValue(); List> branchTasks = new ArrayList<>(); for (WorkflowTask branchWorkflowTask : branchWorkflowTasks) { branchTasks.add(TaskRegistry.getTask(branchWorkflowTask)); } this.branches.put(branchName, branchTasks); }); List defaultCases = workflowTask.getDefaultCase(); for (WorkflowTask defaultCase : defaultCases) { this.defaultTasks.add(TaskRegistry.getTask(defaultCase)); } } public Switch defaultCase(Task... tasks) { defaultTasks = Arrays.asList(tasks); return this; } public Switch defaultCase(List> defaultTasks) { this.defaultTasks = defaultTasks; return this; } public Switch decisionCases(Map>> branches) { this.branches = branches; return this; } public Switch defaultCase(String... workerTasks) { for (String workerTask : workerTasks) { this.defaultTasks.add(new SimpleTask(workerTask, workerTask)); } return this; } public Switch switchCase(String caseValue, Task... tasks) { branches.put(caseValue, Arrays.asList(tasks)); return this; } public Switch switchCase(String caseValue, String... workerTasks) { List> tasks = new ArrayList<>(workerTasks.length); int i = 0; for (String workerTask : workerTasks) { tasks.add(new SimpleTask(workerTask, workerTask)); } branches.put(caseValue, tasks); return this; } public List> getDefaultTasks() { return defaultTasks; } public Map>> getBranches() { return branches; } @Override public void updateWorkflowTask(WorkflowTask workflowTask) { if (useJavascript) { workflowTask.setEvaluatorType(JAVASCRIPT_NAME); workflowTask.setExpression(caseExpression); } else { workflowTask.setEvaluatorType(VALUE_PARAM_NAME); workflowTask.getInputParameters().put("switchCaseValue", caseExpression); workflowTask.setExpression("switchCaseValue"); } Map> decisionCases = new HashMap<>(); branches.entrySet() .forEach( entry -> { String decisionCase = entry.getKey(); List> decisionTasks = entry.getValue(); List decionTaskDefs = new ArrayList<>(decisionTasks.size()); for (Task decisionTask : decisionTasks) { decionTaskDefs.addAll(decisionTask.getWorkflowDefTasks()); } decisionCases.put(decisionCase, decionTaskDefs); }); workflowTask.setDecisionCases(decisionCases); List defaultCaseTaskDefs = new ArrayList<>(defaultTasks.size()); for (Task defaultTask : defaultTasks) { defaultCaseTaskDefs.addAll(defaultTask.getWorkflowDefTasks()); } workflowTask.setDefaultCase(defaultCaseTaskDefs); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Task.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.*; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.utils.InputOutputGetter; import com.netflix.conductor.sdk.workflow.utils.MapBuilder; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; /** Workflow Task */ public abstract class Task { private String name; private String description; private String taskReferenceName; private boolean optional; private int startDelay; private TaskType type; private Map input = new HashMap<>(); protected final ObjectMapper om = new ObjectMapperProvider().getObjectMapper(); public final InputOutputGetter taskInput; public final InputOutputGetter taskOutput; public Task(String taskReferenceName, TaskType type) { if (Strings.isNullOrEmpty(taskReferenceName)) { throw new AssertionError("taskReferenceName cannot be null"); } if (type == null) { throw new AssertionError("type cannot be null"); } this.name = taskReferenceName; this.taskReferenceName = taskReferenceName; this.type = type; this.taskInput = new InputOutputGetter(taskReferenceName, InputOutputGetter.Field.input); this.taskOutput = new InputOutputGetter(taskReferenceName, InputOutputGetter.Field.output); } Task(WorkflowTask workflowTask) { this(workflowTask.getTaskReferenceName(), TaskType.valueOf(workflowTask.getType())); this.input = workflowTask.getInputParameters(); this.description = workflowTask.getDescription(); this.name = workflowTask.getName(); } public T name(String name) { this.name = name; return (T) this; } public T description(String description) { this.description = description; return (T) this; } public T input(String key, boolean value) { input.put(key, value); return (T) this; } public T input(String key, Object value) { input.put(key, value); return (T) this; } public T input(String key, char value) { input.put(key, value); return (T) this; } public T input(String key, InputOutputGetter value) { input.put(key, value.getParent()); return (T) this; } public T input(InputOutputGetter value) { return input("input", value); } public T input(String key, String value) { input.put(key, value); return (T) this; } public T input(String key, Number value) { input.put(key, value); return (T) this; } public T input(String key, Map value) { input.put(key, value); return (T) this; } public T input(Map map) { input.putAll(map); return (T) this; } public T input(MapBuilder builder) { input.putAll(builder.build()); return (T) this; } public T input(Object... keyValues) { if (keyValues.length == 1) { Object kv = keyValues[0]; Map objectMap = om.convertValue(kv, Map.class); input.putAll(objectMap); return (T) this; } if (keyValues.length % 2 == 1) { throw new IllegalArgumentException("Not all keys have value specified"); } for (int i = 0; i < keyValues.length; ) { String key = keyValues[i].toString(); Object value = keyValues[i + 1]; input.put(key, value); i += 2; } return (T) this; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getTaskReferenceName() { return taskReferenceName; } public void setTaskReferenceName(String taskReferenceName) { this.taskReferenceName = taskReferenceName; } public boolean isOptional() { return optional; } public void setOptional(boolean optional) { this.optional = optional; } public int getStartDelay() { return startDelay; } public void setStartDelay(int startDelay) { this.startDelay = startDelay; } public TaskType getType() { return type; } public String getDescription() { return description; } public Map getInput() { return input; } public final List getWorkflowDefTasks() { List workflowTasks = new ArrayList<>(); workflowTasks.addAll(getParentTasks()); workflowTasks.add(toWorkflowTask()); workflowTasks.addAll(getChildrenTasks()); return workflowTasks; } protected final WorkflowTask toWorkflowTask() { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName(name); workflowTask.setTaskReferenceName(taskReferenceName); workflowTask.setWorkflowTaskType(type); workflowTask.setDescription(description); workflowTask.setInputParameters(input); workflowTask.setStartDelay(startDelay); workflowTask.setOptional(optional); // Let the sub-classes enrich the workflow task before returning back updateWorkflowTask(workflowTask); return workflowTask; } /** * Override this method when the sub-class should update the default WorkflowTask generated * using {@link #toWorkflowTask()} * * @param workflowTask */ protected void updateWorkflowTask(WorkflowTask workflowTask) {} /** * Override this method when sub-classes will generate multiple workflow tasks. Used by tasks * which have children tasks such as do_while, fork, etc. * * @return */ protected List getChildrenTasks() { return List.of(); } protected List getParentTasks() { return List.of(); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/TaskRegistry.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; public class TaskRegistry { private static final Logger LOGGER = LoggerFactory.getLogger(TaskRegistry.class); private static Map> taskTypeMap = new HashMap<>(); public static void register(String taskType, Class taskImplementation) { taskTypeMap.put(taskType, taskImplementation); } public static Task getTask(WorkflowTask workflowTask) { Class clazz = taskTypeMap.get(workflowTask.getType()); if (clazz == null) { throw new UnsupportedOperationException( "No support to convert " + workflowTask.getType()); } Task task = null; try { task = clazz.getDeclaredConstructor(WorkflowTask.class).newInstance(workflowTask); } catch (Exception e) { LOGGER.error(e.getMessage(), e); return task; } return task; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Terminate.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.util.HashMap; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; public class Terminate extends Task { private static final String TERMINATION_STATUS_PARAMETER = "terminationStatus"; private static final String TERMINATION_WORKFLOW_OUTPUT = "workflowOutput"; private static final String TERMINATION_REASON_PARAMETER = "terminationReason"; /** * Terminate the workflow and mark it as FAILED * * @param taskReferenceName * @param reason */ public Terminate(String taskReferenceName, String reason) { this(taskReferenceName, Workflow.WorkflowStatus.FAILED, reason, new HashMap<>()); } /** * Terminate the workflow with a specific terminate status * * @param taskReferenceName * @param terminationStatus * @param reason */ public Terminate( String taskReferenceName, Workflow.WorkflowStatus terminationStatus, String reason) { this(taskReferenceName, terminationStatus, reason, new HashMap<>()); } public Terminate( String taskReferenceName, Workflow.WorkflowStatus terminationStatus, String reason, Object workflowOutput) { super(taskReferenceName, TaskType.TERMINATE); input(TERMINATION_STATUS_PARAMETER, terminationStatus.name()); input(TERMINATION_WORKFLOW_OUTPUT, workflowOutput); input(TERMINATION_REASON_PARAMETER, reason); } Terminate(WorkflowTask workflowTask) { super(workflowTask); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Wait.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def.tasks; import java.time.Duration; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import javax.swing.*; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; /** Wait task */ public class Wait extends Task { public static final String DURATION_INPUT = "duration"; public static final String UNTIL_INPUT = "until"; public static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm z"); /** * Wait until and external signal completes the task. The external signal can be either an API * call (POST /api/task) to update the task status or an event coming from a supported external * queue integration like SQS, Kafka, NATS, AMQP etc. * *


    * see * https://netflix.github.io/conductor/reference-docs/wait-task for more details * * @param taskReferenceName */ public Wait(String taskReferenceName) { super(taskReferenceName, TaskType.WAIT); } public Wait(String taskReferenceName, Duration waitFor) { super(taskReferenceName, TaskType.WAIT); long seconds = waitFor.getSeconds(); input(DURATION_INPUT, seconds + "s"); } public Wait(String taskReferenceName, ZonedDateTime waitUntil) { super(taskReferenceName, TaskType.WAIT); String formattedDateTime = waitUntil.format(dateTimeFormatter); input(UNTIL_INPUT, formattedDateTime); } Wait(WorkflowTask workflowTask) { super(workflowTask); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/WorkflowExecutor.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.http.MetadataClient; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.http.WorkflowClient; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; public class WorkflowExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowExecutor.class); private final TypeReference> listOfTaskDefs = new TypeReference<>() {}; private Map> runningWorkflowFutures = new ConcurrentHashMap<>(); private final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); private final TaskClient taskClient; private final WorkflowClient workflowClient; private final MetadataClient metadataClient; private final AnnotatedWorkerExecutor annotatedWorkerExecutor; private final ScheduledExecutorService scheduledWorkflowMonitor = Executors.newSingleThreadScheduledExecutor(); static { initTaskImplementations(); } public static void initTaskImplementations() { TaskRegistry.register(TaskType.DO_WHILE.name(), DoWhile.class); TaskRegistry.register(TaskType.DYNAMIC.name(), Dynamic.class); TaskRegistry.register(TaskType.FORK_JOIN_DYNAMIC.name(), DynamicFork.class); TaskRegistry.register(TaskType.FORK_JOIN.name(), ForkJoin.class); TaskRegistry.register(TaskType.HTTP.name(), Http.class); TaskRegistry.register(TaskType.INLINE.name(), Javascript.class); TaskRegistry.register(TaskType.JOIN.name(), Join.class); TaskRegistry.register(TaskType.JSON_JQ_TRANSFORM.name(), JQ.class); TaskRegistry.register(TaskType.SET_VARIABLE.name(), SetVariable.class); TaskRegistry.register(TaskType.SIMPLE.name(), SimpleTask.class); TaskRegistry.register(TaskType.SUB_WORKFLOW.name(), SubWorkflow.class); TaskRegistry.register(TaskType.SWITCH.name(), Switch.class); TaskRegistry.register(TaskType.TERMINATE.name(), Terminate.class); TaskRegistry.register(TaskType.WAIT.name(), Wait.class); TaskRegistry.register(TaskType.EVENT.name(), Event.class); } public WorkflowExecutor(String apiServerURL) { this(apiServerURL, 100); } public WorkflowExecutor( String apiServerURL, int pollingInterval, ClientFilter... clientFilter) { taskClient = new TaskClient(new DefaultClientConfig(), (ClientHandler) null, clientFilter); taskClient.setRootURI(apiServerURL); workflowClient = new WorkflowClient(new DefaultClientConfig(), (ClientHandler) null, clientFilter); workflowClient.setRootURI(apiServerURL); metadataClient = new MetadataClient(new DefaultClientConfig(), (ClientHandler) null, clientFilter); metadataClient.setRootURI(apiServerURL); annotatedWorkerExecutor = new AnnotatedWorkerExecutor(taskClient, pollingInterval); scheduledWorkflowMonitor.scheduleAtFixedRate( () -> { for (Map.Entry> entry : runningWorkflowFutures.entrySet()) { String workflowId = entry.getKey(); CompletableFuture future = entry.getValue(); Workflow workflow = workflowClient.getWorkflow(workflowId, true); if (workflow.getStatus().isTerminal()) { future.complete(workflow); runningWorkflowFutures.remove(workflowId); } } }, 100, 100, TimeUnit.MILLISECONDS); } public WorkflowExecutor( TaskClient taskClient, WorkflowClient workflowClient, MetadataClient metadataClient, int pollingInterval) { this.taskClient = taskClient; this.workflowClient = workflowClient; this.metadataClient = metadataClient; annotatedWorkerExecutor = new AnnotatedWorkerExecutor(taskClient, pollingInterval); scheduledWorkflowMonitor.scheduleAtFixedRate( () -> { for (Map.Entry> entry : runningWorkflowFutures.entrySet()) { String workflowId = entry.getKey(); CompletableFuture future = entry.getValue(); Workflow workflow = workflowClient.getWorkflow(workflowId, true); if (workflow.getStatus().isTerminal()) { future.complete(workflow); runningWorkflowFutures.remove(workflowId); } } }, 100, 100, TimeUnit.MILLISECONDS); } public void initWorkers(String packagesToScan) { annotatedWorkerExecutor.initWorkers(packagesToScan); } public CompletableFuture executeWorkflow(String name, Integer version, Object input) { CompletableFuture future = new CompletableFuture<>(); Map inputMap = objectMapper.convertValue(input, Map.class); StartWorkflowRequest request = new StartWorkflowRequest(); request.setInput(inputMap); request.setName(name); request.setVersion(version); String workflowId = workflowClient.startWorkflow(request); runningWorkflowFutures.put(workflowId, future); return future; } public CompletableFuture executeWorkflow( ConductorWorkflow conductorWorkflow, Object input) { CompletableFuture future = new CompletableFuture<>(); Map inputMap = objectMapper.convertValue(input, Map.class); StartWorkflowRequest request = new StartWorkflowRequest(); request.setInput(inputMap); request.setName(conductorWorkflow.getName()); request.setVersion(conductorWorkflow.getVersion()); request.setWorkflowDef(conductorWorkflow.toWorkflowDef()); String workflowId = workflowClient.startWorkflow(request); runningWorkflowFutures.put(workflowId, future); return future; } public void loadTaskDefs(String resourcePath) throws IOException { InputStream resource = WorkflowExecutor.class.getResourceAsStream(resourcePath); if (resource != null) { List taskDefs = objectMapper.readValue(resource, listOfTaskDefs); loadMetadata(taskDefs); } } public void loadWorkflowDefs(String resourcePath) throws IOException { InputStream resource = WorkflowExecutor.class.getResourceAsStream(resourcePath); if (resource != null) { WorkflowDef workflowDef = objectMapper.readValue(resource, WorkflowDef.class); loadMetadata(workflowDef); } } public void loadMetadata(WorkflowDef workflowDef) { metadataClient.registerWorkflowDef(workflowDef); } public void loadMetadata(List taskDefs) { metadataClient.registerTaskDefs(taskDefs); } public void shutdown() { scheduledWorkflowMonitor.shutdown(); annotatedWorkerExecutor.shutdown(); } public boolean registerWorkflow(WorkflowDef workflowDef, boolean overwrite) { try { if (overwrite) { metadataClient.updateWorkflowDefs(Arrays.asList(workflowDef)); } else { metadataClient.registerWorkflowDef(workflowDef); } return true; } catch (Exception e) { LOGGER.error(e.getMessage(), e); return false; } } public MetadataClient getMetadataClient() { return metadataClient; } public TaskClient getTaskClient() { return taskClient; } public WorkflowClient getWorkflowClient() { return workflowClient; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorker.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import java.lang.annotation.Annotation; import java.lang.reflect.*; import java.util.*; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.tasks.DynamicFork; import com.netflix.conductor.sdk.workflow.def.tasks.DynamicForkInput; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; public class AnnotatedWorker implements Worker { private String name; private Method workerMethod; private Object obj; private ObjectMapper om = new ObjectMapperProvider().getObjectMapper(); private int pollingInterval = 100; private Set failedStatuses = Set.of(TaskResult.Status.FAILED, TaskResult.Status.FAILED_WITH_TERMINAL_ERROR); public AnnotatedWorker(String name, Method workerMethod, Object obj) { this.name = name; this.workerMethod = workerMethod; this.obj = obj; om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } @Override public String getTaskDefName() { return name; } @Override public TaskResult execute(Task task) { TaskResult result = null; try { TaskContext context = TaskContext.set(task); Object[] parameters = getInvocationParameters(task); Object invocationResult = workerMethod.invoke(obj, parameters); result = setValue(invocationResult, context.getTaskResult()); if (!failedStatuses.contains(result.getStatus()) && result.getCallbackAfterSeconds() > 0) { result.setStatus(TaskResult.Status.IN_PROGRESS); } } catch (InvocationTargetException invocationTargetException) { if (result == null) { result = new TaskResult(task); } Throwable e = invocationTargetException.getCause(); e.printStackTrace(); if (e instanceof NonRetryableException) { result.setStatus(TaskResult.Status.FAILED_WITH_TERMINAL_ERROR); } else { result.setStatus(TaskResult.Status.FAILED); } result.setReasonForIncompletion(e.getMessage()); StringBuilder stackTrace = new StringBuilder(); for (StackTraceElement stackTraceElement : e.getStackTrace()) { String className = stackTraceElement.getClassName(); if (className.startsWith("jdk.") || className.startsWith(AnnotatedWorker.class.getName())) { break; } stackTrace.append(stackTraceElement); stackTrace.append("\n"); } result.log(stackTrace.toString()); } catch (Exception e) { throw new RuntimeException(e); } return result; } private Object[] getInvocationParameters(Task task) { Class[] parameterTypes = workerMethod.getParameterTypes(); Parameter[] parameters = workerMethod.getParameters(); if (parameterTypes.length == 1 && parameterTypes[0].equals(Task.class)) { return new Object[] {task}; } else if (parameterTypes.length == 1 && parameterTypes[0].equals(Map.class)) { return new Object[] {task.getInputData()}; } return getParameters(task, parameterTypes, parameters); } private Object[] getParameters(Task task, Class[] parameterTypes, Parameter[] parameters) { Annotation[][] parameterAnnotations = workerMethod.getParameterAnnotations(); Object[] values = new Object[parameterTypes.length]; for (int i = 0; i < parameterTypes.length; i++) { Annotation[] paramAnnotation = parameterAnnotations[i]; if (paramAnnotation != null && paramAnnotation.length > 0) { Type type = parameters[i].getParameterizedType(); Class parameterType = parameterTypes[i]; values[i] = getInputValue(task, parameterType, type, paramAnnotation); } else { values[i] = om.convertValue(task.getInputData(), parameterTypes[i]); } } return values; } private Object getInputValue( Task task, Class parameterType, Type type, Annotation[] paramAnnotation) { InputParam ip = findInputParamAnnotation(paramAnnotation); if (ip == null) { return om.convertValue(task.getInputData(), parameterType); } final String name = ip.value(); final Object value = task.getInputData().get(name); if (value == null) { return null; } if (List.class.isAssignableFrom(parameterType)) { List list = om.convertValue(value, List.class); if (type instanceof ParameterizedType) { ParameterizedType parameterizedType = (ParameterizedType) type; Class typeOfParameter = (Class) parameterizedType.getActualTypeArguments()[0]; List parameterizedList = new ArrayList<>(); for (Object item : list) { parameterizedList.add(om.convertValue(item, typeOfParameter)); } return parameterizedList; } else { return list; } } else { return om.convertValue(value, parameterType); } } private static InputParam findInputParamAnnotation(Annotation[] paramAnnotation) { return (InputParam) Arrays.stream(paramAnnotation) .filter(ann -> ann.annotationType().equals(InputParam.class)) .findFirst() .orElse(null); } private TaskResult setValue(Object invocationResult, TaskResult result) { if (invocationResult == null) { result.setStatus(TaskResult.Status.COMPLETED); return result; } OutputParam opAnnotation = workerMethod.getAnnotatedReturnType().getAnnotation(OutputParam.class); if (opAnnotation != null) { String name = opAnnotation.value(); result.getOutputData().put(name, invocationResult); result.setStatus(TaskResult.Status.COMPLETED); return result; } else if (invocationResult instanceof TaskResult) { return (TaskResult) invocationResult; } else if (invocationResult instanceof Map) { Map resultAsMap = (Map) invocationResult; result.getOutputData().putAll(resultAsMap); result.setStatus(TaskResult.Status.COMPLETED); return result; } else if (invocationResult instanceof String || invocationResult instanceof Number || invocationResult instanceof Boolean) { result.getOutputData().put("result", invocationResult); result.setStatus(TaskResult.Status.COMPLETED); return result; } else if (invocationResult instanceof List) { List resultAsList = om.convertValue(invocationResult, List.class); result.getOutputData().put("result", resultAsList); result.setStatus(TaskResult.Status.COMPLETED); return result; } else if (invocationResult instanceof DynamicForkInput) { DynamicForkInput forkInput = (DynamicForkInput) invocationResult; List> tasks = forkInput.getTasks(); List workflowTasks = new ArrayList<>(); for (com.netflix.conductor.sdk.workflow.def.tasks.Task sdkTask : tasks) { workflowTasks.addAll(sdkTask.getWorkflowDefTasks()); } result.getOutputData().put(DynamicFork.FORK_TASK_PARAM, workflowTasks); result.getOutputData().put(DynamicFork.FORK_TASK_INPUT_PARAM, forkInput.getInputs()); result.setStatus(TaskResult.Status.COMPLETED); return result; } else { Map resultAsMap = om.convertValue(invocationResult, Map.class); result.getOutputData().putAll(resultAsMap); result.setStatus(TaskResult.Status.COMPLETED); return result; } } public void setPollingInterval(int pollingInterval) { System.out.println( "Setting the polling interval for " + getTaskDefName() + ", to " + pollingInterval); this.pollingInterval = pollingInterval; } @Override public int getPollingInterval() { System.out.println("Sending the polling interval to " + pollingInterval); return pollingInterval; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import java.lang.reflect.Method; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.sdk.workflow.task.WorkerTask; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.reflect.ClassPath; public class AnnotatedWorkerExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(AnnotatedWorkerExecutor.class); private TaskClient taskClient; private TaskRunnerConfigurer taskRunner; private List executors = new ArrayList<>(); private Map workerExecutors = new HashMap<>(); private Map workerToThreadCount = new HashMap<>(); private Map workerToPollingInterval = new HashMap<>(); private Map workerDomains = new HashMap<>(); private Map workerClassObjs = new HashMap<>(); private static Set scannedPackages = new HashSet<>(); private WorkerConfiguration workerConfiguration; public AnnotatedWorkerExecutor(TaskClient taskClient) { this.taskClient = taskClient; this.workerConfiguration = new WorkerConfiguration(); } public AnnotatedWorkerExecutor(TaskClient taskClient, int pollingIntervalInMillis) { this.taskClient = taskClient; this.workerConfiguration = new WorkerConfiguration(pollingIntervalInMillis); } public AnnotatedWorkerExecutor(TaskClient taskClient, WorkerConfiguration workerConfiguration) { this.taskClient = taskClient; this.workerConfiguration = workerConfiguration; } /** * Finds any worker implementation and starts polling for tasks * * @param basePackage list of packages - comma separated - to scan for annotated worker * implementation */ public synchronized void initWorkers(String basePackage) { scanWorkers(basePackage); startPolling(); } /** Shuts down the workers */ public void shutdown() { if (taskRunner != null) { taskRunner.shutdown(); } } private void scanWorkers(String basePackage) { try { if (scannedPackages.contains(basePackage)) { // skip LOGGER.info("Package {} already scanned and will skip", basePackage); return; } // Add here so to avoid infinite recursion where a class in the package contains the // code to init workers scannedPackages.add(basePackage); List packagesToScan = new ArrayList<>(); if (basePackage != null) { String[] packages = basePackage.split(","); Collections.addAll(packagesToScan, packages); } LOGGER.info("packages to scan {}", packagesToScan); long s = System.currentTimeMillis(); ClassPath.from(AnnotatedWorkerExecutor.class.getClassLoader()) .getAllClasses() .forEach( classMeta -> { String name = classMeta.getName(); if (!includePackage(packagesToScan, name)) { return; } try { Class clazz = classMeta.load(); Object obj = clazz.getConstructor().newInstance(); addBean(obj); } catch (Throwable t) { // trace because many classes won't have a default no-args // constructor and will fail LOGGER.trace( "Caught exception while loading and scanning class {}", t.getMessage()); } }); LOGGER.info( "Took {} ms to scan all the classes, loading {} tasks", (System.currentTimeMillis() - s), workerExecutors.size()); } catch (Exception e) { LOGGER.error("Error while scanning for workers: ", e); } } private boolean includePackage(List packagesToScan, String name) { for (String scanPkg : packagesToScan) { if (name.startsWith(scanPkg)) return true; } return false; } public void addBean(Object bean) { Class clazz = bean.getClass(); for (Method method : clazz.getMethods()) { WorkerTask annotation = method.getAnnotation(WorkerTask.class); if (annotation == null) { continue; } addMethod(annotation, method, bean); } } private void addMethod(WorkerTask annotation, Method method, Object bean) { String name = annotation.value(); int threadCount = workerConfiguration.getThreadCount(name); if (threadCount == 0) { threadCount = annotation.threadCount(); } workerToThreadCount.put(name, threadCount); int pollingInterval = workerConfiguration.getPollingInterval(name); if (pollingInterval == 0) { pollingInterval = annotation.pollingInterval(); } workerToPollingInterval.put(name, pollingInterval); String domain = workerConfiguration.getDomain(name); if (Strings.isNullOrEmpty(domain)) { domain = annotation.domain(); } if (!Strings.isNullOrEmpty(domain)) { workerDomains.put(name, domain); } workerClassObjs.put(name, bean); workerExecutors.put(name, method); LOGGER.info( "Adding worker for task {}, method {} with threadCount {} and polling interval set to {} ms", name, method, threadCount, pollingInterval); } public void startPolling() { workerExecutors.forEach( (taskName, method) -> { Object obj = workerClassObjs.get(taskName); AnnotatedWorker executor = new AnnotatedWorker(taskName, method, obj); executor.setPollingInterval(workerToPollingInterval.get(taskName)); executors.add(executor); }); if (executors.isEmpty()) { return; } LOGGER.info("Starting workers with threadCount {}", workerToThreadCount); LOGGER.info("Worker domains {}", workerDomains); taskRunner = new TaskRunnerConfigurer.Builder(taskClient, executors) .withTaskThreadCount(workerToThreadCount) .withTaskToDomain(workerDomains) .build(); taskRunner.init(); } @VisibleForTesting List getExecutors() { return executors; } @VisibleForTesting TaskRunnerConfigurer getTaskRunner() { return taskRunner; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/DynamicForkWorker.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import java.lang.reflect.Method; import java.util.Map; import java.util.function.Function; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.sdk.workflow.def.tasks.DynamicFork; import com.netflix.conductor.sdk.workflow.def.tasks.DynamicForkInput; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.utils.ObjectMapperProvider; import com.fasterxml.jackson.databind.ObjectMapper; public class DynamicForkWorker implements Worker { private final int pollingInterval; private final Function workerMethod; private final String name; private ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); public DynamicForkWorker( String name, Function workerMethod, int pollingInterval) { this.name = name; this.workerMethod = workerMethod; this.pollingInterval = pollingInterval; } @Override public String getTaskDefName() { return name; } @Override public TaskResult execute(Task task) { TaskResult result = new TaskResult(task); try { Object parameter = getInvocationParameters(this.workerMethod, task); DynamicForkInput output = this.workerMethod.apply(parameter); result.getOutputData().put(DynamicFork.FORK_TASK_PARAM, output.getTasks()); result.getOutputData().put(DynamicFork.FORK_TASK_INPUT_PARAM, output.getInputs()); result.setStatus(TaskResult.Status.COMPLETED); } catch (Exception e) { throw new RuntimeException(e); } return result; } @Override public int getPollingInterval() { return pollingInterval; } private Object getInvocationParameters(Function function, Task task) { InputParam annotation = null; Class parameterType = null; for (Method method : function.getClass().getDeclaredMethods()) { if (method.getReturnType().equals(DynamicForkInput.class)) { annotation = method.getParameters()[0].getAnnotation(InputParam.class); parameterType = method.getParameters()[0].getType(); } } if (parameterType.equals(Task.class)) { return task; } else if (parameterType.equals(Map.class)) { return task.getInputData(); } if (annotation != null) { String name = annotation.value(); Object value = task.getInputData().get(name); return objectMapper.convertValue(value, parameterType); } return objectMapper.convertValue(task.getInputData(), parameterType); } public static void main(String[] args) { Function fn = new Function() { @Override public DynamicForkInput apply(@InputParam("a") TaskDef s) { return null; } }; for (Method method : fn.getClass().getDeclaredMethods()) { if (method.getReturnType().equals(DynamicForkInput.class)) { System.out.println( "\n\n-->method: " + method + ", input: " + method.getParameters()[0].getType()); System.out.println("I take input as " + method.getParameters()[0].getType()); InputParam annotation = method.getParameters()[0].getAnnotation(InputParam.class); System.out.println("I have annotation " + annotation); } } } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/NonRetryableException.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; /** * Runtime exception indicating the non-retriable error with the task execution. If thrown, the task * will fail with FAILED_WITH_TERMINAL_ERROR and will not kick off retries. */ public class NonRetryableException extends RuntimeException { public NonRetryableException(String message) { super(message); } public NonRetryableException(String message, Throwable cause) { super(message, cause); } public NonRetryableException(Throwable cause) { super(cause); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/TaskContext.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; /** Context for the task */ public class TaskContext { public static final ThreadLocal TASK_CONTEXT_INHERITABLE_THREAD_LOCAL = InheritableThreadLocal.withInitial(() -> null); public TaskContext(Task task, TaskResult taskResult) { this.task = task; this.taskResult = taskResult; } public static TaskContext get() { return TASK_CONTEXT_INHERITABLE_THREAD_LOCAL.get(); } public static TaskContext set(Task task) { TaskResult result = new TaskResult(task); TaskContext context = new TaskContext(task, result); TASK_CONTEXT_INHERITABLE_THREAD_LOCAL.set(context); return context; } private final Task task; private final TaskResult taskResult; public String getWorkflowInstanceId() { return task.getWorkflowInstanceId(); } public String getTaskId() { return task.getTaskId(); } public int getRetryCount() { return task.getRetryCount(); } public int getPollCount() { return task.getPollCount(); } public long getCallbackAfterSeconds() { return task.getCallbackAfterSeconds(); } public void addLog(String log) { this.taskResult.log(log); } public Task getTask() { return task; } public TaskResult getTaskResult() { return taskResult; } public void setCallbackAfter(int seconds) { this.taskResult.setCallbackAfterSeconds(seconds); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/WorkerConfiguration.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; public class WorkerConfiguration { private int defaultPollingInterval = 0; public WorkerConfiguration(int defaultPollingInterval) { this.defaultPollingInterval = defaultPollingInterval; } public WorkerConfiguration() {} public int getPollingInterval(String taskName) { return defaultPollingInterval; } public int getThreadCount(String taskName) { return 0; } public String getDomain(String taskName) { return null; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/task/InputParam.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.task; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.PARAMETER) public @interface InputParam { String value(); boolean required() default false; } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/task/OutputParam.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.task; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE_USE) public @interface OutputParam { String value(); } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/task/WorkerTask.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.task; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** Identifies a simple worker task. */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.METHOD}) public @interface WorkerTask { String value(); // No. of threads to use for executing the task int threadCount() default 1; int pollingInterval() default 100; String domain() default ""; } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/utils/InputOutputGetter.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.utils; import com.fasterxml.jackson.annotation.JsonIgnore; public class InputOutputGetter { public enum Field { input, output } public static final class Map { private final String parent; public Map(String parent) { this.parent = parent; } public String get(String key) { return parent + "." + key + "}"; } public Map map(String key) { return new Map(parent + "." + key); } public List list(String key) { return new List(parent + "." + key); } @Override public String toString() { return parent + "}"; } } public static final class List { private final String parent; public List(String parent) { this.parent = parent; } public List list(String key) { return new List(parent + "." + key); } public Map map(String key) { return new Map(parent + "." + key); } public String get(String key, int index) { return parent + "." + key + "[" + index + "]}"; } public String get(int index) { return parent + "[" + index + "]}"; } @Override public String toString() { return parent + "}"; } } private final String name; private final Field field; public InputOutputGetter(String name, Field field) { this.name = name; this.field = field; } public String get(String key) { return "${" + name + "." + field + "." + key + "}"; } public String getParent() { return "${" + name + "." + field + "}"; } @JsonIgnore public Map map(String key) { return new Map("${" + name + "." + field + "." + key); } @JsonIgnore public List list(String key) { return new List("${" + name + "." + field + "." + key); } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/utils/MapBuilder.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.utils; import java.util.HashMap; import java.util.Map; public class MapBuilder { private Map map = new HashMap<>(); public MapBuilder add(String key, String value) { map.put(key, value); return this; } public MapBuilder add(String key, Number value) { map.put(key, value); return this; } public MapBuilder add(String key, MapBuilder value) { map.put(key, value.build()); return this; } public Map build() { return map; } } ================================================ FILE: java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/utils/ObjectMapperProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.utils; import com.netflix.conductor.common.jackson.JsonProtoModule; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; public class ObjectMapperProvider { public ObjectMapper getObjectMapper() { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); objectMapper.setDefaultPropertyInclusion( JsonInclude.Value.construct( JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_EMPTY)); objectMapper.setSerializationInclusion(JsonInclude.Include.ALWAYS); // objectMapper.setSerializationInclusion(JsonInclude.Include.); objectMapper.registerModule(new JsonProtoModule()); return objectMapper; } } ================================================ FILE: java-sdk/src/main/resources/test-server.properties ================================================ conductor.db.type=memory conductor.indexing.enabled=false conductor.workflow-repair-service.enabled=false loadSample=true conductor.system-task-workers.enabled=false ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/def/TaskConversionsTests.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; import java.time.Duration; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import static org.junit.jupiter.api.Assertions.*; public class TaskConversionsTests { static { WorkflowExecutor.initTaskImplementations(); } @Test public void testSimpleTaskConversion() { SimpleTask simpleTask = new SimpleTask("task_name", "task_ref_name"); Map map = new HashMap<>(); map.put("key11", "value11"); map.put("key12", 100); simpleTask.input("key1", "value"); simpleTask.input("key2", 42); simpleTask.input("key3", true); simpleTask.input("key4", map); WorkflowTask workflowTask = simpleTask.getWorkflowDefTasks().get(0); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof SimpleTask); SimpleTask simpleTaskFromWorkflowTask = (SimpleTask) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(simpleTask.getName(), fromWorkflowTask.getName()); assertEquals(simpleTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(simpleTask.getTaskDef(), simpleTaskFromWorkflowTask.getTaskDef()); assertEquals(simpleTask.getType(), simpleTaskFromWorkflowTask.getType()); assertEquals(simpleTask.getStartDelay(), simpleTaskFromWorkflowTask.getStartDelay()); assertEquals(simpleTask.getInput(), simpleTaskFromWorkflowTask.getInput()); } @Test public void testDynamicTaskCoversion() { Dynamic dynamicTask = new Dynamic("task_name", "task_ref_name"); WorkflowTask workflowTask = dynamicTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters().get(Dynamic.TASK_NAME_INPUT_PARAM)); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof Dynamic); Dynamic taskFromWorkflowTask = (Dynamic) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(dynamicTask.getName(), fromWorkflowTask.getName()); assertEquals(dynamicTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(dynamicTask.getType(), taskFromWorkflowTask.getType()); assertEquals(dynamicTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(dynamicTask.getInput(), taskFromWorkflowTask.getInput()); } @Test public void testForkTaskConversion() { SimpleTask task1 = new SimpleTask("task1", "task1"); SimpleTask task2 = new SimpleTask("task2", "task2"); SimpleTask task3 = new SimpleTask("task3", "task3"); ForkJoin forkTask = new ForkJoin("task_ref_name", new Task[] {task1}, new Task[] {task2, task3}); WorkflowTask workflowTask = forkTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getForkTasks()); assertFalse(workflowTask.getForkTasks().isEmpty()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof ForkJoin); ForkJoin taskFromWorkflowTask = (ForkJoin) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(forkTask.getName(), fromWorkflowTask.getName()); assertEquals(forkTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(forkTask.getType(), taskFromWorkflowTask.getType()); assertEquals(forkTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals( forkTask.getForkedTasks().length, taskFromWorkflowTask.getForkedTasks().length); for (int i = 0; i < forkTask.getForkedTasks().length; i++) { assertEquals( forkTask.getForkedTasks()[i].length, taskFromWorkflowTask.getForkedTasks()[i].length); for (int j = 0; j < forkTask.getForkedTasks()[i].length; j++) { assertEquals( forkTask.getForkedTasks()[i][j].getTaskReferenceName(), taskFromWorkflowTask.getForkedTasks()[i][j].getTaskReferenceName()); assertEquals( forkTask.getForkedTasks()[i][j].getName(), taskFromWorkflowTask.getForkedTasks()[i][j].getName()); assertEquals( forkTask.getForkedTasks()[i][j].getType(), taskFromWorkflowTask.getForkedTasks()[i][j].getType()); } } } @Test public void testDynamicForkTaskCoversion() { DynamicFork dynamicTask = new DynamicFork("task_ref_name", "forkTasks", "forkTaskInputs"); WorkflowTask workflowTask = dynamicTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof DynamicFork); DynamicFork taskFromWorkflowTask = (DynamicFork) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(dynamicTask.getName(), fromWorkflowTask.getName()); assertEquals(dynamicTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(dynamicTask.getType(), taskFromWorkflowTask.getType()); assertEquals(dynamicTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(dynamicTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals( dynamicTask.getForkTasksParameter(), taskFromWorkflowTask.getForkTasksParameter()); assertEquals( dynamicTask.getForkTasksInputsParameter(), taskFromWorkflowTask.getForkTasksInputsParameter()); } @Test public void testDoWhileConversion() { SimpleTask task1 = new SimpleTask("task_name", "task_ref_name"); SimpleTask task2 = new SimpleTask("task_name", "task_ref_name"); DoWhile doWhileTask = new DoWhile("task_ref_name", 2, task1, task2); WorkflowTask workflowTask = doWhileTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof DoWhile); DoWhile taskFromWorkflowTask = (DoWhile) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(doWhileTask.getName(), fromWorkflowTask.getName()); assertEquals(doWhileTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(doWhileTask.getType(), taskFromWorkflowTask.getType()); assertEquals(doWhileTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(doWhileTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals(doWhileTask.getLoopCondition(), taskFromWorkflowTask.getLoopCondition()); assertEquals( doWhileTask.getLoopTasks().stream() .map(task -> task.getTaskReferenceName()) .sorted() .collect(Collectors.toSet()), taskFromWorkflowTask.getLoopTasks().stream() .map(task -> task.getTaskReferenceName()) .sorted() .collect(Collectors.toSet())); } @Test public void testJoin() { Join joinTask = new Join("task_ref_name", "task1", "task2"); WorkflowTask workflowTask = joinTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); assertNotNull(workflowTask.getJoinOn()); assertTrue(!workflowTask.getJoinOn().isEmpty()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Join, "task is not of type Join, but of type " + fromWorkflowTask.getClass().getName()); Join taskFromWorkflowTask = (Join) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(joinTask.getName(), fromWorkflowTask.getName()); assertEquals(joinTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(joinTask.getType(), taskFromWorkflowTask.getType()); assertEquals(joinTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(joinTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals(joinTask.getJoinOn().length, taskFromWorkflowTask.getJoinOn().length); assertEquals( Arrays.asList(joinTask.getJoinOn()).stream().sorted().collect(Collectors.toSet()), Arrays.asList(taskFromWorkflowTask.getJoinOn()).stream() .sorted() .collect(Collectors.toSet())); } @Test public void testEvent() { Event eventTask = new Event("task_ref_name", "sqs:queue11"); WorkflowTask workflowTask = eventTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Event, "task is not of type Event, but of type " + fromWorkflowTask.getClass().getName()); Event taskFromWorkflowTask = (Event) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(eventTask.getName(), fromWorkflowTask.getName()); assertEquals(eventTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(eventTask.getType(), taskFromWorkflowTask.getType()); assertEquals(eventTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(eventTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals(eventTask.getSink(), taskFromWorkflowTask.getSink()); } @Test public void testSetVariableConversion() { SetVariable setVariableTask = new SetVariable("task_ref_name"); WorkflowTask workflowTask = setVariableTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof SetVariable, "task is not of type SetVariable, but of type " + fromWorkflowTask.getClass().getName()); SetVariable taskFromWorkflowTask = (SetVariable) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(setVariableTask.getName(), fromWorkflowTask.getName()); assertEquals( setVariableTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(setVariableTask.getType(), taskFromWorkflowTask.getType()); assertEquals(setVariableTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(setVariableTask.getInput(), taskFromWorkflowTask.getInput()); } @Test public void testSubWorkflowConversion() { SubWorkflow subWorkflowTask = new SubWorkflow("task_ref_name", "sub_flow", 2); WorkflowTask workflowTask = subWorkflowTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof SubWorkflow, "task is not of type SubWorkflow, but of type " + fromWorkflowTask.getClass().getName()); SubWorkflow taskFromWorkflowTask = (SubWorkflow) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(subWorkflowTask.getName(), fromWorkflowTask.getName()); assertEquals( subWorkflowTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(subWorkflowTask.getType(), taskFromWorkflowTask.getType()); assertEquals(subWorkflowTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(subWorkflowTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals(subWorkflowTask.getWorkflowName(), taskFromWorkflowTask.getWorkflowName()); assertEquals( subWorkflowTask.getWorkflowVersion(), taskFromWorkflowTask.getWorkflowVersion()); } @Test public void testSwitchConversion() { SimpleTask task1 = new SimpleTask("task_name", "task_ref_name1"); SimpleTask task2 = new SimpleTask("task_name", "task_ref_name2"); SimpleTask task3 = new SimpleTask("task_name", "task_ref_name3"); Switch decision = new Switch("switch", "${workflow.input.zip"); decision.switchCase("caseA", task1); decision.switchCase("caseB", task2, task3); decision.defaultCase( new Terminate("terminate", Workflow.WorkflowStatus.FAILED, "", new HashMap<>())); WorkflowTask workflowTask = decision.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Switch, "task is not of type Switch, but of type " + fromWorkflowTask.getClass().getName()); Switch taskFromWorkflowTask = (Switch) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(decision.getName(), fromWorkflowTask.getName()); assertEquals(decision.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(decision.getType(), taskFromWorkflowTask.getType()); assertEquals(decision.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(decision.getInput(), taskFromWorkflowTask.getInput()); // TODO: ADD CASES FOR DEFAULT CASE assertEquals(decision.getBranches().keySet(), taskFromWorkflowTask.getBranches().keySet()); assertEquals( decision.getBranches().values().stream() .map( tasks -> tasks.stream() .map(Task::getTaskReferenceName) .collect(Collectors.toSet())) .collect(Collectors.toSet()), taskFromWorkflowTask.getBranches().values().stream() .map( tasks -> tasks.stream() .map(Task::getTaskReferenceName) .collect(Collectors.toSet())) .collect(Collectors.toSet())); assertEquals(decision.getBranches().size(), taskFromWorkflowTask.getBranches().size()); } @Test public void testTerminateConversion() { Terminate terminateTask = new Terminate("terminate", Workflow.WorkflowStatus.FAILED, "", new HashMap<>()); WorkflowTask workflowTask = terminateTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Terminate, "task is not of type Terminate, but of type " + fromWorkflowTask.getClass().getName()); Terminate taskFromWorkflowTask = (Terminate) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(terminateTask.getName(), fromWorkflowTask.getName()); assertEquals(terminateTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(terminateTask.getType(), taskFromWorkflowTask.getType()); assertEquals(terminateTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(terminateTask.getInput(), taskFromWorkflowTask.getInput()); } @Test public void testWaitConversion() { Wait waitTask = new Wait("terminate"); WorkflowTask workflowTask = waitTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Wait, "task is not of type Wait, but of type " + fromWorkflowTask.getClass().getName()); Wait taskFromWorkflowTask = (Wait) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(waitTask.getName(), fromWorkflowTask.getName()); assertEquals(waitTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(waitTask.getType(), taskFromWorkflowTask.getType()); assertEquals(waitTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(waitTask.getInput(), taskFromWorkflowTask.getInput()); // Wait for 10 seconds waitTask = new Wait("wait_for_10_seconds", Duration.of(10, ChronoUnit.SECONDS)); workflowTask = waitTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); assertEquals("10s", workflowTask.getInputParameters().get(Wait.DURATION_INPUT)); // Wait for 10 minutes waitTask = new Wait("wait_for_10_seconds", Duration.of(10, ChronoUnit.MINUTES)); workflowTask = waitTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); assertEquals("600s", workflowTask.getInputParameters().get(Wait.DURATION_INPUT)); // Wait till next week some time ZonedDateTime nextWeek = ZonedDateTime.now().plusDays(7); String formattedDateTime = Wait.dateTimeFormatter.format(nextWeek); waitTask = new Wait("wait_till_next_week", nextWeek); workflowTask = waitTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); assertEquals(formattedDateTime, workflowTask.getInputParameters().get(Wait.UNTIL_INPUT)); } @Test public void testHttpConverter() { Http httpTask = new Http("terminate"); Http.Input input = new Http.Input(); input.setUri("http://example.com"); input.setMethod(Http.Input.HttpMethod.POST); input.setBody("Hello World"); input.setReadTimeOut(100); Map headers = new HashMap<>(); headers.put("X-AUTHORIZATION", "my_api_key"); input.setHeaders(headers); httpTask.input(input); WorkflowTask workflowTask = httpTask.getWorkflowDefTasks().get(0); assertNotNull(workflowTask.getInputParameters()); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Http, "task is not of type Http, but of type " + fromWorkflowTask.getClass().getName()); Http taskFromWorkflowTask = (Http) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(httpTask.getName(), fromWorkflowTask.getName()); assertEquals(httpTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(httpTask.getType(), taskFromWorkflowTask.getType()); assertEquals(httpTask.getStartDelay(), taskFromWorkflowTask.getStartDelay()); assertEquals(httpTask.getInput(), taskFromWorkflowTask.getInput()); assertEquals(httpTask.getHttpRequest(), taskFromWorkflowTask.getHttpRequest()); System.out.println(httpTask.getInput()); System.out.println(taskFromWorkflowTask.getInput()); } @Test public void testJQTaskConversion() { JQ jqTask = new JQ("task_name", "{ key3: (.key1.value1 + .key2.value2) }"); Map map = new HashMap<>(); map.put("key11", "value11"); map.put("key12", 100); jqTask.input("key1", "value"); jqTask.input("key2", 42); jqTask.input("key3", true); jqTask.input("key4", map); WorkflowTask workflowTask = jqTask.getWorkflowDefTasks().get(0); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue(fromWorkflowTask instanceof JQ, "Found the instance " + fromWorkflowTask); JQ taskFromWorkflowTask = (JQ) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(jqTask.getName(), fromWorkflowTask.getName()); assertEquals(jqTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(jqTask.getQueryExpression(), taskFromWorkflowTask.getQueryExpression()); assertEquals(jqTask.getType(), taskFromWorkflowTask.getType()); assertEquals(jqTask.getInput(), taskFromWorkflowTask.getInput()); } @Test public void testInlineTaskConversion() { Javascript inlineTask = new Javascript( "task_name", "function e() { if ($.value == 1){return {\"result\": true}} else { return {\"result\": false}}} e();"); inlineTask.validate(); Map map = new HashMap<>(); map.put("key11", "value11"); map.put("key12", 100); inlineTask.input("key1", "value"); inlineTask.input("key2", 42); inlineTask.input("key3", true); inlineTask.input("key4", map); WorkflowTask workflowTask = inlineTask.getWorkflowDefTasks().get(0); Task fromWorkflowTask = TaskRegistry.getTask(workflowTask); assertTrue( fromWorkflowTask instanceof Javascript, "Found the instance " + fromWorkflowTask); Javascript taskFromWorkflowTask = (Javascript) fromWorkflowTask; assertNotNull(fromWorkflowTask); assertEquals(inlineTask.getName(), fromWorkflowTask.getName()); assertEquals(inlineTask.getTaskReferenceName(), fromWorkflowTask.getTaskReferenceName()); assertEquals(inlineTask.getExpression(), taskFromWorkflowTask.getExpression()); assertEquals(inlineTask.getType(), taskFromWorkflowTask.getType()); assertEquals(inlineTask.getInput(), taskFromWorkflowTask.getInput()); } @Test public void testJavascriptValidation() { // This script has errors Javascript inlineTask = new Javascript( "task_name", "function e() { if ($.value ==> 1){return {\"result\": true}} else { return {\"result\": false}}} e();"); boolean failed = false; try { inlineTask.validate(); } catch (ValidationError ve) { failed = true; } assertTrue(failed); // This script does NOT have errors inlineTask = new Javascript( "task_name", "function e() { if ($.value == 1){return {\"result\": true}} else { return {\"result\": false}}} e();"); failed = false; try { inlineTask.validate(); } catch (ValidationError ve) { failed = true; } assertFalse(failed); } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/def/WorkflowCreationTests.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.testing.WorkflowTestRunner; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.task.WorkerTask; import com.netflix.conductor.sdk.workflow.testing.TestWorkflowInput; import static org.junit.jupiter.api.Assertions.*; @Disabled public class WorkflowCreationTests { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowCreationTests.class); private static WorkflowExecutor executor; private static WorkflowTestRunner runner; @BeforeAll public static void init() throws IOException { runner = new WorkflowTestRunner(8080, "3.7.3"); runner.init("com.netflix.conductor.sdk"); executor = runner.getWorkflowExecutor(); } @AfterAll public static void cleanUp() { runner.shutdown(); } @WorkerTask("get_user_info") public @OutputParam("zipCode") String getZipCode(@InputParam("name") String userName) { return "95014"; } @WorkerTask("task2") public @OutputParam("greetings") String task2() { return "Hello World"; } @WorkerTask("task3") public @OutputParam("greetings") String task3() { return "Hello World-3"; } @WorkerTask("fork_gen") public DynamicForkInput generateDynamicFork() { DynamicForkInput forks = new DynamicForkInput(); Map inputs = new HashMap<>(); forks.setInputs(inputs); List> tasks = new ArrayList<>(); forks.setTasks(tasks); for (int i = 0; i < 3; i++) { SimpleTask task = new SimpleTask("task2", "fork_task_" + i); tasks.add(task); HashMap taskInput = new HashMap<>(); taskInput.put("key", "value"); taskInput.put("key2", 101); inputs.put(task.getTaskReferenceName(), taskInput); } return forks; } private ConductorWorkflow registerTestWorkflow() throws InterruptedException { InputStream script = getClass().getResourceAsStream("/script.js"); SimpleTask getUserInfo = new SimpleTask("get_user_info", "get_user_info"); getUserInfo.input("name", ConductorWorkflow.input.get("name")); SimpleTask sendToCupertino = new SimpleTask("task2", "cupertino"); SimpleTask sendToNYC = new SimpleTask("task2", "nyc"); int len = 4; Task[][] parallelTasks = new Task[len][1]; for (int i = 0; i < len; i++) { parallelTasks[i][0] = new SimpleTask("task2", "task_parallel_" + i); } WorkflowBuilder builder = new WorkflowBuilder<>(executor); TestWorkflowInput defaultInput = new TestWorkflowInput(); defaultInput.setName("defaultName"); builder.name("sdk_workflow_example") .version(1) .ownerEmail("hello@example.com") .description("Example Workflow") .restartable(true) .variables(new WorkflowState()) .timeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF, 100) .defaultInput(defaultInput) .add(new Javascript("js", script)) .add(new ForkJoin("parallel", parallelTasks)) .add(getUserInfo) .add( new Switch("decide2", "${workflow.input.zipCode}") .switchCase("95014", sendToCupertino) .switchCase("10121", sendToNYC)) // .add(new SubWorkflow("subflow", "sub_workflow_example", 5)) .add(new SimpleTask("task2", "task222")) .add(new DynamicFork("dynamic_fork", new SimpleTask("fork_gen", "fork_gen"))); ConductorWorkflow workflow = builder.build(); boolean registered = workflow.registerWorkflow(true, true); assertTrue(registered); return workflow; } @Test public void verifyCreatedWorkflow() throws Exception { ConductorWorkflow conductorWorkflow = registerTestWorkflow(); WorkflowDef def = conductorWorkflow.toWorkflowDef(); assertNotNull(def); assertTrue( def.getTasks() .get(def.getTasks().size() - 2) .getType() .equals(TaskType.TASK_TYPE_FORK_JOIN_DYNAMIC)); assertTrue( def.getTasks() .get(def.getTasks().size() - 1) .getType() .equals(TaskType.TASK_TYPE_JOIN)); } @Test public void verifyInlineWorkflowExecution() throws ValidationError { TestWorkflowInput workflowInput = new TestWorkflowInput("username", "10121", "US"); try { Workflow run = registerTestWorkflow().execute(workflowInput).get(10, TimeUnit.SECONDS); assertEquals( Workflow.WorkflowStatus.COMPLETED, run.getStatus(), run.getReasonForIncompletion()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } @Test public void testWorkflowExecutionByName() throws ExecutionException, InterruptedException { // Register the workflow first registerTestWorkflow(); TestWorkflowInput input = new TestWorkflowInput("username", "10121", "US"); ConductorWorkflow conductorWorkflow = new ConductorWorkflow(executor) .from("sdk_workflow_example", null); CompletableFuture execution = conductorWorkflow.execute(input); try { execution.get(10, TimeUnit.SECONDS); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } @Test public void verifyWorkflowExecutionFailsIfNotExists() throws ExecutionException, InterruptedException { // Register the workflow first registerTestWorkflow(); TestWorkflowInput input = new TestWorkflowInput("username", "10121", "US"); try { ConductorWorkflow conductorWorkflow = new ConductorWorkflow(executor) .from("non_existent_workflow", null); conductorWorkflow.execute(input); fail("execution should have failed"); } catch (Exception e) { } } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/def/WorkflowDefTaskTests.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; import org.junit.jupiter.api.Test; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.sdk.workflow.def.tasks.*; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import static org.junit.jupiter.api.Assertions.*; public class WorkflowDefTaskTests { static { WorkflowExecutor.initTaskImplementations(); } @Test public void testWorkflowDefTaskWithStartDelay() { SimpleTask simpleTask = new SimpleTask("task_name", "task_ref_name"); int startDelay = 5; simpleTask.setStartDelay(startDelay); WorkflowTask workflowTask = simpleTask.getWorkflowDefTasks().get(0); assertEquals(simpleTask.getStartDelay(), workflowTask.getStartDelay()); assertEquals(startDelay, simpleTask.getStartDelay()); assertEquals(startDelay, workflowTask.getStartDelay()); } @Test public void testWorkflowDefTaskWithOptionalEnabled() { SimpleTask simpleTask = new SimpleTask("task_name", "task_ref_name"); simpleTask.setOptional(true); WorkflowTask workflowTask = simpleTask.getWorkflowDefTasks().get(0); assertEquals(simpleTask.getStartDelay(), workflowTask.getStartDelay()); assertEquals(true, simpleTask.isOptional()); assertEquals(true, workflowTask.isOptional()); } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/def/WorkflowState.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.def; public class WorkflowState { private boolean paymentCompleted; private int timeTaken; public boolean isPaymentCompleted() { return paymentCompleted; } public void setPaymentCompleted(boolean paymentCompleted) { this.paymentCompleted = paymentCompleted; } public int getTimeTaken() { return timeTaken; } public void setTimeTaken(int timeTaken) { this.timeTaken = timeTaken; } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerTests.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.List; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.task.WorkerTask; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; public class AnnotatedWorkerTests { static class Car { String brand; String getBrand() { return brand; } void setBrand(String brand) { this.brand = brand; } } static class Bike { String brand; String getBrand() { return brand; } void setBrand(String brand) { this.brand = brand; } } static class CarWorker { @WorkerTask("test_1") public @OutputParam("result") List doWork(@InputParam("input") List input) { return input; } } @Test @DisplayName("it should handle null values when InputParam is a List") void nullListAsInputParam() throws NoSuchMethodException { var worker = new CarWorker(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", List.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); assertNull(outputData.get("result")); } @Test @DisplayName("it should handle an empty List as InputParam") void emptyListAsInputParam() throws NoSuchMethodException { var worker = new CarWorker(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", List.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setInputData(Map.of("input", List.of())); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); @SuppressWarnings("unchecked") List result = (List) outputData.get("result"); assertTrue(result.isEmpty()); } @Test @DisplayName("it should handle a non empty List as InputParam") void nonEmptyListAsInputParam() throws NoSuchMethodException { var worker = new CarWorker(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", List.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setInputData(Map.of("input", List.of(Map.of("brand", "BMW")))); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); @SuppressWarnings("unchecked") List result = (List) outputData.get("result"); assertEquals(1, result.size()); Car car = result.get(0); assertEquals("BMW", car.getBrand()); } @SuppressWarnings("rawtypes") static class RawListInput { @WorkerTask("test_1") public @OutputParam("result") List doWork(@InputParam("input") List input) { return input; } } @Test @DisplayName("it should handle a Raw List Type as InputParam") void rawListAsInputParam() throws NoSuchMethodException { var worker = new RawListInput(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", List.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setInputData(Map.of("input", List.of(Map.of("brand", "BMW")))); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); assertEquals(task.getInputData().get("input"), outputData.get("result")); } static class MapInput { @WorkerTask("test_1") public @OutputParam("result") Map doWork(Map input) { return input; } } @Test @DisplayName("it should accept a not annotated Map as input") void mapAsInputParam() throws NoSuchMethodException { var worker = new MapInput(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", Map.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setInputData(Map.of("input", List.of(Map.of("brand", "BMW")))); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); assertEquals(task.getInputData(), outputData.get("result")); } static class TaskInput { @WorkerTask("test_1") public @OutputParam("result") Task doWork(Task input) { return input; } } @Test @DisplayName("it should accept a Task as input") void taskAsInputParam() throws NoSuchMethodException { var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setTaskId(UUID.randomUUID().toString()); task.setInputData(Map.of("input", List.of(Map.of("brand", "BMW")))); var worker = new TaskInput(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", Task.class), worker); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); var result = (Task) outputData.get("result"); assertEquals(result.getTaskId(), task.getTaskId()); } @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.PARAMETER) public @interface AnotherAnnotation {} static class AnotherAnnotationInput { @WorkerTask("test_2") public @OutputParam("result") Bike doWork(@AnotherAnnotation Bike input) { return input; } } @Test @DisplayName( "it should convert to the correct type even if there's no @InputParam and parameters are annotated with other annotations") void annotatedWithAnotherAnnotation() throws NoSuchMethodException { var worker = new AnotherAnnotationInput(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", Bike.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setTaskId(UUID.randomUUID().toString()); task.setInputData(Map.of("brand", "Trek")); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); var bike = (Bike) outputData.get("result"); assertEquals("Trek", bike.getBrand()); } static class MultipleInputParams { @WorkerTask(value = "test_1", threadCount = 3, pollingInterval = 333) public Map doWork( @InputParam("bike") Bike bike, @InputParam("car") Car car) { return Map.of("bike", bike, "car", car); } } @Test @DisplayName("it should handle multiple input params") void multipleInputParams() throws NoSuchMethodException { var worker = new MultipleInputParams(); var annotatedWorker = new AnnotatedWorker( "test_1", worker.getClass().getMethod("doWork", Bike.class, Car.class), worker); var task = new Task(); task.setStatus(Task.Status.IN_PROGRESS); task.setTaskId(UUID.randomUUID().toString()); task.setInputData(Map.of("bike", Map.of("brand", "Trek"), "car", Map.of("brand", "BMW"))); var result0 = annotatedWorker.execute(task); var outputData = result0.getOutputData(); var bike = (Bike) outputData.get("bike"); assertEquals("Trek", bike.getBrand()); var car = (Car) outputData.get("car"); assertEquals("BMW", car.getBrand()); } @Test @DisplayName("it should honor the polling interval from annotations and config") void pollingIntervalTest() throws NoSuchMethodException { var config = new TestWorkerConfig(); var worker = new MultipleInputParams(); AnnotatedWorkerExecutor annotatedWorkerExecutor = new AnnotatedWorkerExecutor(mock(TaskClient.class)); annotatedWorkerExecutor.addBean(worker); annotatedWorkerExecutor.startPolling(); List workers = annotatedWorkerExecutor.getExecutors(); assertNotNull(workers); assertEquals(1, workers.size()); Worker taskWorker = workers.get(0); assertEquals(333, taskWorker.getPollingInterval()); var worker2 = new AnotherAnnotationInput(); annotatedWorkerExecutor = new AnnotatedWorkerExecutor(mock(TaskClient.class)); annotatedWorkerExecutor.addBean(worker2); annotatedWorkerExecutor.startPolling(); workers = annotatedWorkerExecutor.getExecutors(); assertNotNull(workers); assertEquals(1, workers.size()); taskWorker = workers.get(0); assertEquals(100, taskWorker.getPollingInterval()); config.setPollingInterval("test_2", 123); annotatedWorkerExecutor = new AnnotatedWorkerExecutor(mock(TaskClient.class), config); annotatedWorkerExecutor.addBean(worker2); annotatedWorkerExecutor.startPolling(); workers = annotatedWorkerExecutor.getExecutors(); assertNotNull(workers); assertEquals(1, workers.size()); taskWorker = workers.get(0); assertEquals(123, taskWorker.getPollingInterval()); } @Test @DisplayName("it should honor the polling interval from annotations and config") void threadCountTest() throws NoSuchMethodException { var config = new TestWorkerConfig(); var worker = new MultipleInputParams(); var worker2 = new AnotherAnnotationInput(); AnnotatedWorkerExecutor annotatedWorkerExecutor = new AnnotatedWorkerExecutor(mock(TaskClient.class), config); annotatedWorkerExecutor.addBean(worker); annotatedWorkerExecutor.addBean(worker2); annotatedWorkerExecutor.startPolling(); TaskRunnerConfigurer runner = annotatedWorkerExecutor.getTaskRunner(); assertNotNull(runner); Map taskThreadCount = runner.getTaskThreadCount(); assertNotNull(taskThreadCount); assertEquals(3, taskThreadCount.get("test_1")); assertEquals(1, taskThreadCount.get("test_2")); annotatedWorkerExecutor.shutdown(); config.setThreadCount("test_2", 2); annotatedWorkerExecutor = new AnnotatedWorkerExecutor(mock(TaskClient.class), config); annotatedWorkerExecutor.addBean(worker); annotatedWorkerExecutor.addBean(worker2); annotatedWorkerExecutor.startPolling(); runner = annotatedWorkerExecutor.getTaskRunner(); taskThreadCount = runner.getTaskThreadCount(); assertNotNull(taskThreadCount); assertEquals(3, taskThreadCount.get("test_1")); assertEquals(2, taskThreadCount.get("test_2")); } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/executor/task/TestWorkerConfig.java ================================================ /* * Copyright 2023 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.executor.task; import java.util.HashMap; import java.util.Map; public class TestWorkerConfig extends WorkerConfiguration { private Map pollingIntervals = new HashMap<>(); private Map threadCounts = new HashMap<>(); @Override public int getPollingInterval(String taskName) { return pollingIntervals.getOrDefault(taskName, 0); } public void setPollingInterval(String taskName, int interval) { pollingIntervals.put(taskName, interval); } public void setThreadCount(String taskName, int threadCount) { threadCounts.put(taskName, threadCount); } @Override public int getThreadCount(String taskName) { return threadCounts.getOrDefault(taskName, 0); } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/testing/Task1Input.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.testing; public class Task1Input { private int mod; private int oddEven; public int getMod() { return mod; } public void setMod(int mod) { this.mod = mod; } public int getOddEven() { return oddEven; } public void setOddEven(int oddEven) { this.oddEven = oddEven; } @Override public String toString() { return "Task1Input{" + "mod=" + mod + ", oddEven=" + oddEven + '}'; } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/testing/TestWorkflowInput.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.testing; public class TestWorkflowInput { private String name; private String zipCode; private String countryCode; public TestWorkflowInput(String name, String zipCode, String countryCode) { this.name = name; this.zipCode = zipCode; this.countryCode = countryCode; } public TestWorkflowInput() {} public String getName() { return name; } public void setName(String name) { this.name = name; } public String getZipCode() { return zipCode; } public void setZipCode(String zipCode) { this.zipCode = zipCode; } public String getCountryCode() { return countryCode; } public void setCountryCode(String countryCode) { this.countryCode = countryCode; } } ================================================ FILE: java-sdk/src/test/java/com/netflix/conductor/sdk/workflow/testing/WorkflowTestFrameworkTests.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.sdk.workflow.testing; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.sdk.testing.WorkflowTestRunner; import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; import com.netflix.conductor.sdk.workflow.task.InputParam; import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.task.WorkerTask; import static org.junit.jupiter.api.Assertions.*; public class WorkflowTestFrameworkTests { private static WorkflowTestRunner testRunner; private static WorkflowExecutor executor; @BeforeAll public static void init() throws IOException { testRunner = new WorkflowTestRunner(8080, "3.7.3"); testRunner.init("com.netflix.conductor.sdk.workflow.testing"); executor = testRunner.getWorkflowExecutor(); executor.loadTaskDefs("/tasks.json"); executor.loadWorkflowDefs("/simple_workflow.json"); } @AfterAll public static void cleanUp() { testRunner.shutdown(); } @Test public void testDynamicTaskExecuted() throws Exception { Map input = new HashMap<>(); input.put("task2Name", "task_2"); input.put("mod", "1"); input.put("oddEven", "12"); input.put("number", 0); // Start the workflow and wait for it to complete Workflow workflow = executor.executeWorkflow("Decision_TaskExample", 1, input).get(); assertNotNull(workflow); assertEquals(Workflow.WorkflowStatus.COMPLETED, workflow.getStatus()); assertNotNull(workflow.getOutput()); assertNotNull(workflow.getTasks()); assertFalse(workflow.getTasks().isEmpty()); assertTrue( workflow.getTasks().stream() .anyMatch(task -> task.getTaskDefName().equals("task_6"))); // task_2's implementation fails at the first try, so we should have to instances of task_2 // execution // 2 executions of task_2 should be present assertEquals( 2, workflow.getTasks().stream() .filter(task -> task.getTaskDefName().equals("task_2")) .count()); List task2Executions = workflow.getTasks().stream() .filter(task -> task.getTaskDefName().equals("task_2")) .collect(Collectors.toList()); assertNotNull(task2Executions); assertEquals(2, task2Executions.size()); // First instance would have failed and second succeeded. assertEquals(Task.Status.FAILED, task2Executions.get(0).getStatus()); assertEquals(Task.Status.COMPLETED, task2Executions.get(1).getStatus()); // task10's output assertEquals(100, workflow.getOutput().get("c")); } @Test public void testWorkflowFailure() throws Exception { Map input = new HashMap<>(); // task2Name is missing which will cause workflow to fail input.put("mod", "1"); input.put("oddEven", "12"); input.put("number", 0); // we are missing task2Name parameter which is required to wire up dynamictask // The workflow should fail as we are not passing it as input Workflow workflow = executor.executeWorkflow("Decision_TaskExample", 1, input).get(); assertNotNull(workflow); assertEquals(Workflow.WorkflowStatus.FAILED, workflow.getStatus()); assertNotNull(workflow.getReasonForIncompletion()); } @WorkerTask("task_1") public Map task1(Task1Input input) { Map result = new HashMap<>(); result.put("input", input); return result; } @WorkerTask("task_2") public TaskResult task2(Task task) { if (task.getRetryCount() < 1) { task.setStatus(Task.Status.FAILED); task.setReasonForIncompletion("try again"); return new TaskResult(task); } task.setStatus(Task.Status.COMPLETED); return new TaskResult(task); } @WorkerTask("task_6") public TaskResult task6(Task task) { task.setStatus(Task.Status.COMPLETED); return new TaskResult(task); } @WorkerTask("task_10") public TaskResult task10(Task task) { task.setStatus(Task.Status.COMPLETED); task.getOutputData().put("a", "b"); task.getOutputData().put("c", 100); task.getOutputData().put("x", false); return new TaskResult(task); } @WorkerTask("task_8") public TaskResult task8(Task task) { task.setStatus(Task.Status.COMPLETED); return new TaskResult(task); } @WorkerTask("task_5") public TaskResult task5(Task task) { task.setStatus(Task.Status.COMPLETED); return new TaskResult(task); } @WorkerTask("task_3") public @OutputParam("z1") String task3(@InputParam("taskToExecute") String p1) { return "output of task3, p1=" + p1; } @WorkerTask("task_30") public Map task30(Task task) { Map output = new HashMap<>(); output.put("v1", "b"); output.put("v2", Arrays.asList("one", "two", 3)); output.put("v3", 5); return output; } @WorkerTask("task_31") public Map task31(Task task) { Map output = new HashMap<>(); output.put("a1", "b"); output.put("a2", Arrays.asList("one", "two", 3)); output.put("a3", 5); return output; } @WorkerTask("HTTP") public Map http(Task task) { Map output = new HashMap<>(); output.put("a1", "b"); output.put("a2", Arrays.asList("one", "two", 3)); output.put("a3", 5); return output; } @WorkerTask("EVENT") public Map event(Task task) { Map output = new HashMap<>(); output.put("a1", "b"); output.put("a2", Arrays.asList("one", "two", 3)); output.put("a3", 5); return output; } } ================================================ FILE: java-sdk/src/test/resources/application-integrationtest.properties ================================================ # # /* # * Copyright 2021 Netflix, Inc. # *

    # * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with # * the License. You may obtain a copy of the License at # *

    # * http://www.apache.org/licenses/LICENSE-2.0 # *

    # * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on # * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the # * specific language governing permissions and limitations under the License. # */ # conductor.db.type=memory conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false conductor.app.stack=test conductor.app.appId=conductor conductor.app.workflow-offset-timeout=30s conductor.system-task-workers.enabled=false conductor.app.system-task-worker-callback-duration=0 conductor.app.event-message-indexing-enabled=true conductor.app.event-execution-indexing-enabled=true conductor.workflow-reconciler.enabled=true conductor.workflow-repair-service.enabled=false conductor.app.workflow-execution-lock-enabled=false conductor.app.workflow-input-payload-size-threshold=10KB conductor.app.max-workflow-input-payload-size-threshold=10240KB conductor.app.workflow-output-payload-size-threshold=10KB conductor.app.max-workflow-output-payload-size-threshold=10240KB conductor.app.task-input-payload-size-threshold=10KB conductor.app.max-task-input-payload-size-threshold=10240KB conductor.app.task-output-payload-size-threshold=10KB conductor.app.max-task-output-payload-size-threshold=10240KB conductor.app.max-workflow-variables-payload-size-threshold=2KB conductor.redis.availability-zone=us-east-1c conductor.redis.data-center-region=us-east-1 conductor.redis.workflow-namespace-prefix=integration-test conductor.redis.queue-namespace-prefix=integtest conductor.elasticsearch.index-prefix=conductor conductor.elasticsearch.cluster-health-color=yellow ================================================ FILE: java-sdk/src/test/resources/log4j2.xml ================================================ ================================================ FILE: java-sdk/src/test/resources/script.js ================================================ function e() { if ($.value > 1){ return { "key": "value", "key2": 42 }; } else { return {}; } } e(); ================================================ FILE: java-sdk/src/test/resources/simple_workflow.json ================================================ { "createTime": 1635491472393, "updateTime": 1635356450472, "name": "Decision_TaskExample", "description": "Decision_TaskExample", "version": 1, "tasks": [ { "name": "decision_task", "taskReferenceName": "decision_task", "inputParameters": { "case_value_param": "${workflow.input.number}" }, "type": "DECISION", "caseValueParam": "case_value_param", "decisionCases": { "0": [ { "name": "task_5", "taskReferenceName": "task_5", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute":"${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam":"taskToExecute", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "task_6", "taskReferenceName": "task_6", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "1": [ { "name": "task_8", "taskReferenceName": "task_8", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "task_10", "taskReferenceName": "task_10", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "task_8", "taskReferenceName": "task_8_default", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "task_10", "taskReferenceName": "task_10_last", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "abc@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ================================================ FILE: java-sdk/src/test/resources/tasks.json ================================================ [ { "createTime": 1635656118884, "createdBy": "", "name": "task_38", "description": "task_38", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638846956, "createdBy": "", "name": "encode", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "fileLocation" ], "outputKeys": [ "encodeLocation" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 1200, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "encode_admin@test.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635656118436, "createdBy": "", "name": "task_8", "description": "task_8", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118873, "createdBy": "", "name": "task_37", "description": "task_37", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118460, "createdBy": "", "name": "task_9", "description": "task_9", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118390, "createdBy": "", "name": "task_6", "description": "task_6", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118861, "createdBy": "", "name": "task_36", "description": "task_36", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847017, "createdBy": "", "name": "collect_payment_task", "description": "collect_payment_task", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118847, "createdBy": "", "name": "task_35", "description": "task_35", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118422, "createdBy": "", "name": "task_7", "description": "task_7", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118835, "createdBy": "", "name": "task_34", "description": "task_34", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118349, "createdBy": "", "name": "task_4", "description": "task_4", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118819, "createdBy": "", "name": "task_33", "description": "task_33", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118371, "createdBy": "", "name": "task_5", "description": "task_5", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118808, "createdBy": "", "name": "task_32", "description": "task_32", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118302, "createdBy": "", "name": "task_2", "description": "task_2", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 1, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118797, "createdBy": "", "name": "task_31", "description": "task_31", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118323, "createdBy": "", "name": "task_3", "description": "task_3", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118228, "createdBy": "", "name": "task_0", "description": "task_0", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118775, "createdBy": "", "name": "task_30", "description": "task_30", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118267, "createdBy": "", "name": "task_1", "description": "task_1", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847161, "createdBy": "", "name": "BookHotels", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "ui@example.com" }, { "createTime": 1635638847170, "createdBy": "", "name": "deploy", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "fileLocation" ], "outputKeys": [ "deployLocation" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 1200, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "encode_admin@test.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635763310960, "createdBy": "", "name": "ship_via_dhl", "retryCount": 3, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 2, "ownerEmail": "abc@example.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635638847180, "createdBy": "", "name": "StartBooking", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "ui@example.com" }, { "createTime": 1635434088645, "createdBy": "", "name": "Read_Name", "retryCount": 1, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 1, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "abc@example.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635638847189, "createdBy": "", "name": "book_flight_task", "description": "book_flight_task", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847198, "createdBy": "", "name": "book_car_task", "description": "book_car_task", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118758, "createdBy": "", "name": "task_29", "description": "task_29", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118747, "createdBy": "", "name": "task_28", "description": "task_28", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635278952348, "createdBy": "", "name": "ship_via_ups", "retryCount": 3, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 2, "ownerEmail": "abc@example.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635638847226, "createdBy": "", "name": "image_convert_resize", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "fileLocation", "outputFormat", "outputWidth", "outputHeight", "maintainAspectRatio" ], "outputKeys": [ "fileLocation" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 1200, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "test@example.com", "pollTimeoutSeconds": 3600 }, { "createTime": 1635638847238, "createdBy": "", "name": "deposit_money", "description": "deposit_money", "retryCount": 5, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118912, "createdBy": "", "name": "search_elasticsearch", "description": "search_elasticsearch", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118900, "createdBy": "", "name": "task_39", "description": "task_39", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118586, "createdBy": "", "name": "task_16", "description": "task_16", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635278952330, "createdBy": "", "name": "shipping_info", "retryCount": 1, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 2, "ownerEmail": "abc@example.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635656118567, "createdBy": "", "name": "task_15", "description": "task_15", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118545, "createdBy": "", "name": "task_14", "description": "task_14", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118528, "createdBy": "", "name": "task_13", "description": "task_13", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118513, "createdBy": "", "name": "task_12", "description": "task_12", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847312, "createdBy": "", "name": "withdraw_money", "description": "withdraw_money", "retryCount": 5, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118495, "createdBy": "", "name": "task_11", "description": "task_11", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118480, "createdBy": "", "name": "task_10", "description": "task_10", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "updateTime": 1636574526469, "createdBy": "user", "updatedBy": "", "name": "sample_task_name_1", "description": "This is a sample task for demo", "retryCount": 3, "timeoutSeconds": 30, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 5, "responseTimeoutSeconds": 10, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1636051623273, "createdBy": "", "name": "order_details", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "abc@example.com" }, { "createTime": 1635638847343, "createdBy": "", "name": "CompleteFlightBooking", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "ui@example.com" }, { "createTime": 1635278952339, "createdBy": "", "name": "ship_via_fedex", "retryCount": 3, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 300, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 2, "ownerEmail": "abc@example.com", "pollTimeoutSeconds": 1200 }, { "createTime": 1635638847353, "createdBy": "", "name": "map_state_codes", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 180, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@gmail.com" }, { "createTime": 1635638847362, "createdBy": "", "name": "compute_median_top_states", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 180, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@gmail.com" }, { "createTime": 1635638847374, "createdBy": "", "name": "scaleS3Image", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [ "inputBucketName", "inputKeyName", "scalingFactor", "outputBucketName", "outputKeyName" ], "outputKeys": [ "response" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 180, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "conductor@example.com" }, { "createTime": 1635656118736, "createdBy": "", "name": "task_27", "description": "task_27", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118725, "createdBy": "", "name": "task_26", "description": "task_26", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118713, "createdBy": "", "name": "task_25", "description": "task_25", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118700, "createdBy": "", "name": "task_24", "description": "task_24", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635878631466, "createdBy": "", "name": "task_23", "retryCount": 1, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "pollTimeoutSeconds": 600, "ownerEmail": "test@example.com" }, { "createTime": 1635878631456, "createdBy": "", "name": "task_22", "retryCount": 1, "timeoutSeconds": 600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 300, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "pollTimeoutSeconds": 600, "ownerEmail": "test@example.com" }, { "createTime": 1635638847436, "createdBy": "", "name": "send_email_task", "description": "send_email_task", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118658, "createdBy": "", "name": "task_21", "description": "task_21", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847459, "createdBy": "", "name": "image_multiple_convert_resize", "retryCount": 3, "timeoutSeconds": 1200, "inputKeys": [ "fileLocation", "outputFormats", "outputSizes", "maintainAspectRatio" ], "outputKeys": [ "dynamicTasks", "dynamicTasksInput" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 600, "responseTimeoutSeconds": 1200, "concurrentExecLimit": 100, "inputTemplate": {}, "rateLimitPerFrequency": 50, "rateLimitFrequencyInSeconds": 60, "ownerEmail": "exampl@example.com", "pollTimeoutSeconds": 3600 }, { "createTime": 1635656118644, "createdBy": "", "name": "task_20", "description": "task_20", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847477, "createdBy": "", "name": "simple_worker", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 180, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@gmail.com" }, { "createTime": 1635638847486, "createdBy": "", "name": "book_hotel_task", "description": "book_hotel_task", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635638847495, "createdBy": "", "name": "watermarkS3Image", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [ "inputBucketName", "inputKeyName", "watermarkBucketName", "watermarkKeyName", "outputBucketName", "outputKeyName" ], "outputKeys": [ "response" ], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 10, "responseTimeoutSeconds": 180, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "conductor@example.com" }, { "createTime": 1635656118631, "createdBy": "", "name": "task_19", "description": "task_19", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118616, "createdBy": "", "name": "task_18", "description": "task_18", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" }, { "createTime": 1635656118601, "createdBy": "", "name": "task_17", "description": "task_17", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "example@email.com" } ] ================================================ FILE: java-sdk/testing_framework.md ================================================ # Unit Testing Framework for Workflows The framework allows you to test the workflow definitions against a specific version of Conductor server. The unit tests allow the following: 1. **Input/Output Wiring**: Ensure the tasks are wired up correctly. 2. **Parameter check**: Workflow behavior with missing mandatory parameters is expected (fail if required). 3. **Task Failure behavior**: Ensure the task definitions have the right number of retries etc. For example, if the task is not idempotent, it does not get retried. 4. **Branch Testing**: Given a specific input, ensure the workflow executes a specific branch of the fork/decision. The local test server is self-contained with no additional dependencies required and stores all the data in memory. Once the test completes, the server is terminated and all the data is wiped out. ## Unit Testing Frameworks The unit testing framework is agnostic to the framework you use for testing and can be easily integrated into JUnit, Spock and other testing frameworks being used. ## Setting Up Local Server for Testing​ ```java //Setup method code - should be called once per the test lifecycle //e.g. @BeforeClass in JUnit //Download the published conductor server version 3.5.2 //Start the local server at port 8096 testRunner = new WorkflowTestRunner(8096, "3.5.2"); //Scan the packages for task workers testRunner.init("com.netflix.conductor.testing.workflows"); //Get the executor instance used for loading workflows executor = testRunner.getWorkflowExecutor(); ``` Clean up method: ```java //Clean up method code -- place in a clean up method e.g. @AfterClass in Junit //Shutdown local workers and servers and clean up any local resources in use. testRunner.shutdown(); ``` Loading workflows from JSON files for testing: ```java executor.loadTaskDefs("/tasks.json"); executor.loadWorkflowDefs("/simple_workflow.json"); ``` ## Sample test code that starts a workflow and verifies its execution ```java GetInsuranceQuote getQuote = new GetInsuranceQuote(); getQuote.setName("personA"); getQuote.setAmount(1000000.0); getQuote.setZipCode("10121"); // Start the workflow and wait for it to complete CompletableFuture workflowFuture = executor.executeWorkflow("InsuranceQuoteWorkflow", 1, getQuote); //Wait for the workflow execution to complete Workflow workflow = workflowFuture.get(); //Assertions assertNotNull(workflow); assertEquals(Workflow.WorkflowStatus.COMPLETED, workflow.getStatus()); assertNotNull(workflow.getOutput()); assertNotNull(workflow.getTasks()); assertFalse(workflow.getTasks().isEmpty()); assertTrue(workflow.getTasks().stream().anyMatch(task -> task.getTaskDefName().equals("task_6"))); ``` ================================================ FILE: java-sdk/worker_sdk.md ================================================ # Worker SDK Worker SDK makes it easy to write Conductor workers which are strongly typed with specific inputs and outputs. Annotations for the worker methods: * `@WorkerTask` - When annotated, convert a method to a Conductor worker. * `@InputParam` - Name of the input parameter to bind to from the task's input. * `@OutputParam` - Name of the output key of the task's output. Please note inputs and outputs to a task in Conductor are JSON documents. **Examples** Create a worker named `task1` that gets Task as input and produces TaskResult as output. ```java @WorkerTask("task1") public TaskResult task1(Task task) { task.setStatus(Task.Status.COMPLETED); return new TaskResult(task); } ``` Create a worker named `task2` that takes the `name` as a String input and produces an output `return "Hello, " + name` ```java @WorkerTask("task2") public @OutputParam("greetings") String task2(@InputParam("name") String name) { return "Hello, " + name; } ``` Example Task Input/Output Input: ```json { "name": "conductor" } ``` Output: ```json { "greetings": "Hello, conductor" } ``` A worker that takes complex java type as input and produces the complex output: ```java @WorkerTask("get_insurance_quote") public InsuranceQuote getInsuranceQuote(GetInsuranceQuote quoteInput) { InsuranceQuote quote = new InsuranceQuote(); //Implementation return quote; } ``` Example Task Input/Output Input: ```json { "name": "personA", "zipCode": "10121", "amount": 1000000 } ``` Output: ```json { "name": "personA", "quotedPremium": 123.50, "quotedAmount": 1000000 } ``` ## Managing Task Workers Annotated Workers are managed by [WorkflowExecutor](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/executor/WorkflowExecutor.java) ### Start Workers ```java WorkflowExecutor executor = new WorkflowExecutor("http://server/api/"); //List of packages (comma separated) to scan for annotated workers. // Please note, the worker method MUST be public and the class in which they are defined //MUST have a no-args constructor executor.initWorkers("com.company.package1,com.company.package2"); ``` ### Stop Workers The code fragment to stop workers at shutdown of the application. ```java executor.shutdown(); ``` ### Unit Testing Workers Workers implemented with the annotations are regular Java methods that can be unit tested with any testing framework. #### Mock Workers for Workflow Testing​ Create a mock worker in a different package (e.g., test) and scan for these packages when loading up the workers for integration testing. See [Unit Testing Framework](testing_framework.md) for more details on testing. ## Best Practices In a typical production environment, you will have multiple workers across different machines/VMs/pods polling for the same task. As with all Conductor workers, the following best practices apply: 1. Workers should be stateless and should not maintain any state on the process they are running. 2. Ideally, workers should be idempotent. 3. The worker should follow the Single Responsibility Principle and do exactly one thing they are responsible for. 4. The worker should not embed any workflow logic - i.e., scheduling another worker, sending a message, etc. The Conductor has features to do this, making it possible to decouple your workflow logic from worker implementation. ================================================ FILE: java-sdk/workflow_sdk.md ================================================ # Workflow SDK Workflow SDK provides fluent API to create workflows with strongly typed interfaces. ## APIs ### ConductorWorkflow [ConductorWorkflow](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/ConductorWorkflow.java) is the SDK representation of a Conductor workflow. #### Create a `ConductorWorkflow` Instance ```java ConductorWorkflow conductorWorkflow = new WorkflowBuilder(executor) .name("sdk_workflow_example") .version(1) .ownerEmail("hello@example.com") .description("Example Workflow") .timeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF, 100) .add(new SimpleTask("calculate_insurance_premium", "calculate_insurance_premium")) .add(new SimpleTask("send_email", "send_email")) .build(); ``` ### Working with Simple Worker Tasks Use [SimpleTask](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SimpleTask.java) to add a simple task to a workflow. Example: ```java ... builder.add(new SimpleTask("send_email", "send_email")) ... ``` ### Wiring Inputs to Task Use `input` methods to configure the inputs to the task. See our doc on [task inputs](https://conductor.netflix.com/how-tos/Tasks/task-inputs.html) for more details. Example ```java builder.add( new SimpleTask("send_email", "send_email") .input("email", "${workflow.input.email}") .input("subject", "Your insurance quote for the amount ${generate_quote.output.amount}") ); ``` ### Working with Operators Each operator has its own class that can be added to the workflow builder. * [ForkJoin](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/ForkJoin.java) * [Wait](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Wait.java) * [Switch](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Switch.java) * [DynamicFork](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/DynamicFork.java) * [DoWhile](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/DoWhile.java) * [Join](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Join.java) * [Dynamic](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Dynamic.java) * [Terminate](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Terminate.java) * [SubWorkflow](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SubWorkflow.java) * [SetVariable](https://github.com/Netflix/conductor/blob/main/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/SetVariable.java) #### Register Workflow with Conductor Server ```java //Returns true if the workflow is successfully created //Reasons why this method will return false //1. Network connectivity issue //2. Workflow already exists with the specified name and version //3. There are missing task definitions boolean registered = workflow.registerWorkflow(); ``` #### Overwrite Existing Workflow Definition​ ```java boolean registered = workflow.registerWorkflow(true); ``` #### Overwrite existing workflow definitions & registering any missing task definitions ```java boolean registered = workflow.registerWorkflow(true, true); ``` #### Create `ConductorWorkflow` based on the definition registered on the server ```java ConductorWorkflow conductorWorkflow = new ConductorWorkflow(executor) .from("sdk_workflow_example", 1); ``` #### Start Workflow Execution Start the execution of the workflow based on the definition registered on the server. Use the register method to register a workflow on the server before executing. ```java //Returns a completable future CompletableFuture execution = conductorWorkflow.execute(input); //Wait for the workflow to complete -- useful if workflow completes within a reasonable amount of time Workflow workflowRun = execution.get(); //Get the workflowId String workflowId = workflowRun.getWorkflowId(); //Get the status of workflow execution WorkflowStatus status = workflowRun.getStatus(); ``` See [Workflow](https://github.com/Netflix/conductor/blob/main/common/src/main/java/com/netflix/conductor/common/run/Workflow.java) for more details on the Workflow object. #### Start Dynamic Workflow Execution Dynamic workflows are executed by specifying the workflow definition along with the execution and do not require registering the workflow on the server before executing. ##### Use cases for dynamic workflows 1. Each workflow run has a unique workflow definition 2. Workflows are defined based on the user data and cannot be modeled ahead of time statically ```java //1. Use WorkflowBuilder to create ConductorWorkflow. //2. Execute using the definition created by SDK. CompletableFuture execution = conductorWorkflow.executeDynamic(input); ``` ================================================ FILE: json-jq-task/build.gradle ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "net.thisptr:jackson-jq:${revJq}" implementation "com.github.ben-manes.caffeine:caffeine" } ================================================ FILE: json-jq-task/src/main/java/com/netflix/conductor/tasks/json/JsonJqTransform.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.json; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import net.thisptr.jackson.jq.JsonQuery; import net.thisptr.jackson.jq.Scope; @Component(JsonJqTransform.NAME) public class JsonJqTransform extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(JsonJqTransform.class); public static final String NAME = "JSON_JQ_TRANSFORM"; private static final String QUERY_EXPRESSION_PARAMETER = "queryExpression"; private static final String OUTPUT_RESULT = "result"; private static final String OUTPUT_RESULT_LIST = "resultList"; private static final String OUTPUT_ERROR = "error"; private static final TypeReference> mapType = new TypeReference<>() {}; private final TypeReference> listType = new TypeReference<>() {}; private final Scope rootScope; private final ObjectMapper objectMapper; private final LoadingCache queryCache = createQueryCache(); @Autowired public JsonJqTransform(ObjectMapper objectMapper) { super(NAME); this.objectMapper = objectMapper; this.rootScope = Scope.newEmptyScope(); this.rootScope.loadFunctions(Scope.class.getClassLoader()); } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { final Map taskInput = task.getInputData(); final String queryExpression = (String) taskInput.get(QUERY_EXPRESSION_PARAMETER); if (queryExpression == null) { task.setReasonForIncompletion( "Missing '" + QUERY_EXPRESSION_PARAMETER + "' in input parameters"); task.setStatus(TaskModel.Status.FAILED); return; } try { final JsonNode input = objectMapper.valueToTree(taskInput); final JsonQuery query = queryCache.get(queryExpression); final Scope childScope = Scope.newChildScope(rootScope); final List result = query.apply(childScope, input); task.setStatus(TaskModel.Status.COMPLETED); if (result == null) { task.addOutput(OUTPUT_RESULT, null); task.addOutput(OUTPUT_RESULT_LIST, null); } else if (result.isEmpty()) { task.addOutput(OUTPUT_RESULT, null); task.addOutput(OUTPUT_RESULT_LIST, result); } else { task.addOutput(OUTPUT_RESULT, extractBody(result.get(0))); task.addOutput(OUTPUT_RESULT_LIST, result); } } catch (final Exception e) { LOGGER.error( "Error executing task: {} in workflow: {}", task.getTaskId(), workflow.getWorkflowId(), e); task.setStatus(TaskModel.Status.FAILED); final String message = extractFirstValidMessage(e); task.setReasonForIncompletion(message); task.addOutput(OUTPUT_ERROR, message); } } private LoadingCache createQueryCache() { final CacheLoader loader = JsonQuery::compile; return Caffeine.newBuilder() .expireAfterWrite(1, TimeUnit.HOURS) .maximumSize(1000) .build(loader); } @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { this.start(workflow, task, workflowExecutor); return true; } private String extractFirstValidMessage(final Exception e) { Throwable currentStack = e; final List messages = new ArrayList<>(); messages.add(currentStack.getMessage()); while (currentStack.getCause() != null) { currentStack = currentStack.getCause(); messages.add(currentStack.getMessage()); } return messages.stream().filter(it -> !it.contains("N/A")).findFirst().orElse(""); } private Object extractBody(JsonNode node) { if (node.isNull()) { return null; } else if (node.isObject()) { return objectMapper.convertValue(node, mapType); } else if (node.isArray()) { return objectMapper.convertValue(node, listType); } else if (node.isBoolean()) { return node.asBoolean(); } else if (node.isNumber()) { if (node.isIntegralNumber()) { return node.asLong(); } else { return node.asDouble(); } } else { return node.asText(); } } } ================================================ FILE: json-jq-task/src/test/java/com/netflix/conductor/tasks/json/JsonJqTransformTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.tasks.json; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Test; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.*; public class JsonJqTransformTest { private final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); @Test public void dataShouldBeCorrectlySelected() { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map inputData = new HashMap<>(); inputData.put("queryExpression", ".inputJson.key[0]"); final Map inputJson = new HashMap<>(); inputJson.put("key", Collections.singletonList("VALUE")); inputData.put("inputJson", inputJson); task.setInputData(inputData); task.setOutputData(new HashMap<>()); jsonJqTransform.start(workflow, task, null); assertNull(task.getOutputData().get("error")); assertEquals("VALUE", task.getOutputData().get("result").toString()); assertEquals("[\"VALUE\"]", task.getOutputData().get("resultList").toString()); } @Test public void simpleErrorShouldBeDisplayed() { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map inputData = new HashMap<>(); inputData.put("queryExpression", "{"); task.setInputData(inputData); task.setOutputData(new HashMap<>()); jsonJqTransform.start(workflow, task, null); assertTrue( ((String) task.getOutputData().get("error")) .startsWith("Encountered \"\" at line 1, column 1.")); } @Test public void nestedExceptionsWithNACausesShouldBeDisregarded() { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map inputData = new HashMap<>(); inputData.put( "queryExpression", "{officeID: (.inputJson.OIDs | unique)[], requestedIndicatorList: .inputJson.requestedindicatorList}"); final Map inputJson = new HashMap<>(); inputJson.put("OIDs", Collections.singletonList("VALUE")); final Map indicatorList = new HashMap<>(); indicatorList.put("indicator", "AFA"); indicatorList.put("value", false); inputJson.put("requestedindicatorList", Collections.singletonList(indicatorList)); inputData.put("inputJson", inputJson); task.setInputData(inputData); task.setOutputData(new HashMap<>()); jsonJqTransform.start(workflow, task, null); assertTrue( ((String) task.getOutputData().get("error")) .startsWith("Encountered \" \"[\" \"[ \"\" at line 1")); } @Test public void mapResultShouldBeCorrectlyExtracted() { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map taskInput = new HashMap<>(); Map inputData = new HashMap<>(); inputData.put("method", "POST"); inputData.put("successExpression", null); inputData.put("requestTransform", "{name: (.body.name + \" you are a \" + .body.title) }"); inputData.put("responseTransform", "{result: \"reply: \" + .response.body.message}"); taskInput.put("input", inputData); taskInput.put( "queryExpression", "{ requestTransform: .input.requestTransform // \".body\" , responseTransform: .input.responseTransform // \".response.body\", method: .input.method // \"GET\", document: .input.document // \"rgt_results\", successExpression: .input.successExpression // \"true\" }"); task.setInputData(taskInput); task.setOutputData(new HashMap<>()); jsonJqTransform.start(workflow, task, null); assertNull(task.getOutputData().get("error")); assertTrue(task.getOutputData().get("result") instanceof Map); HashMap result = (HashMap) task.getOutputData().get("result"); assertEquals("POST", result.get("method")); assertEquals( "{name: (.body.name + \" you are a \" + .body.title) }", result.get("requestTransform")); assertEquals( "{result: \"reply: \" + .response.body.message}", result.get("responseTransform")); } @Test public void stringResultShouldBeCorrectlyExtracted() { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map taskInput = new HashMap<>(); taskInput.put("data", new ArrayList<>()); taskInput.put( "queryExpression", "if(.data | length >0) then \"EXISTS\" else \"CREATE\" end"); task.setInputData(taskInput); jsonJqTransform.start(workflow, task, null); assertNull(task.getOutputData().get("error")); assertTrue(task.getOutputData().get("result") instanceof String); String result = (String) task.getOutputData().get("result"); assertEquals("CREATE", result); } @SuppressWarnings("unchecked") @Test public void listResultShouldBeCorrectlyExtracted() throws JsonProcessingException { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); String json = "{ \"request\": { \"transitions\": [ { \"name\": \"redeliver\" }, { \"name\": \"redeliver_from_validation_error\" }, { \"name\": \"redelivery\" } ] } }"; Map inputData = objectMapper.readValue(json, Map.class); final Map taskInput = new HashMap<>(); taskInput.put("inputData", inputData); taskInput.put("queryExpression", ".inputData.request.transitions | map(.name)"); task.setInputData(taskInput); jsonJqTransform.start(workflow, task, null); assertNull(task.getOutputData().get("error")); assertTrue(task.getOutputData().get("result") instanceof List); List result = (List) task.getOutputData().get("result"); assertEquals(3, result.size()); } @Test public void nullResultShouldBeCorrectlyExtracted() throws JsonProcessingException { final JsonJqTransform jsonJqTransform = new JsonJqTransform(objectMapper); final WorkflowModel workflow = new WorkflowModel(); final TaskModel task = new TaskModel(); final Map taskInput = new HashMap<>(); taskInput.put("queryExpression", "null"); task.setInputData(taskInput); jsonJqTransform.start(workflow, task, null); assertNull(task.getOutputData().get("error")); assertNull(task.getOutputData().get("result")); } } ================================================ FILE: licenseheader.txt ================================================ /* * Copyright $YEAR Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ ================================================ FILE: polyglot-clients/README.md ================================================ # SDKs for other languages Language specific client SDKs are maintained at a dedicated [conductor-sdk](https://github.com/conductor-sdk) repository. Check the repository for the latest list, but there are SDK clients for: ## SDK List * [Clojure](https://github.com/conductor-sdk/conductor-clojure) * [C#](https://github.com/conductor-sdk/conductor-csharp) * [Go](https://github.com/conductor-sdk/conductor-go) * [Python](https://github.com/conductor-sdk/conductor-python) ### In progress (PRs encouraged!) * [JavaScript](https://github.com/conductor-sdk/conductor-javascript) ================================================ FILE: redis-concurrency-limit/build.gradle ================================================ plugins { id 'groovy' } dependencies { compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.data:spring-data-redis' implementation project(':conductor-common') implementation project(':conductor-core') implementation "redis.clients:jedis:3.6.0" // Jedis version "revJedis=3.3.0" does not play well with Spring Data Redis implementation "org.apache.commons:commons-lang3" testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" testImplementation "org.testcontainers:spock:${revTestContainer}" testImplementation "org.testcontainers:testcontainers:${revTestContainer}" testImplementation "com.google.protobuf:protobuf-java:${revProtoBuf}" testImplementation 'org.springframework.data:spring-data-redis' } ================================================ FILE: redis-concurrency-limit/src/main/java/com/netflix/conductor/redis/limit/RedisConcurrentExecutionLimitDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.limit; import java.util.Optional; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.ConcurrentExecutionLimitDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.redis.limit.config.RedisConcurrentExecutionLimitProperties; @Trace @Component @ConditionalOnProperty( value = "conductor.redis-concurrent-execution-limit.enabled", havingValue = "true") public class RedisConcurrentExecutionLimitDAO implements ConcurrentExecutionLimitDAO { private static final Logger LOGGER = LoggerFactory.getLogger(RedisConcurrentExecutionLimitDAO.class); private static final String CLASS_NAME = RedisConcurrentExecutionLimitDAO.class.getSimpleName(); private final StringRedisTemplate stringRedisTemplate; private final RedisConcurrentExecutionLimitProperties properties; public RedisConcurrentExecutionLimitDAO( StringRedisTemplate stringRedisTemplate, RedisConcurrentExecutionLimitProperties properties) { this.stringRedisTemplate = stringRedisTemplate; this.properties = properties; } /** * Adds the {@link TaskModel} identifier to a Redis Set for the {@link TaskDef}'s name. * * @param task The {@link TaskModel} object. */ @Override public void addTaskToLimit(TaskModel task) { try { Monitors.recordDaoRequests( CLASS_NAME, "addTaskToLimit", task.getTaskType(), task.getWorkflowType()); String taskId = task.getTaskId(); String taskDefName = task.getTaskDefName(); String keyName = createKeyName(taskDefName); stringRedisTemplate.opsForSet().add(keyName, taskId); LOGGER.debug("Added taskId: {} to key: {}", taskId, keyName); } catch (Exception e) { Monitors.error(CLASS_NAME, "addTaskToLimit"); String errorMsg = String.format( "Error updating taskDefLimit for task - %s:%s in workflow: %s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } /** * Remove the {@link TaskModel} identifier from the Redis Set for the {@link TaskDef}'s name. * * @param task The {@link TaskModel} object. */ @Override public void removeTaskFromLimit(TaskModel task) { try { Monitors.recordDaoRequests( CLASS_NAME, "removeTaskFromLimit", task.getTaskType(), task.getWorkflowType()); String taskId = task.getTaskId(); String taskDefName = task.getTaskDefName(); String keyName = createKeyName(taskDefName); stringRedisTemplate.opsForSet().remove(keyName, taskId); LOGGER.debug("Removed taskId: {} from key: {}", taskId, keyName); } catch (Exception e) { Monitors.error(CLASS_NAME, "removeTaskFromLimit"); String errorMsg = String.format( "Error updating taskDefLimit for task - %s:%s in workflow: %s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg, e); } } /** * Checks if the {@link TaskModel} identifier is in the Redis Set and size of the set is more * than the {@link TaskDef#concurrencyLimit()}. * * @param task The {@link TaskModel} object. * @return true if the task id is not in the set and size of the set is more than the {@link * TaskDef#concurrencyLimit()}. */ @Override public boolean exceedsLimit(TaskModel task) { Optional taskDefinition = task.getTaskDefinition(); if (taskDefinition.isEmpty()) { return false; } int limit = taskDefinition.get().concurrencyLimit(); if (limit <= 0) { return false; } try { Monitors.recordDaoRequests( CLASS_NAME, "exceedsLimit", task.getTaskType(), task.getWorkflowType()); String taskId = task.getTaskId(); String taskDefName = task.getTaskDefName(); String keyName = createKeyName(taskDefName); boolean isMember = ObjectUtils.defaultIfNull( stringRedisTemplate.opsForSet().isMember(keyName, taskId), false); long size = ObjectUtils.defaultIfNull(stringRedisTemplate.opsForSet().size(keyName), -1L); LOGGER.debug( "Task: {} is {} of {}, size: {} and limit: {}", taskId, isMember ? "a member" : "not a member", keyName, size, limit); return !isMember && size >= limit; } catch (Exception e) { Monitors.error(CLASS_NAME, "exceedsLimit"); String errorMsg = String.format( "Failed to get in progress limit - %s:%s in workflow :%s", task.getTaskDefName(), task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.error(errorMsg, e); throw new TransientException(errorMsg); } } private String createKeyName(String taskDefName) { StringBuilder builder = new StringBuilder(); String namespace = properties.getNamespace(); if (StringUtils.isNotBlank(namespace)) { builder.append(namespace).append(':'); } return builder.append(taskDefName).toString(); } } ================================================ FILE: redis-concurrency-limit/src/main/java/com/netflix/conductor/redis/limit/config/RedisConcurrentExecutionLimitConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.limit.config; import java.util.List; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.connection.RedisClusterConfiguration; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.RedisStandaloneConfiguration; import org.springframework.data.redis.connection.jedis.JedisClientConfiguration; import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; @Configuration @ConditionalOnProperty( value = "conductor.redis-concurrent-execution-limit.enabled", havingValue = "true") @EnableConfigurationProperties(RedisConcurrentExecutionLimitProperties.class) public class RedisConcurrentExecutionLimitConfiguration { @Bean @ConditionalOnProperty( value = "conductor.redis-concurrent-execution-limit.type", havingValue = "cluster") public RedisConnectionFactory redisClusterConnectionFactory( RedisConcurrentExecutionLimitProperties properties) { GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig<>(); poolConfig.setMaxTotal(properties.getMaxConnectionsPerHost()); poolConfig.setTestWhileIdle(true); JedisClientConfiguration clientConfig = JedisClientConfiguration.builder() .usePooling() .poolConfig(poolConfig) .and() .clientName(properties.getClientName()) .build(); RedisClusterConfiguration redisClusterConfiguration = new RedisClusterConfiguration( List.of(properties.getHost() + ":" + properties.getPort())); return new JedisConnectionFactory(redisClusterConfiguration, clientConfig); } @Bean @ConditionalOnProperty( value = "conductor.redis-concurrent-execution-limit.type", havingValue = "standalone", matchIfMissing = true) public RedisConnectionFactory redisStandaloneConnectionFactory( RedisConcurrentExecutionLimitProperties properties) { RedisStandaloneConfiguration config = new RedisStandaloneConfiguration(properties.getHost(), properties.getPort()); return new JedisConnectionFactory(config); } } ================================================ FILE: redis-concurrency-limit/src/main/java/com/netflix/conductor/redis/limit/config/RedisConcurrentExecutionLimitProperties.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.limit.config; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("conductor.redis-concurrent-execution-limit") public class RedisConcurrentExecutionLimitProperties { public enum RedisType { STANDALONE, CLUSTER } private RedisType type; private String host; private int port; private String password; private int maxConnectionsPerHost; private String clientName; private String namespace = "conductor"; public RedisType getType() { return type; } public void setType(RedisType type) { this.type = type; } public int getMaxConnectionsPerHost() { return maxConnectionsPerHost; } public void setMaxConnectionsPerHost(int maxConnectionsPerHost) { this.maxConnectionsPerHost = maxConnectionsPerHost; } public String getClientName() { return clientName; } public void setClientName(String clientName) { this.clientName = clientName; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } public String getNamespace() { return namespace; } public void setNamespace(String namespace) { this.namespace = namespace; } } ================================================ FILE: redis-concurrency-limit/src/test/groovy/com/netflix/conductor/redis/limit/RedisConcurrentExecutionLimitDAOSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.limit import org.springframework.data.redis.connection.RedisStandaloneConfiguration import org.springframework.data.redis.connection.jedis.JedisConnectionFactory import org.springframework.data.redis.core.StringRedisTemplate import org.testcontainers.containers.GenericContainer import org.testcontainers.spock.Testcontainers import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.model.TaskModel import com.netflix.conductor.redis.limit.config.RedisConcurrentExecutionLimitProperties import spock.lang.Specification import spock.lang.Subject import spock.lang.Unroll @Testcontainers class RedisConcurrentExecutionLimitDAOSpec extends Specification { GenericContainer redis = new GenericContainer("redis:5.0.3-alpine") .withExposedPorts(6379) @Subject RedisConcurrentExecutionLimitDAO dao StringRedisTemplate redisTemplate RedisConcurrentExecutionLimitProperties properties def setup() { properties = new RedisConcurrentExecutionLimitProperties(namespace: 'conductor') def factory = new JedisConnectionFactory(new RedisStandaloneConfiguration(redis.host, redis.firstMappedPort)) factory.afterPropertiesSet() redisTemplate = new StringRedisTemplate(factory) dao = new RedisConcurrentExecutionLimitDAO(redisTemplate, properties) } def "verify addTaskToLimit adds the taskId to the right set"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' def keyName = "${properties.namespace}:$taskDefName" as String TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName) when: dao.addTaskToLimit(task) then: redisTemplate.hasKey(keyName) redisTemplate.opsForSet().size(keyName) == 1 redisTemplate.opsForSet().isMember(keyName, taskId) } def "verify removeTaskFromLimit removes the taskId from the right set"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' def keyName = "${properties.namespace}:$taskDefName" as String redisTemplate.opsForSet().add(keyName, taskId) TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName) when: dao.removeTaskFromLimit(task) then: !redisTemplate.hasKey(keyName) // since the only element in the set is removed, Redis removes the set } @Unroll def "verify exceedsLimit returns false for #testCase"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName, workflowTask: workflowTask) when: def retVal = dao.exceedsLimit(task) then: !retVal where: workflowTask << [new WorkflowTask(taskDefinition: null), new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: -2))] testCase << ['a task with no TaskDefinition', 'TaskDefinition with concurrentExecLimit is less than 0'] } def "verify exceedsLimit returns false for tasks less than concurrentExecLimit"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' def keyName = "${properties.namespace}:$taskDefName" as String TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName, workflowTask: new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: 2))) redisTemplate.opsForSet().add(keyName, taskId) when: def retVal = dao.exceedsLimit(task) then: !retVal } def "verify exceedsLimit returns false for taskId already in the set but more than concurrentExecLimit"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' def keyName = "${properties.namespace}:$taskDefName" as String TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName, workflowTask: new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: 2))) redisTemplate.opsForSet().add(keyName, taskId) // add the id of the task passed as argument to exceedsLimit redisTemplate.opsForSet().add(keyName, 'taskId2') when: def retVal = dao.exceedsLimit(task) then: !retVal } def "verify exceedsLimit returns true for a new taskId more than concurrentExecLimit"() { given: def taskId = 'task1' def taskDefName = 'task_def_name1' def keyName = "${properties.namespace}:$taskDefName" as String TaskModel task = new TaskModel(taskId: taskId, taskDefName: taskDefName, workflowTask: new WorkflowTask(taskDefinition: new TaskDef(concurrentExecLimit: 2))) // add task ids different from the id of the task passed to exceedsLimit redisTemplate.opsForSet().add(keyName, 'taskId2') redisTemplate.opsForSet().add(keyName, 'taskId3') when: def retVal = dao.exceedsLimit(task) then: retVal } def "verify createKeyName ignores namespace if its not present"() { given: def dao = new RedisConcurrentExecutionLimitDAO(null, conductorProperties) when: def keyName = dao.createKeyName('taskdefname') then: keyName == expectedKeyName where: conductorProperties << [new RedisConcurrentExecutionLimitProperties(), new RedisConcurrentExecutionLimitProperties(namespace: null), new RedisConcurrentExecutionLimitProperties(namespace: 'test')] expectedKeyName << ['conductor:taskdefname', 'taskdefname', 'test:taskdefname'] } } ================================================ FILE: redis-lock/build.gradle ================================================ dependencies { implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "org.apache.commons:commons-lang3" implementation "org.redisson:redisson:${revRedisson}" testImplementation "com.github.kstyrc:embedded-redis:${revEmbeddedRedis}" } ================================================ FILE: redis-lock/src/main/java/com/netflix/conductor/redislock/config/RedisLockConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redislock.config; import java.util.Arrays; import org.redisson.Redisson; import org.redisson.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.sync.Lock; import com.netflix.conductor.redislock.config.RedisLockProperties.REDIS_SERVER_TYPE; import com.netflix.conductor.redislock.lock.RedisLock; @Configuration @EnableConfigurationProperties(RedisLockProperties.class) @ConditionalOnProperty(name = "conductor.workflow-execution-lock.type", havingValue = "redis") public class RedisLockConfiguration { private static final Logger LOGGER = LoggerFactory.getLogger(RedisLockConfiguration.class); @Bean public Redisson getRedisson(RedisLockProperties properties) { RedisLockProperties.REDIS_SERVER_TYPE redisServerType; try { redisServerType = properties.getServerType(); } catch (IllegalArgumentException ie) { final String message = "Invalid Redis server type: " + properties.getServerType() + ", supported values are: " + Arrays.toString(REDIS_SERVER_TYPE.values()); LOGGER.error(message); throw new RuntimeException(message, ie); } String redisServerAddress = properties.getServerAddress(); String redisServerPassword = properties.getServerPassword(); String masterName = properties.getServerMasterName(); Config redisConfig = new Config(); if (properties.getNumNettyThreads() != null && properties.getNumNettyThreads() > 0) { redisConfig.setNettyThreads(properties.getNumNettyThreads()); } int connectionTimeout = 10000; switch (redisServerType) { case SINGLE: LOGGER.info("Setting up Redis Single Server for RedisLockConfiguration"); redisConfig .useSingleServer() .setAddress(redisServerAddress) .setPassword(redisServerPassword) .setTimeout(connectionTimeout); break; case CLUSTER: LOGGER.info("Setting up Redis Cluster for RedisLockConfiguration"); redisConfig .useClusterServers() .setScanInterval(2000) // cluster state scan interval in milliseconds .addNodeAddress(redisServerAddress.split(",")) .setPassword(redisServerPassword) .setTimeout(connectionTimeout) .setSlaveConnectionMinimumIdleSize( properties.getClusterReplicaConnectionMinIdleSize()) .setSlaveConnectionPoolSize( properties.getClusterReplicaConnectionPoolSize()) .setMasterConnectionMinimumIdleSize( properties.getClusterPrimaryConnectionMinIdleSize()) .setMasterConnectionPoolSize( properties.getClusterPrimaryConnectionPoolSize()); break; case SENTINEL: LOGGER.info("Setting up Redis Sentinel Servers for RedisLockConfiguration"); redisConfig .useSentinelServers() .setScanInterval(2000) .setMasterName(masterName) .addSentinelAddress(redisServerAddress) .setPassword(redisServerPassword) .setTimeout(connectionTimeout); break; } return (Redisson) Redisson.create(redisConfig); } @Bean public Lock provideLock(Redisson redisson, RedisLockProperties properties) { return new RedisLock(redisson, properties); } } ================================================ FILE: redis-lock/src/main/java/com/netflix/conductor/redislock/config/RedisLockProperties.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redislock.config; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("conductor.redis-lock") public class RedisLockProperties { /** The redis server configuration to be used. */ private REDIS_SERVER_TYPE serverType = REDIS_SERVER_TYPE.SINGLE; /** The address of the redis server following format -- host:port */ private String serverAddress = "redis://127.0.0.1:6379"; /** The password for redis authentication */ private String serverPassword = null; /** The master server name used by Redis Sentinel servers and master change monitoring task */ private String serverMasterName = "master"; /** The namespace to use to prepend keys used for locking in redis */ private String namespace = ""; /** The number of natty threads to use */ private Integer numNettyThreads; /** If using Cluster Mode, you can use this to set num of min idle connections for replica */ private int clusterReplicaConnectionMinIdleSize = 24; /** If using Cluster Mode, you can use this to set num of min idle connections for replica */ private int clusterReplicaConnectionPoolSize = 64; /** If using Cluster Mode, you can use this to set num of min idle connections for replica */ private int clusterPrimaryConnectionMinIdleSize = 24; /** If using Cluster Mode, you can use this to set num of min idle connections for replica */ private int clusterPrimaryConnectionPoolSize = 64; /** * Enable to otionally continue without a lock to not block executions until the locking service * becomes available */ private boolean ignoreLockingExceptions = false; public REDIS_SERVER_TYPE getServerType() { return serverType; } public void setServerType(REDIS_SERVER_TYPE serverType) { this.serverType = serverType; } public String getServerAddress() { return serverAddress; } public void setServerAddress(String serverAddress) { this.serverAddress = serverAddress; } public String getServerPassword() { return serverPassword; } public void setServerPassword(String serverPassword) { this.serverPassword = serverPassword; } public String getServerMasterName() { return serverMasterName; } public void setServerMasterName(String serverMasterName) { this.serverMasterName = serverMasterName; } public String getNamespace() { return namespace; } public void setNamespace(String namespace) { this.namespace = namespace; } public boolean isIgnoreLockingExceptions() { return ignoreLockingExceptions; } public void setIgnoreLockingExceptions(boolean ignoreLockingExceptions) { this.ignoreLockingExceptions = ignoreLockingExceptions; } public Integer getNumNettyThreads() { return numNettyThreads; } public void setNumNettyThreads(Integer numNettyThreads) { this.numNettyThreads = numNettyThreads; } public Integer getClusterReplicaConnectionMinIdleSize() { return clusterReplicaConnectionMinIdleSize; } public void setClusterReplicaConnectionMinIdleSize( Integer clusterReplicaConnectionMinIdleSize) { this.clusterReplicaConnectionMinIdleSize = clusterReplicaConnectionMinIdleSize; } public Integer getClusterReplicaConnectionPoolSize() { return clusterReplicaConnectionPoolSize; } public void setClusterReplicaConnectionPoolSize(Integer clusterReplicaConnectionPoolSize) { this.clusterReplicaConnectionPoolSize = clusterReplicaConnectionPoolSize; } public Integer getClusterPrimaryConnectionMinIdleSize() { return clusterPrimaryConnectionMinIdleSize; } public void setClusterPrimaryConnectionMinIdleSize( Integer clusterPrimaryConnectionMinIdleSize) { this.clusterPrimaryConnectionMinIdleSize = clusterPrimaryConnectionMinIdleSize; } public Integer getClusterPrimaryConnectionPoolSize() { return clusterPrimaryConnectionPoolSize; } public void setClusterPrimaryConnectionPoolSize(Integer clusterPrimaryConnectionPoolSize) { this.clusterPrimaryConnectionPoolSize = clusterPrimaryConnectionPoolSize; } public enum REDIS_SERVER_TYPE { SINGLE, CLUSTER, SENTINEL } } ================================================ FILE: redis-lock/src/main/java/com/netflix/conductor/redislock/lock/RedisLock.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redislock.lock; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.redisson.Redisson; import org.redisson.api.RLock; import org.redisson.api.RedissonClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.core.sync.Lock; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.redislock.config.RedisLockProperties; public class RedisLock implements Lock { private static final Logger LOGGER = LoggerFactory.getLogger(RedisLock.class); private final RedisLockProperties properties; private final RedissonClient redisson; private static String LOCK_NAMESPACE = ""; public RedisLock(Redisson redisson, RedisLockProperties properties) { this.properties = properties; this.redisson = redisson; LOCK_NAMESPACE = properties.getNamespace(); } @Override public void acquireLock(String lockId) { RLock lock = redisson.getLock(parseLockId(lockId)); lock.lock(); } @Override public boolean acquireLock(String lockId, long timeToTry, TimeUnit unit) { RLock lock = redisson.getLock(parseLockId(lockId)); try { return lock.tryLock(timeToTry, unit); } catch (Exception e) { return handleAcquireLockFailure(lockId, e); } } /** * @param lockId resource to lock on * @param timeToTry blocks up to timeToTry duration in attempt to acquire the lock * @param leaseTime Lock lease expiration duration. Redisson default is -1, meaning it holds the * lock until explicitly unlocked. * @param unit time unit * @return */ @Override public boolean acquireLock(String lockId, long timeToTry, long leaseTime, TimeUnit unit) { RLock lock = redisson.getLock(parseLockId(lockId)); try { return lock.tryLock(timeToTry, leaseTime, unit); } catch (Exception e) { return handleAcquireLockFailure(lockId, e); } } @Override public void releaseLock(String lockId) { RLock lock = redisson.getLock(parseLockId(lockId)); try { lock.unlock(); } catch (IllegalMonitorStateException e) { // Releasing a lock twice using Redisson can cause this exception, which can be ignored. } } @Override public void deleteLock(String lockId) { // Noop for Redlock algorithm as releaseLock / unlock deletes it. } private String parseLockId(String lockId) { if (StringUtils.isEmpty(lockId)) { throw new IllegalArgumentException("lockId cannot be NULL or empty: lockId=" + lockId); } return LOCK_NAMESPACE + "." + lockId; } private boolean handleAcquireLockFailure(String lockId, Exception e) { LOGGER.error("Failed to acquireLock for lockId: {}", lockId, e); Monitors.recordAcquireLockFailure(e.getClass().getName()); // A Valid failure to acquire lock when another thread has acquired it returns false. // However, when an exception is thrown while acquiring lock, due to connection or others // issues, // we can optionally continue without a "lock" to not block executions until Locking service // is available. return properties.isIgnoreLockingExceptions(); } } ================================================ FILE: redis-lock/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.redis-lock.server-type", "defaultValue": "SINGLE" } ], "hints": [ { "name": "conductor.workflow-execution-lock.type", "values": [ { "value": "redis", "description": "Use the redis-lock implementation as the lock provider." } ] }, { "name": "conductor.redis-lock.server-type", "providers": [ { "name": "handle-as", "parameters": { "target": "java.lang.Enum" } } ] } ] } ================================================ FILE: redis-lock/src/test/java/com/netflix/conductor/redis/lock/RedisLockTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.lock; import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.redisson.Redisson; import org.redisson.api.RLock; import org.redisson.api.RedissonClient; import org.redisson.config.Config; import com.netflix.conductor.redislock.config.RedisLockProperties; import com.netflix.conductor.redislock.config.RedisLockProperties.REDIS_SERVER_TYPE; import com.netflix.conductor.redislock.lock.RedisLock; import redis.embedded.RedisServer; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class RedisLockTest { private static RedisLock redisLock; private static Config config; private static RedissonClient redisson; private static RedisServer redisServer = null; @BeforeClass public static void setUp() throws Exception { String testServerAddress = "redis://127.0.0.1:6371"; redisServer = new RedisServer(6371); if (redisServer.isActive()) { redisServer.stop(); } redisServer.start(); RedisLockProperties properties = mock(RedisLockProperties.class); when(properties.getServerType()).thenReturn(REDIS_SERVER_TYPE.SINGLE); when(properties.getServerAddress()).thenReturn(testServerAddress); when(properties.getServerMasterName()).thenReturn("master"); when(properties.getNamespace()).thenReturn(""); when(properties.isIgnoreLockingExceptions()).thenReturn(false); Config redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(testServerAddress).setTimeout(10000); redisLock = new RedisLock((Redisson) Redisson.create(redissonConfig), properties); // Create another instance of redisson for tests. RedisLockTest.config = new Config(); RedisLockTest.config.useSingleServer().setAddress(testServerAddress).setTimeout(10000); redisson = Redisson.create(RedisLockTest.config); } @AfterClass public static void tearDown() { redisServer.stop(); } @Test public void testLocking() { redisson.getKeys().flushall(); String lockId = "abcd-1234"; assertTrue(redisLock.acquireLock(lockId, 1000, 1000, TimeUnit.MILLISECONDS)); } @Test public void testLockExpiration() throws InterruptedException { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 1000, TimeUnit.MILLISECONDS); assertTrue(isLocked); Thread.sleep(2000); RLock lock = redisson.getLock(lockId); assertFalse(lock.isLocked()); } @Test public void testLockReentry() throws InterruptedException { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 60000, TimeUnit.MILLISECONDS); assertTrue(isLocked); Thread.sleep(1000); // get the lock back isLocked = redisLock.acquireLock(lockId, 1000, 1000, TimeUnit.MILLISECONDS); assertTrue(isLocked); RLock lock = redisson.getLock(lockId); assertTrue(isLocked); } @Test public void testReleaseLock() { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 10000, TimeUnit.MILLISECONDS); assertTrue(isLocked); redisLock.releaseLock(lockId); RLock lock = redisson.getLock(lockId); assertFalse(lock.isLocked()); } @Test public void testLockReleaseAndAcquire() throws InterruptedException { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 10000, TimeUnit.MILLISECONDS); assertTrue(isLocked); redisLock.releaseLock(lockId); Worker worker1 = new Worker(redisLock, lockId); worker1.start(); worker1.join(); assertTrue(worker1.isLocked); } @Test public void testLockingDuplicateThreads() throws InterruptedException { redisson.getKeys().flushall(); String lockId = "abcd-1234"; Worker worker1 = new Worker(redisLock, lockId); Worker worker2 = new Worker(redisLock, lockId); worker1.start(); worker2.start(); worker1.join(); worker2.join(); // Ensure only one of them had got the lock. assertFalse(worker1.isLocked && worker2.isLocked); assertTrue(worker1.isLocked || worker2.isLocked); } @Test public void testDuplicateLockAcquireFailure() throws InterruptedException { redisson.getKeys().flushall(); String lockId = "abcd-1234"; Worker worker1 = new Worker(redisLock, lockId, 100L, 60000L); worker1.start(); worker1.join(); boolean isLocked = redisLock.acquireLock(lockId, 500L, 1000L, TimeUnit.MILLISECONDS); // Ensure only one of them had got the lock. assertFalse(isLocked); assertTrue(worker1.isLocked); } @Test public void testReacquireLostKey() { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 10000, TimeUnit.MILLISECONDS); assertTrue(isLocked); // Delete key from the cluster to reacquire // Simulating the case when cluster goes down and possibly loses some keys. redisson.getKeys().flushall(); isLocked = redisLock.acquireLock(lockId, 100, 10000, TimeUnit.MILLISECONDS); assertTrue(isLocked); } @Test public void testReleaseLockTwice() { redisson.getKeys().flushall(); String lockId = "abcd-1234"; boolean isLocked = redisLock.acquireLock(lockId, 1000, 10000, TimeUnit.MILLISECONDS); assertTrue(isLocked); redisLock.releaseLock(lockId); redisLock.releaseLock(lockId); } private static class Worker extends Thread { private final RedisLock lock; private final String lockID; boolean isLocked; private Long timeToTry = 50L; private Long leaseTime = 1000L; Worker(RedisLock lock, String lockID) { super("TestWorker-" + lockID); this.lock = lock; this.lockID = lockID; } Worker(RedisLock lock, String lockID, Long timeToTry, Long leaseTime) { super("TestWorker-" + lockID); this.lock = lock; this.lockID = lockID; this.timeToTry = timeToTry; this.leaseTime = leaseTime; } @Override public void run() { isLocked = lock.acquireLock(lockID, timeToTry, leaseTime, TimeUnit.MILLISECONDS); } } } ================================================ FILE: redis-persistence/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' implementation "redis.clients:jedis:${revJedis}" implementation "com.netflix.dyno-queues:dyno-queues-redis:${revDynoQueues}" implementation('com.thoughtworks.xstream:xstream:1.4.20') //In memory implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}" testImplementation project(':conductor-core').sourceSets.test.output testImplementation project(':conductor-common').sourceSets.test.output } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/AnyRedisCondition.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; public class AnyRedisCondition extends AnyNestedCondition { public AnyRedisCondition() { super(ConfigurationPhase.PARSE_CONFIGURATION); } @ConditionalOnProperty(name = "conductor.db.type", havingValue = "dynomite") static class DynomiteClusterCondition {} @ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory") static class InMemoryRedisCondition {} @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster") static class RedisClusterConfiguration {} @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel") static class RedisSentinelConfiguration {} @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone") static class RedisStandaloneConfiguration {} } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/DynomiteClusterConfiguration.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; import com.netflix.dyno.jedis.DynoJedisClient; import redis.clients.jedis.commands.JedisCommands; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "dynomite") public class DynomiteClusterConfiguration extends JedisCommandsConfigurer { protected JedisCommands createJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { ConnectionPoolConfigurationImpl connectionPoolConfiguration = new ConnectionPoolConfigurationImpl(properties.getClusterName()) .withTokenSupplier(tokenMapSupplier) .setLocalRack(properties.getAvailabilityZone()) .setLocalDataCenter(properties.getDataCenterRegion()) .setSocketTimeout(0) .setConnectTimeout(0) .setMaxConnsPerHost(properties.getMaxConnectionsPerHost()) .setMaxTimeoutWhenExhausted( (int) properties.getMaxTimeoutWhenExhausted().toMillis()) .setRetryPolicyFactory(properties.getConnectionRetryPolicy()); return new DynoJedisClient.Builder() .withHostSupplier(hostSupplier) .withApplicationName(conductorProperties.getAppId()) .withDynomiteClusterName(properties.getClusterName()) .withCPConfig(connectionPoolConfiguration) .build(); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/InMemoryRedisConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.redis.dynoqueue.LocalhostHostSupplier; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.dyno.connectionpool.HostSupplier; import static com.netflix.conductor.redis.config.RedisCommonConfiguration.DEFAULT_CLIENT_INJECTION_NAME; import static com.netflix.conductor.redis.config.RedisCommonConfiguration.READ_CLIENT_INJECTION_NAME; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory") public class InMemoryRedisConfiguration { @Bean public HostSupplier hostSupplier(RedisProperties properties) { return new LocalhostHostSupplier(properties); } @Bean(name = {DEFAULT_CLIENT_INJECTION_NAME, READ_CLIENT_INJECTION_NAME}) public JedisMock jedisMock() { return new JedisMock(); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/JedisCommandsConfigurer.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import org.springframework.context.annotation.Bean; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import redis.clients.jedis.commands.JedisCommands; import static com.netflix.conductor.redis.config.RedisCommonConfiguration.DEFAULT_CLIENT_INJECTION_NAME; import static com.netflix.conductor.redis.config.RedisCommonConfiguration.READ_CLIENT_INJECTION_NAME; abstract class JedisCommandsConfigurer { @Bean public HostSupplier hostSupplier(RedisProperties properties) { return new ConfigurationHostSupplier(properties); } @Bean(name = DEFAULT_CLIENT_INJECTION_NAME) public JedisCommands jedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { return createJedisCommands(properties, conductorProperties, hostSupplier, tokenMapSupplier); } @Bean(name = READ_CLIENT_INJECTION_NAME) public JedisCommands readJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { return createJedisCommands(properties, conductorProperties, hostSupplier, tokenMapSupplier); } protected abstract JedisCommands createJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier); } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/RedisClusterConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.jedis.JedisCluster; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import redis.clients.jedis.HostAndPort; import redis.clients.jedis.Protocol; import redis.clients.jedis.commands.JedisCommands; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster") public class RedisClusterConfiguration extends JedisCommandsConfigurer { private static final Logger log = LoggerFactory.getLogger(JedisCommandsConfigurer.class); // Same as redis.clients.jedis.BinaryJedisCluster protected static final int DEFAULT_MAX_ATTEMPTS = 5; @Override protected JedisCommands createJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>(); genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost()); Set hosts = hostSupplier.getHosts().stream() .map(h -> new HostAndPort(h.getHostName(), h.getPort())) .collect(Collectors.toSet()); String password = getPassword(hostSupplier.getHosts()); if (password != null) { log.info("Connecting to Redis Cluster with AUTH"); return new JedisCluster( new redis.clients.jedis.JedisCluster( hosts, Protocol.DEFAULT_TIMEOUT, Protocol.DEFAULT_TIMEOUT, DEFAULT_MAX_ATTEMPTS, password, genericObjectPoolConfig)); } else { return new JedisCluster( new redis.clients.jedis.JedisCluster(hosts, genericObjectPoolConfig)); } } private String getPassword(List hosts) { return hosts.isEmpty() ? null : hosts.get(0).getPassword(); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/RedisCommonConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import java.util.ArrayList; import java.util.List; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.redis.dynoqueue.RedisQueuesShardingStrategyProvider; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.lb.HostToken; import com.netflix.dyno.connectionpool.impl.utils.CollectionUtils; import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; import com.netflix.dyno.queues.shard.DynoShardSupplier; import com.google.inject.ProvisionException; import redis.clients.jedis.commands.JedisCommands; @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(RedisProperties.class) @Conditional(AnyRedisCondition.class) public class RedisCommonConfiguration { public static final String DEFAULT_CLIENT_INJECTION_NAME = "DefaultJedisCommands"; public static final String READ_CLIENT_INJECTION_NAME = "ReadJedisCommands"; private static final Logger LOGGER = LoggerFactory.getLogger(RedisCommonConfiguration.class); @Bean public ShardSupplier shardSupplier(HostSupplier hostSupplier, RedisProperties properties) { if (properties.getAvailabilityZone() == null) { throw new ProvisionException( "Availability zone is not defined. Ensure Configuration.getAvailabilityZone() returns a non-null " + "and non-empty value."); } String localDC = properties.getAvailabilityZone().replaceAll(properties.getDataCenterRegion(), ""); return new DynoShardSupplier(hostSupplier, properties.getDataCenterRegion(), localDC); } @Bean public TokenMapSupplier tokenMapSupplier() { final List hostTokens = new ArrayList<>(); return new TokenMapSupplier() { @Override public List getTokens(Set activeHosts) { long i = activeHosts.size(); for (Host host : activeHosts) { HostToken hostToken = new HostToken(i, host); hostTokens.add(hostToken); i--; } return hostTokens; } @Override public HostToken getTokenForHost(Host host, Set activeHosts) { return CollectionUtils.find( hostTokens, token -> token.getHost().compareTo(host) == 0); } }; } @Bean public ShardingStrategy shardingStrategy( ShardSupplier shardSupplier, RedisProperties properties) { return new RedisQueuesShardingStrategyProvider(shardSupplier, properties).get(); } @Bean public RedisQueues redisQueues( @Qualifier(DEFAULT_CLIENT_INJECTION_NAME) JedisCommands jedisCommands, @Qualifier(READ_CLIENT_INJECTION_NAME) JedisCommands jedisCommandsRead, ShardSupplier shardSupplier, RedisProperties properties, ShardingStrategy shardingStrategy) { RedisQueues queues = new RedisQueues( jedisCommands, jedisCommandsRead, properties.getQueuePrefix(), shardSupplier, 60_000, 60_000, shardingStrategy); LOGGER.info("DynoQueueDAO initialized with prefix " + properties.getQueuePrefix() + "!"); return queues; } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/RedisProperties.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import java.time.Duration; import java.time.temporal.ChronoUnit; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.dynoqueue.RedisQueuesShardingStrategyProvider; import com.netflix.dyno.connectionpool.RetryPolicy.RetryPolicyFactory; import com.netflix.dyno.connectionpool.impl.RetryNTimes; import com.netflix.dyno.connectionpool.impl.RunOnce; @ConfigurationProperties("conductor.redis") public class RedisProperties { private final ConductorProperties conductorProperties; @Autowired public RedisProperties(ConductorProperties conductorProperties) { this.conductorProperties = conductorProperties; } /** * Data center region. If hosting on Amazon the value is something like us-east-1, us-west-2 * etc. */ private String dataCenterRegion = "us-east-1"; /** * Local rack / availability zone. For AWS deployments, the value is something like us-east-1a, * etc. */ private String availabilityZone = "us-east-1c"; /** The name of the redis / dynomite cluster */ private String clusterName = ""; /** Dynomite Cluster details. Format is host:port:rack separated by semicolon */ private String hosts = null; /** The prefix used to prepend workflow data in redis */ private String workflowNamespacePrefix = null; /** The prefix used to prepend keys for queues in redis */ private String queueNamespacePrefix = null; /** * The domain name to be used in the key prefix for logical separation of workflow data and * queues in a shared redis setup */ private String keyspaceDomain = null; /** * The maximum number of connections that can be managed by the connection pool on a given * instance */ private int maxConnectionsPerHost = 10; /** * The maximum amount of time to wait for a connection to become available from the connection * pool */ private Duration maxTimeoutWhenExhausted = Duration.ofMillis(800); /** The maximum retry attempts to use with this connection pool */ private int maxRetryAttempts = 0; /** The read connection port to be used for connecting to dyno-queues */ private int queuesNonQuorumPort = 22122; /** The sharding strategy to be used for the dyno queue configuration */ private String queueShardingStrategy = RedisQueuesShardingStrategyProvider.ROUND_ROBIN_STRATEGY; /** The time in seconds after which the in-memory task definitions cache will be refreshed */ @DurationUnit(ChronoUnit.SECONDS) private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60); /** The time to live in seconds for which the event execution will be persisted */ @DurationUnit(ChronoUnit.SECONDS) private Duration eventExecutionPersistenceTTL = Duration.ofSeconds(60); // Maximum number of idle connections to be maintained private int maxIdleConnections = 8; // Minimum number of idle connections to be maintained private int minIdleConnections = 5; private long minEvictableIdleTimeMillis = 1800000; private long timeBetweenEvictionRunsMillis = -1L; private boolean testWhileIdle = false; private int numTestsPerEvictionRun = 3; private int database = 0; private String username = null; public int getNumTestsPerEvictionRun() { return numTestsPerEvictionRun; } public void setNumTestsPerEvictionRun(int numTestsPerEvictionRun) { this.numTestsPerEvictionRun = numTestsPerEvictionRun; } public boolean isTestWhileIdle() { return testWhileIdle; } public void setTestWhileIdle(boolean testWhileIdle) { this.testWhileIdle = testWhileIdle; } public long getMinEvictableIdleTimeMillis() { return minEvictableIdleTimeMillis; } public void setMinEvictableIdleTimeMillis(long minEvictableIdleTimeMillis) { this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; } public long getTimeBetweenEvictionRunsMillis() { return timeBetweenEvictionRunsMillis; } public void setTimeBetweenEvictionRunsMillis(long timeBetweenEvictionRunsMillis) { this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; } public int getMinIdleConnections() { return minIdleConnections; } public void setMinIdleConnections(int minIdleConnections) { this.minIdleConnections = minIdleConnections; } public int getMaxIdleConnections() { return maxIdleConnections; } public void setMaxIdleConnections(int maxIdleConnections) { this.maxIdleConnections = maxIdleConnections; } public String getDataCenterRegion() { return dataCenterRegion; } public void setDataCenterRegion(String dataCenterRegion) { this.dataCenterRegion = dataCenterRegion; } public String getAvailabilityZone() { return availabilityZone; } public void setAvailabilityZone(String availabilityZone) { this.availabilityZone = availabilityZone; } public String getClusterName() { return clusterName; } public void setClusterName(String clusterName) { this.clusterName = clusterName; } public String getHosts() { return hosts; } public void setHosts(String hosts) { this.hosts = hosts; } public String getWorkflowNamespacePrefix() { return workflowNamespacePrefix; } public void setWorkflowNamespacePrefix(String workflowNamespacePrefix) { this.workflowNamespacePrefix = workflowNamespacePrefix; } public String getQueueNamespacePrefix() { return queueNamespacePrefix; } public void setQueueNamespacePrefix(String queueNamespacePrefix) { this.queueNamespacePrefix = queueNamespacePrefix; } public String getKeyspaceDomain() { return keyspaceDomain; } public void setKeyspaceDomain(String keyspaceDomain) { this.keyspaceDomain = keyspaceDomain; } public int getMaxConnectionsPerHost() { return maxConnectionsPerHost; } public void setMaxConnectionsPerHost(int maxConnectionsPerHost) { this.maxConnectionsPerHost = maxConnectionsPerHost; } public Duration getMaxTimeoutWhenExhausted() { return maxTimeoutWhenExhausted; } public void setMaxTimeoutWhenExhausted(Duration maxTimeoutWhenExhausted) { this.maxTimeoutWhenExhausted = maxTimeoutWhenExhausted; } public int getMaxRetryAttempts() { return maxRetryAttempts; } public void setMaxRetryAttempts(int maxRetryAttempts) { this.maxRetryAttempts = maxRetryAttempts; } public int getQueuesNonQuorumPort() { return queuesNonQuorumPort; } public void setQueuesNonQuorumPort(int queuesNonQuorumPort) { this.queuesNonQuorumPort = queuesNonQuorumPort; } public String getQueueShardingStrategy() { return queueShardingStrategy; } public void setQueueShardingStrategy(String queueShardingStrategy) { this.queueShardingStrategy = queueShardingStrategy; } public Duration getTaskDefCacheRefreshInterval() { return taskDefCacheRefreshInterval; } public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) { this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval; } public Duration getEventExecutionPersistenceTTL() { return eventExecutionPersistenceTTL; } public void setEventExecutionPersistenceTTL(Duration eventExecutionPersistenceTTL) { this.eventExecutionPersistenceTTL = eventExecutionPersistenceTTL; } public String getQueuePrefix() { String prefix = getQueueNamespacePrefix() + "." + conductorProperties.getStack(); if (getKeyspaceDomain() != null) { prefix = prefix + "." + getKeyspaceDomain(); } return prefix; } public RetryPolicyFactory getConnectionRetryPolicy() { if (getMaxRetryAttempts() == 0) { return RunOnce::new; } else { return () -> new RetryNTimes(maxRetryAttempts, false); } } public int getDatabase() { return database; } public void setDatabase(int database) { this.database = database; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/RedisSentinelConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.jedis.JedisSentinel; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import redis.clients.jedis.JedisSentinelPool; import redis.clients.jedis.Protocol; import redis.clients.jedis.commands.JedisCommands; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel") public class RedisSentinelConfiguration extends JedisCommandsConfigurer { private static final Logger log = LoggerFactory.getLogger(RedisSentinelConfiguration.class); @Override protected JedisCommands createJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>(); genericObjectPoolConfig.setMinIdle(properties.getMinIdleConnections()); genericObjectPoolConfig.setMaxIdle(properties.getMaxIdleConnections()); genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost()); genericObjectPoolConfig.setTestWhileIdle(properties.isTestWhileIdle()); genericObjectPoolConfig.setMinEvictableIdleTimeMillis( properties.getMinEvictableIdleTimeMillis()); genericObjectPoolConfig.setTimeBetweenEvictionRunsMillis( properties.getTimeBetweenEvictionRunsMillis()); genericObjectPoolConfig.setNumTestsPerEvictionRun(properties.getNumTestsPerEvictionRun()); log.info( "Starting conductor server using redis_sentinel and cluster " + properties.getClusterName()); Set sentinels = new HashSet<>(); for (Host host : hostSupplier.getHosts()) { sentinels.add(host.getHostName() + ":" + host.getPort()); } // We use the password of the first sentinel host as password and sentinelPassword String password = getPassword(hostSupplier.getHosts()); if (properties.getUsername() != null && password != null) { return new JedisSentinel( new JedisSentinelPool( properties.getClusterName(), sentinels, genericObjectPoolConfig, Protocol.DEFAULT_TIMEOUT, Protocol.DEFAULT_TIMEOUT, properties.getUsername(), password, properties.getDatabase(), null, Protocol.DEFAULT_TIMEOUT, Protocol.DEFAULT_TIMEOUT, properties.getUsername(), password, null)); } else if (password != null) { return new JedisSentinel( new JedisSentinelPool( properties.getClusterName(), sentinels, genericObjectPoolConfig, Protocol.DEFAULT_TIMEOUT, Protocol.DEFAULT_TIMEOUT, password, properties.getDatabase(), null, Protocol.DEFAULT_TIMEOUT, Protocol.DEFAULT_TIMEOUT, password, null)); } else { return new JedisSentinel( new JedisSentinelPool( properties.getClusterName(), sentinels, genericObjectPoolConfig)); } } private String getPassword(List hosts) { return hosts.isEmpty() ? null : hosts.get(0).getPassword(); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/config/RedisStandaloneConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.jedis.JedisStandalone; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.Protocol; import redis.clients.jedis.commands.JedisCommands; @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone") public class RedisStandaloneConfiguration extends JedisCommandsConfigurer { private static final Logger log = LoggerFactory.getLogger(RedisSentinelConfiguration.class); @Override protected JedisCommands createJedisCommands( RedisProperties properties, ConductorProperties conductorProperties, HostSupplier hostSupplier, TokenMapSupplier tokenMapSupplier) { JedisPoolConfig config = new JedisPoolConfig(); config.setMinIdle(2); config.setMaxTotal(properties.getMaxConnectionsPerHost()); log.info("Starting conductor server using redis_standalone."); Host host = hostSupplier.getHosts().get(0); return new JedisStandalone(getJedisPool(config, host, properties)); } private JedisPool getJedisPool(JedisPoolConfig config, Host host, RedisProperties properties) { if (properties.getUsername() != null && host.getPassword() != null) { log.info("Connecting to Redis Standalone with AUTH"); return new JedisPool( config, host.getHostName(), host.getPort(), Protocol.DEFAULT_TIMEOUT, properties.getUsername(), host.getPassword(), properties.getDatabase()); } else if (host.getPassword() != null) { log.info("Connecting to Redis Standalone with AUTH"); return new JedisPool( config, host.getHostName(), host.getPort(), Protocol.DEFAULT_TIMEOUT, host.getPassword(), properties.getDatabase()); } else { return new JedisPool(config, host.getHostName(), host.getPort()); } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/BaseDynoDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.io.IOException; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; public class BaseDynoDAO { private static final String NAMESPACE_SEP = "."; private static final String DAO_NAME = "redis"; private final String domain; private final RedisProperties properties; private final ConductorProperties conductorProperties; protected JedisProxy jedisProxy; protected ObjectMapper objectMapper; protected BaseDynoDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { this.jedisProxy = jedisProxy; this.objectMapper = objectMapper; this.conductorProperties = conductorProperties; this.properties = properties; this.domain = properties.getKeyspaceDomain(); } String nsKey(String... nsValues) { String rootNamespace = properties.getWorkflowNamespacePrefix(); StringBuilder namespacedKey = new StringBuilder(); if (StringUtils.isNotBlank(rootNamespace)) { namespacedKey.append(rootNamespace).append(NAMESPACE_SEP); } String stack = conductorProperties.getStack(); if (StringUtils.isNotBlank(stack)) { namespacedKey.append(stack).append(NAMESPACE_SEP); } if (StringUtils.isNotBlank(domain)) { namespacedKey.append(domain).append(NAMESPACE_SEP); } for (String nsValue : nsValues) { namespacedKey.append(nsValue).append(NAMESPACE_SEP); } return StringUtils.removeEnd(namespacedKey.toString(), NAMESPACE_SEP); } public JedisProxy getDyno() { return jedisProxy; } String toJson(Object value) { try { return objectMapper.writeValueAsString(value); } catch (JsonProcessingException e) { throw new RuntimeException(e); } } T readValue(String json, Class clazz) { try { return objectMapper.readValue(json, clazz); } catch (IOException e) { throw new RuntimeException(e); } } void recordRedisDaoRequests(String action) { recordRedisDaoRequests(action, "n/a", "n/a"); } void recordRedisDaoRequests(String action, String taskType, String workflowType) { Monitors.recordDaoRequests(DAO_NAME, action, taskType, workflowType); } void recordRedisDaoEventRequests(String action, String event) { Monitors.recordDaoEventRequests(DAO_NAME, action, event); } void recordRedisDaoPayloadSize(String action, int size, String taskType, String workflowType) { Monitors.recordDaoPayloadSize( DAO_NAME, action, StringUtils.defaultIfBlank(taskType, ""), StringUtils.defaultIfBlank(workflowType, ""), size); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/DynoQueueDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.RedisQueues; @Component @Conditional(AnyRedisCondition.class) public class DynoQueueDAO implements QueueDAO { private final RedisQueues queues; public DynoQueueDAO(RedisQueues queues) { this.queues = queues; } @Override public void push(String queueName, String id, long offsetTimeInSecond) { push(queueName, id, -1, offsetTimeInSecond); } @Override public void push(String queueName, String id, int priority, long offsetTimeInSecond) { Message msg = new Message(id, null); msg.setTimeout(offsetTimeInSecond, TimeUnit.SECONDS); if (priority >= 0 && priority <= 99) { msg.setPriority(priority); } queues.get(queueName).push(Collections.singletonList(msg)); } @Override public void push( String queueName, List messages) { List msgs = messages.stream() .map( msg -> { Message m = new Message(msg.getId(), msg.getPayload()); if (msg.getPriority() > 0) { m.setPriority(msg.getPriority()); } return m; }) .collect(Collectors.toList()); queues.get(queueName).push(msgs); } @Override public boolean pushIfNotExists(String queueName, String id, long offsetTimeInSecond) { return pushIfNotExists(queueName, id, -1, offsetTimeInSecond); } @Override public boolean pushIfNotExists( String queueName, String id, int priority, long offsetTimeInSecond) { DynoQueue queue = queues.get(queueName); if (queue.get(id) != null) { return false; } Message msg = new Message(id, null); if (priority >= 0 && priority <= 99) { msg.setPriority(priority); } msg.setTimeout(offsetTimeInSecond, TimeUnit.SECONDS); queue.push(Collections.singletonList(msg)); return true; } @Override public List pop(String queueName, int count, int timeout) { List msg = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msg.stream().map(Message::getId).collect(Collectors.toList()); } @Override public List pollMessages( String queueName, int count, int timeout) { List msgs = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msgs.stream() .map( msg -> new com.netflix.conductor.core.events.queue.Message( msg.getId(), msg.getPayload(), null, msg.getPriority())) .collect(Collectors.toList()); } @Override public void remove(String queueName, String messageId) { queues.get(queueName).remove(messageId); } @Override public int getSize(String queueName) { return (int) queues.get(queueName).size(); } @Override public boolean ack(String queueName, String messageId) { return queues.get(queueName).ack(messageId); } @Override public boolean setUnackTimeout(String queueName, String messageId, long timeout) { return queues.get(queueName).setUnackTimeout(messageId, timeout); } @Override public void flush(String queueName) { DynoQueue queue = queues.get(queueName); if (queue != null) { queue.clear(); } } @Override public Map queuesDetail() { return queues.queues().stream() .collect(Collectors.toMap(DynoQueue::getName, DynoQueue::size)); } @Override public Map>> queuesDetailVerbose() { return queues.queues().stream() .collect(Collectors.toMap(DynoQueue::getName, DynoQueue::shardSizes)); } public void processUnacks(String queueName) { queues.get(queueName).processUnacks(); } @Override public boolean resetOffsetTime(String queueName, String id) { DynoQueue queue = queues.get(queueName); return queue.setTimeout(id, 0); } @Override public boolean containsMessage(String queueName, String messageId) { DynoQueue queue = queues.get(queueName); Message message = queue.get(messageId); return Objects.nonNull(message); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.EventHandlerDAO; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; @Component @Conditional(AnyRedisCondition.class) public class RedisEventHandlerDAO extends BaseDynoDAO implements EventHandlerDAO { private static final Logger LOGGER = LoggerFactory.getLogger(RedisEventHandlerDAO.class); private static final String EVENT_HANDLERS = "EVENT_HANDLERS"; private static final String EVENT_HANDLERS_BY_EVENT = "EVENT_HANDLERS_BY_EVENT"; public RedisEventHandlerDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { super(jedisProxy, objectMapper, conductorProperties, properties); } @Override public void addEventHandler(EventHandler eventHandler) { Preconditions.checkNotNull(eventHandler.getName(), "Missing Name"); if (getEventHandler(eventHandler.getName()) != null) { throw new ConflictException( "EventHandler with name %s already exists!", eventHandler.getName()); } index(eventHandler); jedisProxy.hset(nsKey(EVENT_HANDLERS), eventHandler.getName(), toJson(eventHandler)); recordRedisDaoRequests("addEventHandler"); } @Override public void updateEventHandler(EventHandler eventHandler) { Preconditions.checkNotNull(eventHandler.getName(), "Missing Name"); EventHandler existing = getEventHandler(eventHandler.getName()); if (existing == null) { throw new NotFoundException( "EventHandler with name %s not found!", eventHandler.getName()); } if (!existing.getEvent().equals(eventHandler.getEvent())) { removeIndex(existing); } index(eventHandler); jedisProxy.hset(nsKey(EVENT_HANDLERS), eventHandler.getName(), toJson(eventHandler)); recordRedisDaoRequests("updateEventHandler"); } @Override public void removeEventHandler(String name) { EventHandler existing = getEventHandler(name); if (existing == null) { throw new NotFoundException("EventHandler with name %s not found!", name); } jedisProxy.hdel(nsKey(EVENT_HANDLERS), name); recordRedisDaoRequests("removeEventHandler"); removeIndex(existing); } @Override public List getAllEventHandlers() { Map all = jedisProxy.hgetAll(nsKey(EVENT_HANDLERS)); List handlers = new LinkedList<>(); all.forEach( (key, json) -> { EventHandler eventHandler = readValue(json, EventHandler.class); handlers.add(eventHandler); }); recordRedisDaoRequests("getAllEventHandlers"); return handlers; } private void index(EventHandler eventHandler) { String event = eventHandler.getEvent(); String key = nsKey(EVENT_HANDLERS_BY_EVENT, event); jedisProxy.sadd(key, eventHandler.getName()); } private void removeIndex(EventHandler eventHandler) { String event = eventHandler.getEvent(); String key = nsKey(EVENT_HANDLERS_BY_EVENT, event); jedisProxy.srem(key, eventHandler.getName()); } @Override public List getEventHandlersForEvent(String event, boolean activeOnly) { String key = nsKey(EVENT_HANDLERS_BY_EVENT, event); Set names = jedisProxy.smembers(key); List handlers = new LinkedList<>(); for (String name : names) { try { EventHandler eventHandler = getEventHandler(name); recordRedisDaoEventRequests("getEventHandler", event); if (eventHandler.getEvent().equals(event) && (!activeOnly || eventHandler.isActive())) { handlers.add(eventHandler); } } catch (NotFoundException nfe) { LOGGER.info("No matching event handler found for event: {}", event); throw nfe; } } return handlers; } private EventHandler getEventHandler(String name) { EventHandler eventHandler = null; String json; try { json = jedisProxy.hget(nsKey(EVENT_HANDLERS), name); } catch (Exception e) { throw new TransientException("Unable to get event handler named " + name, e); } if (json != null) { eventHandler = readValue(json, EventHandler.class); } return eventHandler; } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/RedisExecutionDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.dao.ConcurrentExecutionLimitDAO; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @Component @Conditional(AnyRedisCondition.class) public class RedisExecutionDAO extends BaseDynoDAO implements ExecutionDAO, ConcurrentExecutionLimitDAO { public static final Logger LOGGER = LoggerFactory.getLogger(RedisExecutionDAO.class); // Keys Families private static final String TASK_LIMIT_BUCKET = "TASK_LIMIT_BUCKET"; private static final String IN_PROGRESS_TASKS = "IN_PROGRESS_TASKS"; private static final String TASKS_IN_PROGRESS_STATUS = "TASKS_IN_PROGRESS_STATUS"; // Tasks which are in IN_PROGRESS status. private static final String WORKFLOW_TO_TASKS = "WORKFLOW_TO_TASKS"; private static final String SCHEDULED_TASKS = "SCHEDULED_TASKS"; private static final String TASK = "TASK"; private static final String WORKFLOW = "WORKFLOW"; private static final String PENDING_WORKFLOWS = "PENDING_WORKFLOWS"; private static final String WORKFLOW_DEF_TO_WORKFLOWS = "WORKFLOW_DEF_TO_WORKFLOWS"; private static final String CORR_ID_TO_WORKFLOWS = "CORR_ID_TO_WORKFLOWS"; private static final String EVENT_EXECUTION = "EVENT_EXECUTION"; private final int ttlEventExecutionSeconds; public RedisExecutionDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { super(jedisProxy, objectMapper, conductorProperties, properties); ttlEventExecutionSeconds = (int) properties.getEventExecutionPersistenceTTL().getSeconds(); } private static String dateStr(Long timeInMs) { Date date = new Date(timeInMs); return dateStr(date); } private static String dateStr(Date date) { SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); return format.format(date); } private static List dateStrBetweenDates(Long startdatems, Long enddatems) { List dates = new ArrayList<>(); Calendar calendar = new GregorianCalendar(); Date startdate = new Date(startdatems); Date enddate = new Date(enddatems); calendar.setTime(startdate); while (calendar.getTime().before(enddate) || calendar.getTime().equals(enddate)) { Date result = calendar.getTime(); dates.add(dateStr(result)); calendar.add(Calendar.DATE, 1); } return dates; } @Override public List getPendingTasksByWorkflow(String taskName, String workflowId) { List tasks = new LinkedList<>(); List pendingTasks = getPendingTasksForTaskType(taskName); pendingTasks.forEach( pendingTask -> { if (pendingTask.getWorkflowInstanceId().equals(workflowId)) { tasks.add(pendingTask); } }); return tasks; } @Override public List getTasks(String taskDefName, String startKey, int count) { List tasks = new LinkedList<>(); List pendingTasks = getPendingTasksForTaskType(taskDefName); boolean startKeyFound = startKey == null; int foundcount = 0; for (TaskModel pendingTask : pendingTasks) { if (!startKeyFound) { if (pendingTask.getTaskId().equals(startKey)) { startKeyFound = true; if (startKey != null) { continue; } } } if (startKeyFound && foundcount < count) { tasks.add(pendingTask); foundcount++; } } return tasks; } @Override public List createTasks(List tasks) { List tasksCreated = new LinkedList<>(); for (TaskModel task : tasks) { validate(task); recordRedisDaoRequests("createTask", task.getTaskType(), task.getWorkflowType()); String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount(); Long added = jedisProxy.hset( nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()), taskKey, task.getTaskId()); if (added < 1) { LOGGER.debug( "Task already scheduled, skipping the run " + task.getTaskId() + ", ref=" + task.getReferenceTaskName() + ", key=" + taskKey); continue; } if (task.getStatus() != null && !task.getStatus().isTerminal() && task.getScheduledTime() == 0) { task.setScheduledTime(System.currentTimeMillis()); } correlateTaskToWorkflowInDS(task.getTaskId(), task.getWorkflowInstanceId()); LOGGER.debug( "Scheduled task added to WORKFLOW_TO_TASKS workflowId: {}, taskId: {}, taskType: {} during createTasks", task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType()); String inProgressTaskKey = nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()); jedisProxy.sadd(inProgressTaskKey, task.getTaskId()); LOGGER.debug( "Scheduled task added to IN_PROGRESS_TASKS with inProgressTaskKey: {}, workflowId: {}, taskId: {}, taskType: {} during createTasks", inProgressTaskKey, task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType()); updateTask(task); tasksCreated.add(task); } return tasksCreated; } @Override public void updateTask(TaskModel task) { Optional taskDefinition = task.getTaskDefinition(); if (taskDefinition.isPresent() && taskDefinition.get().concurrencyLimit() > 0) { if (task.getStatus() != null && task.getStatus().equals(TaskModel.Status.IN_PROGRESS)) { jedisProxy.sadd( nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId()); LOGGER.debug( "Workflow Task added to TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask", nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName(), task.getTaskId()), task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType(), task.getStatus().name()); } else { jedisProxy.srem( nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId()); LOGGER.debug( "Workflow Task removed from TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask", nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName(), task.getTaskId()), task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType(), task.getStatus().name()); String key = nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()); jedisProxy.zrem(key, task.getTaskId()); LOGGER.debug( "Workflow Task removed from TASK_LIMIT_BUCKET with taskLimitBucketKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask", key, task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType(), task.getStatus().name()); } } String payload = toJson(task); recordRedisDaoPayloadSize( "updateTask", payload.length(), taskDefinition.map(TaskDef::getName).orElse("n/a"), task.getWorkflowType()); recordRedisDaoRequests("updateTask", task.getTaskType(), task.getWorkflowType()); jedisProxy.set(nsKey(TASK, task.getTaskId()), payload); LOGGER.debug( "Workflow task payload saved to TASK with taskKey: {}, workflowId: {}, taskId: {}, taskType: {} during updateTask", nsKey(TASK, task.getTaskId()), task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType()); if (task.getStatus() != null && task.getStatus().isTerminal()) { jedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId()); LOGGER.debug( "Workflow Task removed from TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask", nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getWorkflowInstanceId(), task.getTaskId(), task.getTaskType(), task.getStatus().name()); } Set taskIds = jedisProxy.smembers(nsKey(WORKFLOW_TO_TASKS, task.getWorkflowInstanceId())); if (!taskIds.contains(task.getTaskId())) { correlateTaskToWorkflowInDS(task.getTaskId(), task.getWorkflowInstanceId()); } } @Override public boolean exceedsLimit(TaskModel task) { Optional taskDefinition = task.getTaskDefinition(); if (taskDefinition.isEmpty()) { return false; } int limit = taskDefinition.get().concurrencyLimit(); if (limit <= 0) { return false; } long current = getInProgressTaskCount(task.getTaskDefName()); if (current >= limit) { LOGGER.info( "Task execution count limited. task - {}:{}, limit: {}, current: {}", task.getTaskId(), task.getTaskDefName(), limit, current); Monitors.recordTaskConcurrentExecutionLimited(task.getTaskDefName(), limit); return true; } String rateLimitKey = nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()); double score = System.currentTimeMillis(); String taskId = task.getTaskId(); jedisProxy.zaddnx(rateLimitKey, score, taskId); recordRedisDaoRequests("checkTaskRateLimiting", task.getTaskType(), task.getWorkflowType()); Set ids = jedisProxy.zrangeByScore(rateLimitKey, 0, score + 1, limit); boolean rateLimited = !ids.contains(taskId); if (rateLimited) { LOGGER.info( "Task execution count limited. task - {}:{}, limit: {}, current: {}", task.getTaskId(), task.getTaskDefName(), limit, current); String inProgressKey = nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()); // Cleanup any items that are still present in the rate limit bucket but not in progress // anymore! ids.stream() .filter(id -> !jedisProxy.sismember(inProgressKey, id)) .forEach(id2 -> jedisProxy.zrem(rateLimitKey, id2)); Monitors.recordTaskRateLimited(task.getTaskDefName(), limit); } return rateLimited; } private void removeTaskMappings(TaskModel task) { String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount(); jedisProxy.hdel(nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()), taskKey); jedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId()); jedisProxy.srem(nsKey(WORKFLOW_TO_TASKS, task.getWorkflowInstanceId()), task.getTaskId()); jedisProxy.srem(nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId()); jedisProxy.zrem(nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()), task.getTaskId()); } private void removeTaskMappingsWithExpiry(TaskModel task) { String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount(); jedisProxy.hdel(nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()), taskKey); jedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId()); jedisProxy.srem(nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId()); jedisProxy.zrem(nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()), task.getTaskId()); } @Override public boolean removeTask(String taskId) { TaskModel task = getTask(taskId); if (task == null) { LOGGER.warn("No such task found by id {}", taskId); return false; } removeTaskMappings(task); jedisProxy.del(nsKey(TASK, task.getTaskId())); recordRedisDaoRequests("removeTask", task.getTaskType(), task.getWorkflowType()); return true; } private boolean removeTaskWithExpiry(String taskId, int ttlSeconds) { TaskModel task = getTask(taskId); if (task == null) { LOGGER.warn("No such task found by id {}", taskId); return false; } removeTaskMappingsWithExpiry(task); jedisProxy.expire(nsKey(TASK, task.getTaskId()), ttlSeconds); recordRedisDaoRequests("removeTask", task.getTaskType(), task.getWorkflowType()); return true; } @Override public TaskModel getTask(String taskId) { Preconditions.checkNotNull(taskId, "taskId cannot be null"); return Optional.ofNullable(jedisProxy.get(nsKey(TASK, taskId))) .map( json -> { TaskModel task = readValue(json, TaskModel.class); recordRedisDaoRequests( "getTask", task.getTaskType(), task.getWorkflowType()); recordRedisDaoPayloadSize( "getTask", toJson(task).length(), task.getTaskType(), task.getWorkflowType()); return task; }) .orElse(null); } @Override public List getTasks(List taskIds) { return taskIds.stream() .map(taskId -> nsKey(TASK, taskId)) .map(jedisProxy::get) .filter(Objects::nonNull) .map( jsonString -> { TaskModel task = readValue(jsonString, TaskModel.class); recordRedisDaoRequests( "getTask", task.getTaskType(), task.getWorkflowType()); recordRedisDaoPayloadSize( "getTask", jsonString.length(), task.getTaskType(), task.getWorkflowType()); return task; }) .collect(Collectors.toList()); } @Override public List getTasksForWorkflow(String workflowId) { Preconditions.checkNotNull(workflowId, "workflowId cannot be null"); Set taskIds = jedisProxy.smembers(nsKey(WORKFLOW_TO_TASKS, workflowId)); recordRedisDaoRequests("getTasksForWorkflow"); return getTasks(new ArrayList<>(taskIds)); } @Override public List getPendingTasksForTaskType(String taskName) { Preconditions.checkNotNull(taskName, "task name cannot be null"); Set taskIds = jedisProxy.smembers(nsKey(IN_PROGRESS_TASKS, taskName)); recordRedisDaoRequests("getPendingTasksForTaskType"); return getTasks(new ArrayList<>(taskIds)); } @Override public String createWorkflow(WorkflowModel workflow) { return insertOrUpdateWorkflow(workflow, false); } @Override public String updateWorkflow(WorkflowModel workflow) { return insertOrUpdateWorkflow(workflow, true); } @Override public boolean removeWorkflow(String workflowId) { WorkflowModel workflow = getWorkflow(workflowId, true); if (workflow != null) { recordRedisDaoRequests("removeWorkflow"); // Remove from lists String key = nsKey( WORKFLOW_DEF_TO_WORKFLOWS, workflow.getWorkflowName(), dateStr(workflow.getCreateTime())); jedisProxy.srem(key, workflowId); jedisProxy.srem(nsKey(CORR_ID_TO_WORKFLOWS, workflow.getCorrelationId()), workflowId); jedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflowId); // Remove the object jedisProxy.del(nsKey(WORKFLOW, workflowId)); for (TaskModel task : workflow.getTasks()) { removeTask(task.getTaskId()); } return true; } return false; } public boolean removeWorkflowWithExpiry(String workflowId, int ttlSeconds) { WorkflowModel workflow = getWorkflow(workflowId, true); if (workflow != null) { recordRedisDaoRequests("removeWorkflow"); // Remove from lists String key = nsKey( WORKFLOW_DEF_TO_WORKFLOWS, workflow.getWorkflowName(), dateStr(workflow.getCreateTime())); jedisProxy.srem(key, workflowId); jedisProxy.srem(nsKey(CORR_ID_TO_WORKFLOWS, workflow.getCorrelationId()), workflowId); jedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflowId); // Remove the object jedisProxy.expire(nsKey(WORKFLOW, workflowId), ttlSeconds); for (TaskModel task : workflow.getTasks()) { removeTaskWithExpiry(task.getTaskId(), ttlSeconds); } jedisProxy.expire(nsKey(WORKFLOW_TO_TASKS, workflowId), ttlSeconds); return true; } return false; } @Override public void removeFromPendingWorkflow(String workflowType, String workflowId) { recordRedisDaoRequests("removePendingWorkflow"); jedisProxy.del(nsKey(SCHEDULED_TASKS, workflowId)); jedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflowType), workflowId); } @Override public WorkflowModel getWorkflow(String workflowId) { return getWorkflow(workflowId, true); } @Override public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) { String json = jedisProxy.get(nsKey(WORKFLOW, workflowId)); WorkflowModel workflow = null; if (json != null) { workflow = readValue(json, WorkflowModel.class); recordRedisDaoRequests("getWorkflow", "n/a", workflow.getWorkflowName()); recordRedisDaoPayloadSize( "getWorkflow", json.length(), "n/a", workflow.getWorkflowName()); if (includeTasks) { List tasks = getTasksForWorkflow(workflowId); tasks.sort(Comparator.comparingInt(TaskModel::getSeq)); workflow.setTasks(tasks); } } return workflow; } /** * @param workflowName name of the workflow * @param version the workflow version * @return list of workflow ids that are in RUNNING state returns workflows of all versions * for the given workflow name */ @Override public List getRunningWorkflowIds(String workflowName, int version) { Preconditions.checkNotNull(workflowName, "workflowName cannot be null"); List workflowIds; recordRedisDaoRequests("getRunningWorkflowsByName"); Set pendingWorkflows = jedisProxy.smembers(nsKey(PENDING_WORKFLOWS, workflowName)); workflowIds = new LinkedList<>(pendingWorkflows); return workflowIds; } /** * @param workflowName name of the workflow * @param version the workflow version * @return list of workflows that are in RUNNING state */ @Override public List getPendingWorkflowsByType(String workflowName, int version) { Preconditions.checkNotNull(workflowName, "workflowName cannot be null"); List workflowIds = getRunningWorkflowIds(workflowName, version); return workflowIds.stream() .map(this::getWorkflow) .filter(workflow -> workflow.getWorkflowVersion() == version) .collect(Collectors.toList()); } @Override public List getWorkflowsByType( String workflowName, Long startTime, Long endTime) { Preconditions.checkNotNull(workflowName, "workflowName cannot be null"); Preconditions.checkNotNull(startTime, "startTime cannot be null"); Preconditions.checkNotNull(endTime, "endTime cannot be null"); List workflows = new LinkedList<>(); // Get all date strings between start and end List dateStrs = dateStrBetweenDates(startTime, endTime); dateStrs.forEach( dateStr -> { String key = nsKey(WORKFLOW_DEF_TO_WORKFLOWS, workflowName, dateStr); jedisProxy .smembers(key) .forEach( workflowId -> { try { WorkflowModel workflow = getWorkflow(workflowId); if (workflow.getCreateTime() >= startTime && workflow.getCreateTime() <= endTime) { workflows.add(workflow); } } catch (Exception e) { LOGGER.error( "Failed to get workflow: {}", workflowId, e); } }); }); return workflows; } @Override public List getWorkflowsByCorrelationId( String workflowName, String correlationId, boolean includeTasks) { throw new UnsupportedOperationException( "This method is not implemented in RedisExecutionDAO. Please use ExecutionDAOFacade instead."); } @Override public boolean canSearchAcrossWorkflows() { return false; } /** * Inserts a new workflow/ updates an existing workflow in the datastore. Additionally, if a * workflow is in terminal state, it is removed from the set of pending workflows. * * @param workflow the workflow instance * @param update flag to identify if update or create operation * @return the workflowId */ private String insertOrUpdateWorkflow(WorkflowModel workflow, boolean update) { Preconditions.checkNotNull(workflow, "workflow object cannot be null"); List tasks = workflow.getTasks(); workflow.setTasks(new LinkedList<>()); String payload = toJson(workflow); // Store the workflow object jedisProxy.set(nsKey(WORKFLOW, workflow.getWorkflowId()), payload); recordRedisDaoRequests("storeWorkflow", "n/a", workflow.getWorkflowName()); recordRedisDaoPayloadSize( "storeWorkflow", payload.length(), "n/a", workflow.getWorkflowName()); if (!update) { // Add to list of workflows for a workflowdef String key = nsKey( WORKFLOW_DEF_TO_WORKFLOWS, workflow.getWorkflowName(), dateStr(workflow.getCreateTime())); jedisProxy.sadd(key, workflow.getWorkflowId()); if (workflow.getCorrelationId() != null) { // Add to list of workflows for a correlationId jedisProxy.sadd( nsKey(CORR_ID_TO_WORKFLOWS, workflow.getCorrelationId()), workflow.getWorkflowId()); } } // Add or remove from the pending workflows if (workflow.getStatus().isTerminal()) { jedisProxy.srem( nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflow.getWorkflowId()); } else { jedisProxy.sadd( nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflow.getWorkflowId()); } workflow.setTasks(tasks); return workflow.getWorkflowId(); } /** * Stores the correlation of a task to the workflow instance in the datastore * * @param taskId the taskId to be correlated * @param workflowInstanceId the workflowId to which the tasks belongs to */ @VisibleForTesting void correlateTaskToWorkflowInDS(String taskId, String workflowInstanceId) { String workflowToTaskKey = nsKey(WORKFLOW_TO_TASKS, workflowInstanceId); jedisProxy.sadd(workflowToTaskKey, taskId); LOGGER.debug( "Task mapped in WORKFLOW_TO_TASKS with workflowToTaskKey: {}, workflowId: {}, taskId: {}", workflowToTaskKey, workflowInstanceId, taskId); } public long getPendingWorkflowCount(String workflowName) { String key = nsKey(PENDING_WORKFLOWS, workflowName); recordRedisDaoRequests("getPendingWorkflowCount"); return jedisProxy.scard(key); } @Override public long getInProgressTaskCount(String taskDefName) { String inProgressKey = nsKey(TASKS_IN_PROGRESS_STATUS, taskDefName); recordRedisDaoRequests("getInProgressTaskCount"); return jedisProxy.scard(inProgressKey); } @Override public boolean addEventExecution(EventExecution eventExecution) { try { String key = nsKey( EVENT_EXECUTION, eventExecution.getName(), eventExecution.getEvent(), eventExecution.getMessageId()); String json = objectMapper.writeValueAsString(eventExecution); recordRedisDaoEventRequests("addEventExecution", eventExecution.getEvent()); recordRedisDaoPayloadSize( "addEventExecution", json.length(), eventExecution.getEvent(), "n/a"); boolean added = jedisProxy.hsetnx(key, eventExecution.getId(), json) == 1L; if (ttlEventExecutionSeconds > 0) { jedisProxy.expire(key, ttlEventExecutionSeconds); } return added; } catch (Exception e) { throw new TransientException( "Unable to add event execution for " + eventExecution.getId(), e); } } @Override public void updateEventExecution(EventExecution eventExecution) { try { String key = nsKey( EVENT_EXECUTION, eventExecution.getName(), eventExecution.getEvent(), eventExecution.getMessageId()); String json = objectMapper.writeValueAsString(eventExecution); LOGGER.info("updating event execution {}", key); jedisProxy.hset(key, eventExecution.getId(), json); recordRedisDaoEventRequests("updateEventExecution", eventExecution.getEvent()); recordRedisDaoPayloadSize( "updateEventExecution", json.length(), eventExecution.getEvent(), "n/a"); } catch (Exception e) { throw new TransientException( "Unable to update event execution for " + eventExecution.getId(), e); } } @Override public void removeEventExecution(EventExecution eventExecution) { try { String key = nsKey( EVENT_EXECUTION, eventExecution.getName(), eventExecution.getEvent(), eventExecution.getMessageId()); LOGGER.info("removing event execution {}", key); jedisProxy.hdel(key, eventExecution.getId()); recordRedisDaoEventRequests("removeEventExecution", eventExecution.getEvent()); } catch (Exception e) { throw new TransientException( "Unable to remove event execution for " + eventExecution.getId(), e); } } public List getEventExecutions( String eventHandlerName, String eventName, String messageId, int max) { try { String key = nsKey(EVENT_EXECUTION, eventHandlerName, eventName, messageId); LOGGER.info("getting event execution {}", key); List executions = new LinkedList<>(); for (int i = 0; i < max; i++) { String field = messageId + "_" + i; String value = jedisProxy.hget(key, field); if (value == null) { break; } recordRedisDaoEventRequests("getEventExecution", eventHandlerName); recordRedisDaoPayloadSize( "getEventExecution", value.length(), eventHandlerName, "n/a"); EventExecution eventExecution = objectMapper.readValue(value, EventExecution.class); executions.add(eventExecution); } return executions; } catch (Exception e) { throw new TransientException( "Unable to get event executions for " + eventHandlerName, e); } } private void validate(TaskModel task) { try { Preconditions.checkNotNull(task, "task object cannot be null"); Preconditions.checkNotNull(task.getTaskId(), "Task id cannot be null"); Preconditions.checkNotNull( task.getWorkflowInstanceId(), "Workflow instance id cannot be null"); Preconditions.checkNotNull( task.getReferenceTaskName(), "Task reference name cannot be null"); } catch (NullPointerException npe) { throw new IllegalArgumentException(npe.getMessage(), npe); } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/RedisMetadataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import static com.netflix.conductor.common.metadata.tasks.TaskDef.ONE_HOUR; @Component @Conditional(AnyRedisCondition.class) public class RedisMetadataDAO extends BaseDynoDAO implements MetadataDAO { private static final Logger LOGGER = LoggerFactory.getLogger(RedisMetadataDAO.class); // Keys Families private static final String ALL_TASK_DEFS = "TASK_DEFS"; private static final String WORKFLOW_DEF_NAMES = "WORKFLOW_DEF_NAMES"; private static final String WORKFLOW_DEF = "WORKFLOW_DEF"; private static final String LATEST = "latest"; private static final String className = RedisMetadataDAO.class.getSimpleName(); private Map taskDefCache = new HashMap<>(); public RedisMetadataDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { super(jedisProxy, objectMapper, conductorProperties, properties); refreshTaskDefs(); long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds(); Executors.newSingleThreadScheduledExecutor() .scheduleWithFixedDelay( this::refreshTaskDefs, cacheRefreshTime, cacheRefreshTime, TimeUnit.SECONDS); } @Override public TaskDef createTaskDef(TaskDef taskDef) { return insertOrUpdateTaskDef(taskDef); } @Override public TaskDef updateTaskDef(TaskDef taskDef) { return insertOrUpdateTaskDef(taskDef); } private TaskDef insertOrUpdateTaskDef(TaskDef taskDef) { // Store all task def in under one key String payload = toJson(taskDef); jedisProxy.hset(nsKey(ALL_TASK_DEFS), taskDef.getName(), payload); recordRedisDaoRequests("storeTaskDef"); recordRedisDaoPayloadSize("storeTaskDef", payload.length(), taskDef.getName(), "n/a"); refreshTaskDefs(); return taskDef; } private void refreshTaskDefs() { try { Map map = new HashMap<>(); getAllTaskDefs().forEach(taskDef -> map.put(taskDef.getName(), taskDef)); this.taskDefCache = map; LOGGER.debug("Refreshed task defs " + this.taskDefCache.size()); } catch (Exception e) { Monitors.error(className, "refreshTaskDefs"); LOGGER.error("refresh TaskDefs failed ", e); } } @Override public TaskDef getTaskDef(String name) { return Optional.ofNullable(taskDefCache.get(name)).orElseGet(() -> getTaskDefFromDB(name)); } private TaskDef getTaskDefFromDB(String name) { Preconditions.checkNotNull(name, "TaskDef name cannot be null"); TaskDef taskDef = null; String taskDefJsonStr = jedisProxy.hget(nsKey(ALL_TASK_DEFS), name); if (taskDefJsonStr != null) { taskDef = readValue(taskDefJsonStr, TaskDef.class); recordRedisDaoRequests("getTaskDef"); recordRedisDaoPayloadSize( "getTaskDef", taskDefJsonStr.length(), taskDef.getName(), "n/a"); } setDefaults(taskDef); return taskDef; } private void setDefaults(TaskDef taskDef) { if (taskDef != null && taskDef.getResponseTimeoutSeconds() == 0) { taskDef.setResponseTimeoutSeconds( taskDef.getTimeoutSeconds() == 0 ? ONE_HOUR : taskDef.getTimeoutSeconds() - 1); } } @Override public List getAllTaskDefs() { List allTaskDefs = new LinkedList<>(); recordRedisDaoRequests("getAllTaskDefs"); Map taskDefs = jedisProxy.hgetAll(nsKey(ALL_TASK_DEFS)); int size = 0; if (taskDefs.size() > 0) { for (String taskDefJsonStr : taskDefs.values()) { if (taskDefJsonStr != null) { TaskDef taskDef = readValue(taskDefJsonStr, TaskDef.class); setDefaults(taskDef); allTaskDefs.add(taskDef); size += taskDefJsonStr.length(); } } recordRedisDaoPayloadSize("getAllTaskDefs", size, "n/a", "n/a"); } return allTaskDefs; } @Override public void removeTaskDef(String name) { Preconditions.checkNotNull(name, "TaskDef name cannot be null"); Long result = jedisProxy.hdel(nsKey(ALL_TASK_DEFS), name); if (!result.equals(1L)) { throw new NotFoundException("Cannot remove the task - no such task definition"); } recordRedisDaoRequests("removeTaskDef"); refreshTaskDefs(); } @Override public void createWorkflowDef(WorkflowDef def) { if (jedisProxy.hexists( nsKey(WORKFLOW_DEF, def.getName()), String.valueOf(def.getVersion()))) { throw new ConflictException("Workflow with %s already exists!", def.key()); } _createOrUpdate(def); } @Override public void updateWorkflowDef(WorkflowDef def) { _createOrUpdate(def); } @Override /* * @param name Name of the workflow definition * @return Latest version of workflow definition * @see WorkflowDef */ public Optional getLatestWorkflowDef(String name) { Preconditions.checkNotNull(name, "WorkflowDef name cannot be null"); WorkflowDef workflowDef = null; Optional optionalMaxVersion = getWorkflowMaxVersion(name); if (optionalMaxVersion.isPresent()) { String latestdata = jedisProxy.hget(nsKey(WORKFLOW_DEF, name), optionalMaxVersion.get().toString()); if (latestdata != null) { workflowDef = readValue(latestdata, WorkflowDef.class); } } return Optional.ofNullable(workflowDef); } private Optional getWorkflowMaxVersion(String workflowName) { return jedisProxy.hkeys(nsKey(WORKFLOW_DEF, workflowName)).stream() .filter(key -> !key.equals(LATEST)) .map(Integer::valueOf) .max(Comparator.naturalOrder()); } public List getAllVersions(String name) { Preconditions.checkNotNull(name, "WorkflowDef name cannot be null"); List workflows = new LinkedList<>(); recordRedisDaoRequests("getAllWorkflowDefsByName"); Map workflowDefs = jedisProxy.hgetAll(nsKey(WORKFLOW_DEF, name)); int size = 0; for (String key : workflowDefs.keySet()) { if (key.equals(LATEST)) { continue; } String workflowDef = workflowDefs.get(key); workflows.add(readValue(workflowDef, WorkflowDef.class)); size += workflowDef.length(); } recordRedisDaoPayloadSize("getAllWorkflowDefsByName", size, "n/a", name); return workflows; } @Override public Optional getWorkflowDef(String name, int version) { Preconditions.checkNotNull(name, "WorkflowDef name cannot be null"); WorkflowDef def = null; recordRedisDaoRequests("getWorkflowDef"); String workflowDefJsonString = jedisProxy.hget(nsKey(WORKFLOW_DEF, name), String.valueOf(version)); if (workflowDefJsonString != null) { def = readValue(workflowDefJsonString, WorkflowDef.class); recordRedisDaoPayloadSize( "getWorkflowDef", workflowDefJsonString.length(), "n/a", name); } return Optional.ofNullable(def); } @Override public void removeWorkflowDef(String name, Integer version) { Preconditions.checkArgument( StringUtils.isNotBlank(name), "WorkflowDef name cannot be null"); Preconditions.checkNotNull(version, "Input version cannot be null"); Long result = jedisProxy.hdel(nsKey(WORKFLOW_DEF, name), String.valueOf(version)); if (!result.equals(1L)) { throw new NotFoundException( "Cannot remove the workflow - no such workflow" + " definition: %s version: %d", name, version); } // check if there are any more versions remaining if not delete the // workflow name Optional optionMaxVersion = getWorkflowMaxVersion(name); // delete workflow name if (optionMaxVersion.isEmpty()) { jedisProxy.srem(nsKey(WORKFLOW_DEF_NAMES), name); } recordRedisDaoRequests("removeWorkflowDef"); } public List findAll() { Set wfNames = jedisProxy.smembers(nsKey(WORKFLOW_DEF_NAMES)); return new ArrayList<>(wfNames); } @Override public List getAllWorkflowDefs() { List workflows = new LinkedList<>(); // Get all from WORKFLOW_DEF_NAMES recordRedisDaoRequests("getAllWorkflowDefs"); Set wfNames = jedisProxy.smembers(nsKey(WORKFLOW_DEF_NAMES)); int size = 0; for (String wfName : wfNames) { Map workflowDefs = jedisProxy.hgetAll(nsKey(WORKFLOW_DEF, wfName)); for (String key : workflowDefs.keySet()) { if (key.equals(LATEST)) { continue; } String workflowDef = workflowDefs.get(key); workflows.add(readValue(workflowDef, WorkflowDef.class)); size += workflowDef.length(); } } recordRedisDaoPayloadSize("getAllWorkflowDefs", size, "n/a", "n/a"); return workflows; } @Override public List getAllWorkflowDefsLatestVersions() { List workflows = new LinkedList<>(); // Get all definitions latest versions from WORKFLOW_DEF_NAMES recordRedisDaoRequests("getAllWorkflowLatestVersionsDefs"); Set wfNames = jedisProxy.smembers(nsKey(WORKFLOW_DEF_NAMES)); int size = 0; // Place all workflows into the Priority Queue. The PQ will allow us to grab the latest // version of the workflows. for (String wfName : wfNames) { WorkflowDef def = getLatestWorkflowDef(wfName).orElse(null); if (def != null) { workflows.add(def); size += def.toString().length(); } } recordRedisDaoPayloadSize("getAllWorkflowLatestVersionsDefs", size, "n/a", "n/a"); return workflows; } private void _createOrUpdate(WorkflowDef workflowDef) { // First set the workflow def jedisProxy.hset( nsKey(WORKFLOW_DEF, workflowDef.getName()), String.valueOf(workflowDef.getVersion()), toJson(workflowDef)); jedisProxy.sadd(nsKey(WORKFLOW_DEF_NAMES), workflowDef.getName()); recordRedisDaoRequests("storeWorkflowDef", "n/a", workflowDef.getName()); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/RedisPollDataDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.PollDataDAO; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; @Component @Conditional(AnyRedisCondition.class) public class RedisPollDataDAO extends BaseDynoDAO implements PollDataDAO { private static final String POLL_DATA = "POLL_DATA"; public RedisPollDataDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { super(jedisProxy, objectMapper, conductorProperties, properties); } @Override public void updateLastPollData(String taskDefName, String domain, String workerId) { Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); PollData pollData = new PollData(taskDefName, domain, workerId, System.currentTimeMillis()); String key = nsKey(POLL_DATA, pollData.getQueueName()); String field = (domain == null) ? "DEFAULT" : domain; String payload = toJson(pollData); recordRedisDaoRequests("updatePollData"); recordRedisDaoPayloadSize("updatePollData", payload.length(), "n/a", "n/a"); jedisProxy.hset(key, field, payload); } @Override public PollData getPollData(String taskDefName, String domain) { Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); String key = nsKey(POLL_DATA, taskDefName); String field = (domain == null) ? "DEFAULT" : domain; String pollDataJsonString = jedisProxy.hget(key, field); recordRedisDaoRequests("getPollData"); recordRedisDaoPayloadSize( "getPollData", StringUtils.length(pollDataJsonString), "n/a", "n/a"); PollData pollData = null; if (StringUtils.isNotBlank(pollDataJsonString)) { pollData = readValue(pollDataJsonString, PollData.class); } return pollData; } @Override public List getPollData(String taskDefName) { Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); String key = nsKey(POLL_DATA, taskDefName); Map pMapdata = jedisProxy.hgetAll(key); List pollData = new ArrayList<>(); if (pMapdata != null) { pMapdata.values() .forEach( pollDataJsonString -> { pollData.add(readValue(pollDataJsonString, PollData.class)); recordRedisDaoRequests("getPollData"); recordRedisDaoPayloadSize( "getPollData", pollDataJsonString.length(), "n/a", "n/a"); }); } return pollData; } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dao/RedisRateLimitingDAO.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.Optional; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.RateLimitingDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.redis.config.AnyRedisCondition; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; @Component @Conditional(AnyRedisCondition.class) public class RedisRateLimitingDAO extends BaseDynoDAO implements RateLimitingDAO { private static final Logger LOGGER = LoggerFactory.getLogger(RedisRateLimitingDAO.class); private static final String TASK_RATE_LIMIT_BUCKET = "TASK_RATE_LIMIT_BUCKET"; public RedisRateLimitingDAO( JedisProxy jedisProxy, ObjectMapper objectMapper, ConductorProperties conductorProperties, RedisProperties properties) { super(jedisProxy, objectMapper, conductorProperties, properties); } /** * This method evaluates if the {@link TaskDef} is rate limited or not based on {@link * TaskModel#getRateLimitPerFrequency()} and {@link TaskModel#getRateLimitFrequencyInSeconds()} * if not checks the {@link TaskModel} is rate limited or not based on {@link * TaskModel#getRateLimitPerFrequency()} and {@link TaskModel#getRateLimitFrequencyInSeconds()} * *

    The rate limiting is implemented using the Redis constructs of sorted set and TTL of each * element in the rate limited bucket. * *

      *
    • All the entries that are in the not in the frequency bucket are cleaned up by * leveraging {@link JedisProxy#zremrangeByScore(String, String, String)}, this is done to * make the next step of evaluation efficient *
    • A current count(tasks executed within the frequency) is calculated based on the current * time and the beginning of the rate limit frequency time(which is current time - {@link * TaskModel#getRateLimitFrequencyInSeconds()} in millis), this is achieved by using * {@link JedisProxy#zcount(String, double, double)} *
    • Once the count is calculated then a evaluation is made to determine if it is within the * bounds of {@link TaskModel#getRateLimitPerFrequency()}, if so the count is increased * and an expiry TTL is added to the entry *
    * * @param task: which needs to be evaluated whether it is rateLimited or not * @return true: If the {@link TaskModel} is rateLimited false: If the {@link TaskModel} is not * rateLimited */ @Override public boolean exceedsRateLimitPerFrequency(TaskModel task, TaskDef taskDef) { // Check if the TaskDefinition is not null then pick the definition values or else pick from // the Task ImmutablePair rateLimitPair = Optional.ofNullable(taskDef) .map( definition -> new ImmutablePair<>( definition.getRateLimitPerFrequency(), definition.getRateLimitFrequencyInSeconds())) .orElse( new ImmutablePair<>( task.getRateLimitPerFrequency(), task.getRateLimitFrequencyInSeconds())); int rateLimitPerFrequency = rateLimitPair.getLeft(); int rateLimitFrequencyInSeconds = rateLimitPair.getRight(); if (rateLimitPerFrequency <= 0 || rateLimitFrequencyInSeconds <= 0) { LOGGER.debug( "Rate limit not applied to the Task: {} either rateLimitPerFrequency: {} or rateLimitFrequencyInSeconds: {} is 0 or less", task, rateLimitPerFrequency, rateLimitFrequencyInSeconds); return false; } else { LOGGER.debug( "Evaluating rate limiting for TaskId: {} with TaskDefinition of: {} with rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {}", task.getTaskId(), task.getTaskDefName(), rateLimitPerFrequency, rateLimitFrequencyInSeconds); long currentTimeEpochMillis = System.currentTimeMillis(); long currentTimeEpochMinusRateLimitBucket = currentTimeEpochMillis - (rateLimitFrequencyInSeconds * 1000L); String key = nsKey(TASK_RATE_LIMIT_BUCKET, task.getTaskDefName()); jedisProxy.zremrangeByScore( key, "-inf", String.valueOf(currentTimeEpochMinusRateLimitBucket)); int currentBucketCount = Math.toIntExact( jedisProxy.zcount( key, currentTimeEpochMinusRateLimitBucket, currentTimeEpochMillis)); if (currentBucketCount < rateLimitPerFrequency) { jedisProxy.zadd( key, currentTimeEpochMillis, String.valueOf(currentTimeEpochMillis)); jedisProxy.expire(key, rateLimitFrequencyInSeconds); LOGGER.info( "TaskId: {} with TaskDefinition of: {} has rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {} within the rate limit with current count {}", task.getTaskId(), task.getTaskDefName(), rateLimitPerFrequency, rateLimitFrequencyInSeconds, ++currentBucketCount); Monitors.recordTaskRateLimited(task.getTaskDefName(), rateLimitPerFrequency); return false; } else { LOGGER.info( "TaskId: {} with TaskDefinition of: {} has rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {} is out of bounds of rate limit with current count {}", task.getTaskId(), task.getTaskDefName(), rateLimitPerFrequency, rateLimitFrequencyInSeconds, currentBucketCount); return true; } } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/ConfigurationHostSupplier.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dynoqueue; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; public class ConfigurationHostSupplier implements HostSupplier { private static final Logger log = LoggerFactory.getLogger(ConfigurationHostSupplier.class); private final RedisProperties properties; public ConfigurationHostSupplier(RedisProperties properties) { this.properties = properties; } @Override public List getHosts() { return parseHostsFromConfig(); } private List parseHostsFromConfig() { String hosts = properties.getHosts(); if (hosts == null) { String message = "Missing dynomite/redis hosts. Ensure 'conductor.redis.hosts' has been set in the supplied configuration."; log.error(message); throw new RuntimeException(message); } return parseHostsFrom(hosts); } private List parseHostsFrom(String hostConfig) { List hostConfigs = Arrays.asList(hostConfig.split(";")); return hostConfigs.stream() .map( hc -> { String[] hostConfigValues = hc.split(":"); String host = hostConfigValues[0]; int port = Integer.parseInt(hostConfigValues[1]); String rack = hostConfigValues[2]; if (hostConfigValues.length >= 4) { String password = hostConfigValues[3]; return new HostBuilder() .setHostname(host) .setPort(port) .setRack(rack) .setStatus(Host.Status.Up) .setPassword(password) .createHost(); } return new HostBuilder() .setHostname(host) .setPort(port) .setRack(rack) .setStatus(Host.Status.Up) .createHost(); }) .collect(Collectors.toList()); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/LocalhostHostSupplier.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dynoqueue; import java.util.List; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.google.common.collect.Lists; public class LocalhostHostSupplier implements HostSupplier { private final RedisProperties properties; public LocalhostHostSupplier(RedisProperties properties) { this.properties = properties; } @Override public List getHosts() { Host dynoHost = new HostBuilder() .setHostname("localhost") .setIpAddress("0") .setRack(properties.getAvailabilityZone()) .setStatus(Host.Status.Up) .createHost(); return Lists.newArrayList(dynoHost); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/RedisQueuesShardingStrategyProvider.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dynoqueue; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; public class RedisQueuesShardingStrategyProvider { public static final String LOCAL_ONLY_STRATEGY = "localOnly"; public static final String ROUND_ROBIN_STRATEGY = "roundRobin"; private static final Logger LOGGER = LoggerFactory.getLogger(RedisQueuesShardingStrategyProvider.class); private final ShardSupplier shardSupplier; private final RedisProperties properties; public RedisQueuesShardingStrategyProvider( ShardSupplier shardSupplier, RedisProperties properties) { this.shardSupplier = shardSupplier; this.properties = properties; } public ShardingStrategy get() { String shardingStrat = properties.getQueueShardingStrategy(); if (shardingStrat.equals(LOCAL_ONLY_STRATEGY)) { LOGGER.info( "Using {} sharding strategy for queues", LocalOnlyStrategy.class.getSimpleName()); return new LocalOnlyStrategy(shardSupplier); } else { LOGGER.info( "Using {} sharding strategy for queues", RoundRobinStrategy.class.getSimpleName()); return new RoundRobinStrategy(); } } public static final class LocalOnlyStrategy implements ShardingStrategy { private static final Logger LOGGER = LoggerFactory.getLogger(LocalOnlyStrategy.class); private final ShardSupplier shardSupplier; public LocalOnlyStrategy(ShardSupplier shardSupplier) { this.shardSupplier = shardSupplier; } @Override public String getNextShard(List allShards, Message message) { LOGGER.debug( "Always using {} shard out of {}", shardSupplier.getCurrentShard(), allShards); return shardSupplier.getCurrentShard(); } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisCluster.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.AbstractMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; import redis.clients.jedis.BitPosParams; import redis.clients.jedis.GeoCoordinate; import redis.clients.jedis.GeoRadiusResponse; import redis.clients.jedis.GeoUnit; import redis.clients.jedis.ListPosition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.SortingParams; import redis.clients.jedis.StreamConsumersInfo; import redis.clients.jedis.StreamEntry; import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.StreamGroupInfo; import redis.clients.jedis.StreamInfo; import redis.clients.jedis.StreamPendingEntry; import redis.clients.jedis.Tuple; import redis.clients.jedis.commands.JedisCommands; import redis.clients.jedis.params.GeoRadiusParam; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.params.ZAddParams; import redis.clients.jedis.params.ZIncrByParams; public class JedisCluster implements JedisCommands { private final redis.clients.jedis.JedisCluster jedisCluster; public JedisCluster(redis.clients.jedis.JedisCluster jedisCluster) { this.jedisCluster = jedisCluster; } @Override public String set(String key, String value) { return jedisCluster.set(key, value); } @Override public String set(String key, String value, SetParams params) { return jedisCluster.set(key, value, params); } @Override public String get(String key) { return jedisCluster.get(key); } @Override public Boolean exists(String key) { return jedisCluster.exists(key); } @Override public Long persist(String key) { return jedisCluster.persist(key); } @Override public String type(String key) { return jedisCluster.type(key); } @Override public byte[] dump(String key) { return jedisCluster.dump(key); } @Override public String restore(String key, int ttl, byte[] serializedValue) { return jedisCluster.restore(key, ttl, serializedValue); } @Override public String restoreReplace(String key, int ttl, byte[] serializedValue) { throw new UnsupportedOperationException(); } @Override public Long expire(String key, int seconds) { return jedisCluster.expire(key, seconds); } @Override public Long pexpire(String key, long milliseconds) { return jedisCluster.pexpire(key, milliseconds); } @Override public Long expireAt(String key, long unixTime) { return jedisCluster.expireAt(key, unixTime); } @Override public Long pexpireAt(String key, long millisecondsTimestamp) { return jedisCluster.pexpireAt(key, millisecondsTimestamp); } @Override public Long ttl(String key) { return jedisCluster.ttl(key); } @Override public Long pttl(String key) { return jedisCluster.pttl(key); } @Override public Long touch(String key) { return jedisCluster.touch(key); } @Override public Boolean setbit(String key, long offset, boolean value) { return jedisCluster.setbit(key, offset, value); } @Override public Boolean setbit(String key, long offset, String value) { return jedisCluster.setbit(key, offset, value); } @Override public Boolean getbit(String key, long offset) { return jedisCluster.getbit(key, offset); } @Override public Long setrange(String key, long offset, String value) { return jedisCluster.setrange(key, offset, value); } @Override public String getrange(String key, long startOffset, long endOffset) { return jedisCluster.getrange(key, startOffset, endOffset); } @Override public String getSet(String key, String value) { return jedisCluster.getSet(key, value); } @Override public Long setnx(String key, String value) { return jedisCluster.setnx(key, value); } @Override public String setex(String key, int seconds, String value) { return jedisCluster.setex(key, seconds, value); } @Override public String psetex(String key, long milliseconds, String value) { return jedisCluster.psetex(key, milliseconds, value); } @Override public Long decrBy(String key, long integer) { return jedisCluster.decrBy(key, integer); } @Override public Long decr(String key) { return jedisCluster.decr(key); } @Override public Long incrBy(String key, long integer) { return jedisCluster.incrBy(key, integer); } @Override public Double incrByFloat(String key, double value) { return jedisCluster.incrByFloat(key, value); } @Override public Long incr(String key) { return jedisCluster.incr(key); } @Override public Long append(String key, String value) { return jedisCluster.append(key, value); } @Override public String substr(String key, int start, int end) { return jedisCluster.substr(key, start, end); } @Override public Long hset(String key, String field, String value) { return jedisCluster.hset(key, field, value); } @Override public Long hset(String key, Map hash) { return jedisCluster.hset(key, hash); } @Override public String hget(String key, String field) { return jedisCluster.hget(key, field); } @Override public Long hsetnx(String key, String field, String value) { return jedisCluster.hsetnx(key, field, value); } @Override public String hmset(String key, Map hash) { return jedisCluster.hmset(key, hash); } @Override public List hmget(String key, String... fields) { return jedisCluster.hmget(key, fields); } @Override public Long hincrBy(String key, String field, long value) { return jedisCluster.hincrBy(key, field, value); } @Override public Double hincrByFloat(String key, String field, double value) { return jedisCluster.hincrByFloat(key.getBytes(), field.getBytes(), value); } @Override public Boolean hexists(String key, String field) { return jedisCluster.hexists(key, field); } @Override public Long hdel(String key, String... field) { return jedisCluster.hdel(key, field); } @Override public Long hlen(String key) { return jedisCluster.hlen(key); } @Override public Set hkeys(String key) { return jedisCluster.hkeys(key); } @Override public List hvals(String key) { return jedisCluster.hvals(key); } @Override public Map hgetAll(String key) { return jedisCluster.hgetAll(key); } @Override public Long rpush(String key, String... string) { return jedisCluster.rpush(key, string); } @Override public Long lpush(String key, String... string) { return jedisCluster.lpush(key, string); } @Override public Long llen(String key) { return jedisCluster.llen(key); } @Override public List lrange(String key, long start, long end) { return jedisCluster.lrange(key, start, end); } @Override public String ltrim(String key, long start, long end) { return jedisCluster.ltrim(key, start, end); } @Override public String lindex(String key, long index) { return jedisCluster.lindex(key, index); } @Override public String lset(String key, long index, String value) { return jedisCluster.lset(key, index, value); } @Override public Long lrem(String key, long count, String value) { return jedisCluster.lrem(key, count, value); } @Override public String lpop(String key) { return jedisCluster.lpop(key); } @Override public String rpop(String key) { return jedisCluster.rpop(key); } @Override public Long sadd(String key, String... member) { return jedisCluster.sadd(key, member); } @Override public Set smembers(String key) { return jedisCluster.smembers(key); } @Override public Long srem(String key, String... member) { return jedisCluster.srem(key, member); } @Override public String spop(String key) { return jedisCluster.spop(key); } @Override public Set spop(String key, long count) { return jedisCluster.spop(key, count); } @Override public Long scard(String key) { return jedisCluster.scard(key); } @Override public Boolean sismember(String key, String member) { return jedisCluster.sismember(key, member); } @Override public String srandmember(String key) { return jedisCluster.srandmember(key); } @Override public List srandmember(String key, int count) { return jedisCluster.srandmember(key, count); } @Override public Long strlen(String key) { return jedisCluster.strlen(key); } @Override public Long zadd(String key, double score, String member) { return jedisCluster.zadd(key, score, member); } @Override public Long zadd(String key, double score, String member, ZAddParams params) { return jedisCluster.zadd(key, score, member, params); } @Override public Long zadd(String key, Map scoreMembers) { return jedisCluster.zadd(key, scoreMembers); } @Override public Long zadd(String key, Map scoreMembers, ZAddParams params) { return jedisCluster.zadd(key, scoreMembers, params); } @Override public Set zrange(String key, long start, long end) { return jedisCluster.zrange(key, start, end); } @Override public Long zrem(String key, String... member) { return jedisCluster.zrem(key, member); } @Override public Double zincrby(String key, double score, String member) { return jedisCluster.zincrby(key, score, member); } @Override public Double zincrby(String key, double score, String member, ZIncrByParams params) { return jedisCluster.zincrby(key, score, member, params); } @Override public Long zrank(String key, String member) { return jedisCluster.zrank(key, member); } @Override public Long zrevrank(String key, String member) { return jedisCluster.zrevrank(key, member); } @Override public Set zrevrange(String key, long start, long end) { return jedisCluster.zrevrange(key, start, end); } @Override public Set zrangeWithScores(String key, long start, long end) { return jedisCluster.zrangeWithScores(key, start, end); } @Override public Set zrevrangeWithScores(String key, long start, long end) { return jedisCluster.zrevrangeWithScores(key, start, end); } @Override public Long zcard(String key) { return jedisCluster.zcard(key); } @Override public Double zscore(String key, String member) { return jedisCluster.zscore(key, member); } @Override public Tuple zpopmax(String key) { return jedisCluster.zpopmax(key); } @Override public Set zpopmax(String key, int count) { return jedisCluster.zpopmax(key, count); } @Override public Tuple zpopmin(String key) { return jedisCluster.zpopmin(key); } @Override public Set zpopmin(String key, int count) { return jedisCluster.zpopmin(key, count); } @Override public List sort(String key) { return jedisCluster.sort(key); } @Override public List sort(String key, SortingParams sortingParameters) { return jedisCluster.sort(key, sortingParameters); } @Override public Long zcount(String key, double min, double max) { return jedisCluster.zcount(key, min, max); } @Override public Long zcount(String key, String min, String max) { return jedisCluster.zcount(key, min, max); } @Override public Set zrangeByScore(String key, double min, double max) { return jedisCluster.zrangeByScore(key, min, max); } @Override public Set zrangeByScore(String key, String min, String max) { return jedisCluster.zrangeByScore(key, min, max); } @Override public Set zrevrangeByScore(String key, double max, double min) { return jedisCluster.zrevrangeByScore(key, max, min); } @Override public Set zrangeByScore(String key, double min, double max, int offset, int count) { return jedisCluster.zrangeByScore(key, min, max, offset, count); } @Override public Set zrevrangeByScore(String key, String max, String min) { return jedisCluster.zrevrangeByScore(key, max, min); } @Override public Set zrangeByScore(String key, String min, String max, int offset, int count) { return jedisCluster.zrangeByScore(key, min, max, offset, count); } @Override public Set zrevrangeByScore(String key, double max, double min, int offset, int count) { return jedisCluster.zrevrangeByScore(key, max, min, offset, count); } @Override public Set zrangeByScoreWithScores(String key, double min, double max) { return jedisCluster.zrangeByScoreWithScores(key, min, max); } @Override public Set zrevrangeByScoreWithScores(String key, double max, double min) { return jedisCluster.zrevrangeByScoreWithScores(key, max, min); } @Override public Set zrangeByScoreWithScores( String key, double min, double max, int offset, int count) { return jedisCluster.zrangeByScoreWithScores(key, min, max, offset, count); } @Override public Set zrevrangeByScore(String key, String max, String min, int offset, int count) { return jedisCluster.zrevrangeByScore(key, max, min, offset, count); } @Override public Set zrangeByScoreWithScores(String key, String min, String max) { return jedisCluster.zrangeByScoreWithScores(key, min, max); } @Override public Set zrevrangeByScoreWithScores(String key, String max, String min) { return jedisCluster.zrevrangeByScoreWithScores(key, max, min); } @Override public Set zrangeByScoreWithScores( String key, String min, String max, int offset, int count) { return jedisCluster.zrangeByScoreWithScores(key, min, max, offset, count); } @Override public Set zrevrangeByScoreWithScores( String key, double max, double min, int offset, int count) { return jedisCluster.zrevrangeByScoreWithScores(key, max, min, offset, count); } @Override public Set zrevrangeByScoreWithScores( String key, String max, String min, int offset, int count) { return jedisCluster.zrevrangeByScoreWithScores(key, max, min, offset, count); } @Override public Long zremrangeByRank(String key, long start, long end) { return jedisCluster.zremrangeByRank(key, start, end); } @Override public Long zremrangeByScore(String key, double start, double end) { return jedisCluster.zremrangeByScore(key, start, end); } @Override public Long zremrangeByScore(String key, String start, String end) { return jedisCluster.zremrangeByScore(key, start, end); } @Override public Long zlexcount(String key, String min, String max) { return jedisCluster.zlexcount(key, min, max); } @Override public Set zrangeByLex(String key, String min, String max) { return jedisCluster.zrangeByLex(key, min, max); } @Override public Set zrangeByLex(String key, String min, String max, int offset, int count) { return jedisCluster.zrangeByLex(key, min, max, offset, count); } @Override public Set zrevrangeByLex(String key, String max, String min) { return jedisCluster.zrevrangeByLex(key, max, min); } @Override public Set zrevrangeByLex(String key, String max, String min, int offset, int count) { return jedisCluster.zrevrangeByLex(key, max, min, offset, count); } @Override public Long zremrangeByLex(String key, String min, String max) { return jedisCluster.zremrangeByLex(key, min, max); } @Override public Long linsert(String key, ListPosition where, String pivot, String value) { return jedisCluster.linsert(key, where, pivot, value); } @Override public Long lpushx(String key, String... string) { return jedisCluster.lpushx(key, string); } @Override public Long rpushx(String key, String... string) { return jedisCluster.rpushx(key, string); } @Override public List blpop(int timeout, String key) { return jedisCluster.blpop(timeout, key); } @Override public List brpop(int timeout, String key) { return jedisCluster.brpop(timeout, key); } @Override public Long del(String key) { return jedisCluster.del(key); } @Override public Long unlink(String key) { return jedisCluster.unlink(key); } @Override public String echo(String string) { return jedisCluster.echo(string); } @Override public Long move(String key, int dbIndex) { throw new UnsupportedOperationException(); } @Override public Long bitcount(String key) { return jedisCluster.bitcount(key); } @Override public Long bitcount(String key, long start, long end) { return jedisCluster.bitcount(key, start, end); } @Override public Long bitpos(String key, boolean value) { throw new UnsupportedOperationException(); } @Override public Long bitpos(String key, boolean value, BitPosParams params) { throw new UnsupportedOperationException(); } @Override public ScanResult> hscan(String key, String cursor) { return jedisCluster.hscan(key, cursor); } @Override public ScanResult> hscan( String key, String cursor, ScanParams params) { ScanResult> scanResult = jedisCluster.hscan(key.getBytes(), cursor.getBytes(), params); List> results = scanResult.getResult().stream() .map( entry -> new AbstractMap.SimpleEntry<>( new String(entry.getKey()), new String(entry.getValue()))) .collect(Collectors.toList()); return new ScanResult<>(scanResult.getCursorAsBytes(), results); } @Override public ScanResult sscan(String key, String cursor) { return jedisCluster.sscan(key, cursor); } @Override public ScanResult sscan(String key, String cursor, ScanParams params) { ScanResult scanResult = jedisCluster.sscan(key.getBytes(), cursor.getBytes(), params); List results = scanResult.getResult().stream().map(String::new).collect(Collectors.toList()); return new ScanResult<>(scanResult.getCursorAsBytes(), results); } @Override public ScanResult zscan(String key, String cursor) { return jedisCluster.zscan(key, cursor); } @Override public ScanResult zscan(String key, String cursor, ScanParams params) { return jedisCluster.zscan(key.getBytes(), cursor.getBytes(), params); } @Override public Long pfadd(String key, String... elements) { return jedisCluster.pfadd(key, elements); } @Override public long pfcount(String key) { return jedisCluster.pfcount(key); } @Override public Long geoadd(String key, double longitude, double latitude, String member) { return jedisCluster.geoadd(key, longitude, latitude, member); } @Override public Long geoadd(String key, Map memberCoordinateMap) { return jedisCluster.geoadd(key, memberCoordinateMap); } @Override public Double geodist(String key, String member1, String member2) { return jedisCluster.geodist(key, member1, member2); } @Override public Double geodist(String key, String member1, String member2, GeoUnit unit) { return jedisCluster.geodist(key, member1, member2, unit); } @Override public List geohash(String key, String... members) { return jedisCluster.geohash(key, members); } @Override public List geopos(String key, String... members) { return jedisCluster.geopos(key, members); } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit) { return jedisCluster.georadius(key, longitude, latitude, radius, unit); } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit) { return jedisCluster.georadiusReadonly(key, longitude, latitude, radius, unit); } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { return jedisCluster.georadius(key, longitude, latitude, radius, unit, param); } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { return jedisCluster.georadiusReadonly(key, longitude, latitude, radius, unit, param); } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit) { return jedisCluster.georadiusByMember(key, member, radius, unit); } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit) { return jedisCluster.georadiusByMemberReadonly(key, member, radius, unit); } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { return jedisCluster.georadiusByMember(key, member, radius, unit, param); } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { return jedisCluster.georadiusByMemberReadonly(key, member, radius, unit, param); } @Override public List bitfield(String key, String... arguments) { return jedisCluster.bitfield(key, arguments); } @Override public List bitfieldReadonly(String key, String... arguments) { return jedisCluster.bitfieldReadonly(key, arguments); } @Override public Long hstrlen(String key, String field) { return jedisCluster.hstrlen(key, field); } @Override public StreamEntryID xadd(String key, StreamEntryID id, Map hash) { return jedisCluster.xadd(key, id, hash); } @Override public StreamEntryID xadd( String key, StreamEntryID id, Map hash, long maxLen, boolean approximateLength) { return jedisCluster.xadd(key, id, hash, maxLen, approximateLength); } @Override public Long xlen(String key) { return jedisCluster.xlen(key); } @Override public List xrange(String key, StreamEntryID start, StreamEntryID end, int count) { return jedisCluster.xrange(key, start, end, count); } @Override public List xrevrange( String key, StreamEntryID end, StreamEntryID start, int count) { return jedisCluster.xrevrange(key, end, start, count); } @Override public long xack(String key, String group, StreamEntryID... ids) { return jedisCluster.xack(key, group, ids); } @Override public String xgroupCreate(String key, String groupname, StreamEntryID id, boolean makeStream) { return jedisCluster.xgroupCreate(key, groupname, id, makeStream); } @Override public String xgroupSetID(String key, String groupname, StreamEntryID id) { return jedisCluster.xgroupSetID(key, groupname, id); } @Override public long xgroupDestroy(String key, String groupname) { return jedisCluster.xgroupDestroy(key, groupname); } @Override public Long xgroupDelConsumer(String key, String groupname, String consumername) { return jedisCluster.xgroupDelConsumer(key, groupname, consumername); } @Override public List xpending( String key, String groupname, StreamEntryID start, StreamEntryID end, int count, String consumername) { return jedisCluster.xpending(key, groupname, start, end, count, consumername); } @Override public long xdel(String key, StreamEntryID... ids) { return jedisCluster.xdel(key, ids); } @Override public long xtrim(String key, long maxLen, boolean approximate) { return jedisCluster.xtrim(key, maxLen, approximate); } @Override public List xclaim( String key, String group, String consumername, long minIdleTime, long newIdleTime, int retries, boolean force, StreamEntryID... ids) { return jedisCluster.xclaim( key, group, consumername, minIdleTime, newIdleTime, retries, force, ids); } @Override public StreamInfo xinfoStream(String key) { return null; } @Override public List xinfoGroup(String key) { return null; } @Override public List xinfoConsumers(String key, String group) { return null; } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisMock.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.rarefiedredis.redis.IRedisClient; import org.rarefiedredis.redis.IRedisSortedSet.ZsetPair; import org.rarefiedredis.redis.RedisMock; import redis.clients.jedis.Jedis; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.Tuple; import redis.clients.jedis.exceptions.JedisException; import redis.clients.jedis.params.ZAddParams; public class JedisMock extends Jedis { private final IRedisClient redis; public JedisMock() { super(""); this.redis = new RedisMock(); } private Set toTupleSet(Set pairs) { Set set = new HashSet<>(); for (ZsetPair pair : pairs) { set.add(new Tuple(pair.member, pair.score)); } return set; } @Override public String set(final String key, String value) { try { return redis.set(key, value); } catch (Exception e) { throw new JedisException(e); } } @Override public String get(final String key) { try { return redis.get(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Boolean exists(final String key) { try { return redis.exists(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long del(final String... keys) { try { return redis.del(keys); } catch (Exception e) { throw new JedisException(e); } } @Override public Long del(String key) { try { return redis.del(key); } catch (Exception e) { throw new JedisException(e); } } @Override public String type(final String key) { try { return redis.type(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long expire(final String key, final int seconds) { try { return redis.expire(key, seconds) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public Long expireAt(final String key, final long unixTime) { try { return redis.expireat(key, unixTime) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public Long ttl(final String key) { try { return redis.ttl(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long move(final String key, final int dbIndex) { try { return redis.move(key, dbIndex); } catch (Exception e) { throw new JedisException(e); } } @Override public String getSet(final String key, final String value) { try { return redis.getset(key, value); } catch (Exception e) { throw new JedisException(e); } } @Override public List mget(final String... keys) { try { String[] mget = redis.mget(keys); List lst = new ArrayList<>(mget.length); for (String get : mget) { lst.add(get); } return lst; } catch (Exception e) { throw new JedisException(e); } } @Override public Long setnx(final String key, final String value) { try { return redis.setnx(key, value); } catch (Exception e) { throw new JedisException(e); } } @Override public String setex(final String key, final int seconds, final String value) { try { return redis.setex(key, seconds, value); } catch (Exception e) { throw new JedisException(e); } } @Override public String mset(final String... keysvalues) { try { return redis.mset(keysvalues); } catch (Exception e) { throw new JedisException(e); } } @Override public Long msetnx(final String... keysvalues) { try { return redis.msetnx(keysvalues) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public Long decrBy(final String key, final long integer) { try { return redis.decrby(key, integer); } catch (Exception e) { throw new JedisException(e); } } @Override public Long decr(final String key) { try { return redis.decr(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long incrBy(final String key, final long integer) { try { return redis.incrby(key, integer); } catch (Exception e) { throw new JedisException(e); } } @Override public Double incrByFloat(final String key, final double value) { try { return Double.parseDouble(redis.incrbyfloat(key, value)); } catch (Exception e) { throw new JedisException(e); } } @Override public Long incr(final String key) { try { return redis.incr(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long append(final String key, final String value) { try { return redis.append(key, value); } catch (Exception e) { throw new JedisException(e); } } @Override public String substr(final String key, final int start, final int end) { try { return redis.getrange(key, start, end); } catch (Exception e) { throw new JedisException(e); } } @Override public Long hset(final String key, final String field, final String value) { try { return redis.hset(key, field, value) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public String hget(final String key, final String field) { try { return redis.hget(key, field); } catch (Exception e) { throw new JedisException(e); } } @Override public Long hsetnx(final String key, final String field, final String value) { try { return redis.hsetnx(key, field, value) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public String hmset(final String key, final Map hash) { try { String field = null, value = null; String[] args = new String[(hash.size() - 1) * 2]; int idx = 0; for (String f : hash.keySet()) { if (field == null) { field = f; value = hash.get(f); continue; } args[idx] = f; args[idx + 1] = hash.get(f); idx += 2; } return redis.hmset(key, field, value, args); } catch (Exception e) { throw new JedisException(e); } } @Override public List hmget(final String key, final String... fields) { try { String field = fields[0]; String[] f = new String[fields.length - 1]; for (int idx = 1; idx < fields.length; ++idx) { f[idx - 1] = fields[idx]; } return redis.hmget(key, field, f); } catch (Exception e) { throw new JedisException(e); } } @Override public Long hincrBy(final String key, final String field, final long value) { try { return redis.hincrby(key, field, value); } catch (Exception e) { throw new JedisException(e); } } @Override public Double hincrByFloat(final String key, final String field, final double value) { try { return Double.parseDouble(redis.hincrbyfloat(key, field, value)); } catch (Exception e) { throw new JedisException(e); } } @Override public Boolean hexists(final String key, final String field) { try { return redis.hexists(key, field); } catch (Exception e) { throw new JedisException(e); } } @Override public Long hdel(final String key, final String... fields) { try { String field = fields[0]; String[] f = new String[fields.length - 1]; for (int idx = 1; idx < fields.length; ++idx) { f[idx - 1] = fields[idx]; } return redis.hdel(key, field, f); } catch (Exception e) { throw new JedisException(e); } } @Override public Long hlen(final String key) { try { return redis.hlen(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Set hkeys(final String key) { try { return redis.hkeys(key); } catch (Exception e) { throw new JedisException(e); } } @Override public List hvals(final String key) { try { return redis.hvals(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Map hgetAll(final String key) { try { return redis.hgetall(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long rpush(final String key, final String... strings) { try { String element = strings[0]; String[] elements = new String[strings.length - 1]; for (int idx = 1; idx < strings.length; ++idx) { elements[idx - 1] = strings[idx]; } return redis.rpush(key, element, elements); } catch (Exception e) { throw new JedisException(e); } } @Override public Long lpush(final String key, final String... strings) { try { String element = strings[0]; String[] elements = new String[strings.length - 1]; for (int idx = 1; idx < strings.length; ++idx) { elements[idx - 1] = strings[idx]; } return redis.lpush(key, element, elements); } catch (Exception e) { throw new JedisException(e); } } @Override public Long llen(final String key) { try { return redis.llen(key); } catch (Exception e) { throw new JedisException(e); } } @Override public List lrange(final String key, final long start, final long end) { try { return redis.lrange(key, start, end); } catch (Exception e) { throw new JedisException(e); } } @Override public String ltrim(final String key, final long start, final long end) { try { return redis.ltrim(key, start, end); } catch (Exception e) { throw new JedisException(e); } } @Override public String lindex(final String key, final long index) { try { return redis.lindex(key, index); } catch (Exception e) { throw new JedisException(e); } } @Override public String lset(final String key, final long index, final String value) { try { return redis.lset(key, index, value); } catch (Exception e) { throw new JedisException(e); } } @Override public Long lrem(final String key, final long count, final String value) { try { return redis.lrem(key, count, value); } catch (Exception e) { throw new JedisException(e); } } @Override public String lpop(final String key) { try { return redis.lpop(key); } catch (Exception e) { throw new JedisException(e); } } @Override public String rpop(final String key) { try { return redis.rpop(key); } catch (Exception e) { throw new JedisException(e); } } @Override public String rpoplpush(final String srckey, final String dstkey) { try { return redis.rpoplpush(srckey, dstkey); } catch (Exception e) { throw new JedisException(e); } } @Override public Long sadd(final String key, final String... members) { try { String member = members[0]; String[] m = new String[members.length - 1]; for (int idx = 1; idx < members.length; ++idx) { m[idx - 1] = members[idx]; } return redis.sadd(key, member, m); } catch (Exception e) { throw new JedisException(e); } } @Override public Set smembers(final String key) { try { return redis.smembers(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long srem(final String key, final String... members) { try { String member = members[0]; String[] m = new String[members.length - 1]; for (int idx = 1; idx < members.length; ++idx) { m[idx - 1] = members[idx]; } return redis.srem(key, member, m); } catch (Exception e) { throw new JedisException(e); } } @Override public String spop(final String key) { try { return redis.spop(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Long smove(final String srckey, final String dstkey, final String member) { try { return redis.smove(srckey, dstkey, member) ? 1L : 0L; } catch (Exception e) { throw new JedisException(e); } } @Override public Long scard(final String key) { try { return redis.scard(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Boolean sismember(final String key, final String member) { try { return redis.sismember(key, member); } catch (Exception e) { throw new JedisException(e); } } @Override public Set sinter(final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sinter(key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public Long sinterstore(final String dstkey, final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sinterstore(dstkey, key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public Set sunion(final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sunion(key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public Long sunionstore(final String dstkey, final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sunionstore(dstkey, key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public Set sdiff(final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sdiff(key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public Long sdiffstore(final String dstkey, final String... keys) { try { String key = keys[0]; String[] k = new String[keys.length - 1]; for (int idx = 0; idx < keys.length; ++idx) { k[idx - 1] = keys[idx]; } return redis.sdiffstore(dstkey, key, k); } catch (Exception e) { throw new JedisException(e); } } @Override public String srandmember(final String key) { try { return redis.srandmember(key); } catch (Exception e) { throw new JedisException(e); } } @Override public List srandmember(final String key, final int count) { try { return redis.srandmember(key, count); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zadd(final String key, final double score, final String member) { try { return redis.zadd(key, new ZsetPair(member, score)); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zadd(String key, double score, String member, ZAddParams params) { try { if (params.getParam("xx") != null) { Double existing = redis.zscore(key, member); if (existing == null) { return 0L; } return redis.zadd(key, new ZsetPair(member, score)); } else { return redis.zadd(key, new ZsetPair(member, score)); } } catch (Exception e) { throw new JedisException(e); } } @Override public Long zadd(final String key, final Map scoreMembers) { try { Double score = null; String member = null; List scoresmembers = new ArrayList<>((scoreMembers.size() - 1) * 2); for (String m : scoreMembers.keySet()) { if (m == null) { member = m; score = scoreMembers.get(m); continue; } scoresmembers.add(new ZsetPair(m, scoreMembers.get(m))); } return redis.zadd( key, new ZsetPair(member, score), (ZsetPair[]) scoresmembers.toArray()); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrange(final String key, final long start, final long end) { try { return ZsetPair.members(redis.zrange(key, start, end)); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zrem(final String key, final String... members) { try { String member = members[0]; String[] ms = new String[members.length - 1]; for (int idx = 1; idx < members.length; ++idx) { ms[idx - 1] = members[idx]; } return redis.zrem(key, member, ms); } catch (Exception e) { throw new JedisException(e); } } @Override public Double zincrby(final String key, final double score, final String member) { try { return Double.parseDouble(redis.zincrby(key, score, member)); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zrank(final String key, final String member) { try { return redis.zrank(key, member); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zrevrank(final String key, final String member) { try { return redis.zrevrank(key, member); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrange(final String key, final long start, final long end) { try { return ZsetPair.members(redis.zrevrange(key, start, end)); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeWithScores(final String key, final long start, final long end) { try { return toTupleSet(redis.zrange(key, start, end, "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeWithScores(final String key, final long start, final long end) { try { return toTupleSet(redis.zrevrange(key, start, end, "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zcard(final String key) { try { return redis.zcard(key); } catch (Exception e) { throw new JedisException(e); } } @Override public Double zscore(final String key, final String member) { try { return redis.zscore(key, member); } catch (Exception e) { throw new JedisException(e); } } @Override public String watch(final String... keys) { try { for (String key : keys) { redis.watch(key); } return "OK"; } catch (Exception e) { throw new JedisException(e); } } @Override public Long zcount(final String key, final double min, final double max) { try { return redis.zcount(key, min, max); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zcount(final String key, final String min, final String max) { try { return redis.zcount(key, Double.parseDouble(min), Double.parseDouble(max)); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScore(final String key, final double min, final double max) { try { return ZsetPair.members( redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScore(final String key, final String min, final String max) { try { return ZsetPair.members(redis.zrangebyscore(key, min, max)); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScore( final String key, final double min, final double max, final int offset, final int count) { try { return ZsetPair.members( redis.zrangebyscore( key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), String.valueOf(count))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScore( final String key, final String min, final String max, final int offset, final int count) { try { return ZsetPair.members( redis.zrangebyscore( key, min, max, "limit", String.valueOf(offset), String.valueOf(count))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScoreWithScores( final String key, final double min, final double max) { try { return toTupleSet( redis.zrangebyscore( key, String.valueOf(min), String.valueOf(max), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScoreWithScores( final String key, final String min, final String max) { try { return toTupleSet(redis.zrangebyscore(key, min, max, "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScoreWithScores( final String key, final double min, final double max, final int offset, final int count) { try { return toTupleSet( redis.zrangebyscore( key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), String.valueOf(count), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrangeByScoreWithScores( final String key, final String min, final String max, final int offset, final int count) { try { return toTupleSet( redis.zrangebyscore( key, min, max, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScore(final String key, final double max, final double min) { try { return ZsetPair.members( redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScore(final String key, final String max, final String min) { try { return ZsetPair.members(redis.zrevrangebyscore(key, max, min)); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScore( final String key, final double max, final double min, final int offset, final int count) { try { return ZsetPair.members( redis.zrevrangebyscore( key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), String.valueOf(count))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScoreWithScores( final String key, final double max, final double min) { try { return toTupleSet( redis.zrevrangebyscore( key, String.valueOf(max), String.valueOf(min), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScoreWithScores( final String key, final double max, final double min, final int offset, final int count) { try { return toTupleSet( redis.zrevrangebyscore( key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), String.valueOf(count), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScoreWithScores( final String key, final String max, final String min, final int offset, final int count) { try { return toTupleSet( redis.zrevrangebyscore( key, max, min, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScore( final String key, final String max, final String min, final int offset, final int count) { try { return ZsetPair.members( redis.zrevrangebyscore( key, max, min, "limit", String.valueOf(offset), String.valueOf(count))); } catch (Exception e) { throw new JedisException(e); } } @Override public Set zrevrangeByScoreWithScores( final String key, final String max, final String min) { try { return toTupleSet(redis.zrevrangebyscore(key, max, min, "withscores")); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zremrangeByRank(final String key, final long start, final long end) { try { return redis.zremrangebyrank(key, start, end); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zremrangeByScore(final String key, final double start, final double end) { try { return redis.zremrangebyscore(key, String.valueOf(start), String.valueOf(end)); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zremrangeByScore(final String key, final String start, final String end) { try { return redis.zremrangebyscore(key, start, end); } catch (Exception e) { throw new JedisException(e); } } @Override public Long zunionstore(final String dstkey, final String... sets) { try { return redis.zunionstore(dstkey, sets.length, sets); } catch (Exception e) { throw new JedisException(e); } } @Override public ScanResult sscan(String key, String cursor, ScanParams params) { try { org.rarefiedredis.redis.ScanResult> sr = redis.sscan(key, Long.parseLong(cursor), "count", "1000000"); List list = new ArrayList<>(sr.results); return new ScanResult<>("0", list); } catch (Exception e) { throw new JedisException(e); } } public ScanResult> hscan(final String key, final String cursor) { try { org.rarefiedredis.redis.ScanResult> mockr = redis.hscan(key, Long.parseLong(cursor), "count", "1000000"); Map results = mockr.results; List> list = new ArrayList<>(results.entrySet()); return new ScanResult<>("0", list); } catch (Exception e) { throw new JedisException(e); } } public ScanResult zscan(final String key, final String cursor) { try { org.rarefiedredis.redis.ScanResult> sr = redis.zscan(key, Long.parseLong(cursor), "count", "1000000"); List list = new ArrayList<>(sr.results); List tl = new LinkedList<>(); list.forEach(p -> tl.add(new Tuple(p.member, p.score))); return new ScanResult<>("0", tl); } catch (Exception e) { throw new JedisException(e); } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisProxy.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Component; import com.netflix.conductor.redis.config.AnyRedisCondition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.Tuple; import redis.clients.jedis.commands.JedisCommands; import redis.clients.jedis.params.ZAddParams; import static com.netflix.conductor.redis.config.RedisCommonConfiguration.DEFAULT_CLIENT_INJECTION_NAME; /** Proxy for the {@link JedisCommands} object. */ @Component @Conditional(AnyRedisCondition.class) public class JedisProxy { private static final Logger LOGGER = LoggerFactory.getLogger(JedisProxy.class); protected JedisCommands jedisCommands; public JedisProxy(@Qualifier(DEFAULT_CLIENT_INJECTION_NAME) JedisCommands jedisCommands) { this.jedisCommands = jedisCommands; } public Set zrange(String key, long start, long end) { return jedisCommands.zrange(key, start, end); } public Set zrangeByScoreWithScores(String key, double maxScore, int count) { return jedisCommands.zrangeByScoreWithScores(key, 0, maxScore, 0, count); } public Set zrangeByScore(String key, double maxScore, int count) { return jedisCommands.zrangeByScore(key, 0, maxScore, 0, count); } public Set zrangeByScore(String key, double minScore, double maxScore, int count) { return jedisCommands.zrangeByScore(key, minScore, maxScore, 0, count); } public ScanResult zscan(String key, int cursor) { return jedisCommands.zscan(key, "" + cursor); } public String get(String key) { return jedisCommands.get(key); } public Long zcard(String key) { return jedisCommands.zcard(key); } public Long del(String key) { return jedisCommands.del(key); } public Long zrem(String key, String member) { return jedisCommands.zrem(key, member); } public long zremrangeByScore(String key, String start, String end) { return jedisCommands.zremrangeByScore(key, start, end); } public long zcount(String key, double min, double max) { return jedisCommands.zcount(key, min, max); } public String set(String key, String value) { return jedisCommands.set(key, value); } public Long setnx(String key, String value) { return jedisCommands.setnx(key, value); } public Long zadd(String key, double score, String member) { return jedisCommands.zadd(key, score, member); } public Long zaddnx(String key, double score, String member) { ZAddParams params = ZAddParams.zAddParams().nx(); return jedisCommands.zadd(key, score, member, params); } public Long hset(String key, String field, String value) { return jedisCommands.hset(key, field, value); } public Long hsetnx(String key, String field, String value) { return jedisCommands.hsetnx(key, field, value); } public Long hlen(String key) { return jedisCommands.hlen(key); } public String hget(String key, String field) { return jedisCommands.hget(key, field); } public Optional optionalHget(String key, String field) { return Optional.ofNullable(jedisCommands.hget(key, field)); } public Map hscan(String key, int count) { Map m = new HashMap<>(); int cursor = 0; do { ScanResult> scanResult = jedisCommands.hscan(key, "" + cursor); cursor = Integer.parseInt(scanResult.getCursor()); for (Entry r : scanResult.getResult()) { m.put(r.getKey(), r.getValue()); } if (m.size() > count) { break; } } while (cursor > 0); return m; } public Map hgetAll(String key) { Map m = new HashMap<>(); int cursor = 0; do { ScanResult> scanResult = jedisCommands.hscan(key, "" + cursor); cursor = Integer.parseInt(scanResult.getCursor()); for (Entry r : scanResult.getResult()) { m.put(r.getKey(), r.getValue()); } } while (cursor > 0); return m; } public List hvals(String key) { LOGGER.trace("hvals {}", key); return jedisCommands.hvals(key); } public Set hkeys(String key) { LOGGER.trace("hkeys {}", key); Set keys = new HashSet<>(); int cursor = 0; do { ScanResult> sr = jedisCommands.hscan(key, "" + cursor); cursor = Integer.parseInt(sr.getCursor()); List> result = sr.getResult(); for (Entry e : result) { keys.add(e.getKey()); } } while (cursor > 0); return keys; } public Long hdel(String key, String... fields) { LOGGER.trace("hdel {} {}", key, fields[0]); return jedisCommands.hdel(key, fields); } public Long expire(String key, int seconds) { return jedisCommands.expire(key, seconds); } public Boolean hexists(String key, String field) { return jedisCommands.hexists(key, field); } public Long sadd(String key, String value) { LOGGER.trace("sadd {} {}", key, value); return jedisCommands.sadd(key, value); } public Long srem(String key, String member) { LOGGER.trace("srem {} {}", key, member); return jedisCommands.srem(key, member); } public boolean sismember(String key, String member) { return jedisCommands.sismember(key, member); } public Set smembers(String key) { LOGGER.trace("smembers {}", key); Set r = new HashSet<>(); int cursor = 0; ScanParams sp = new ScanParams(); sp.count(50); do { ScanResult scanResult = jedisCommands.sscan(key, "" + cursor, sp); cursor = Integer.parseInt(scanResult.getCursor()); r.addAll(scanResult.getResult()); } while (cursor > 0); return r; } public Long scard(String key) { return jedisCommands.scard(key); } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisSentinel.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import redis.clients.jedis.BitPosParams; import redis.clients.jedis.GeoCoordinate; import redis.clients.jedis.GeoRadiusResponse; import redis.clients.jedis.GeoUnit; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPoolAbstract; import redis.clients.jedis.ListPosition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.SortingParams; import redis.clients.jedis.StreamConsumersInfo; import redis.clients.jedis.StreamEntry; import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.StreamGroupInfo; import redis.clients.jedis.StreamInfo; import redis.clients.jedis.StreamPendingEntry; import redis.clients.jedis.Tuple; import redis.clients.jedis.commands.JedisCommands; import redis.clients.jedis.params.GeoRadiusParam; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.params.ZAddParams; import redis.clients.jedis.params.ZIncrByParams; public class JedisSentinel implements JedisCommands { private final JedisPoolAbstract jedisPool; public JedisSentinel(JedisPoolAbstract jedisPool) { this.jedisPool = jedisPool; } @Override public String set(String key, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.set(key, value); } } @Override public String set(String key, String value, SetParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.set(key, value, params); } } @Override public String get(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.get(key); } } @Override public Boolean exists(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.exists(key); } } @Override public Long persist(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.persist(key); } } @Override public String type(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.type(key); } } @Override public byte[] dump(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.dump(key); } } @Override public String restore(String key, int ttl, byte[] serializedValue) { try (Jedis jedis = jedisPool.getResource()) { return jedis.restore(key, ttl, serializedValue); } } @Override public String restoreReplace(String key, int ttl, byte[] serializedValue) { try (Jedis jedis = jedisPool.getResource()) { return jedis.restoreReplace(key, ttl, serializedValue); } } @Override public Long expire(String key, int seconds) { try (Jedis jedis = jedisPool.getResource()) { return jedis.expire(key, seconds); } } @Override public Long pexpire(String key, long milliseconds) { try (Jedis jedis = jedisPool.getResource()) { return jedis.pexpire(key, milliseconds); } } @Override public Long expireAt(String key, long unixTime) { try (Jedis jedis = jedisPool.getResource()) { return jedis.expireAt(key, unixTime); } } @Override public Long pexpireAt(String key, long millisecondsTimestamp) { try (Jedis jedis = jedisPool.getResource()) { return jedis.pexpireAt(key, millisecondsTimestamp); } } @Override public Long ttl(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.ttl(key); } } @Override public Long pttl(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.pttl(key); } } @Override public Long touch(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.touch(key); } } @Override public Boolean setbit(String key, long offset, boolean value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.setbit(key, offset, value); } } @Override public Boolean setbit(String key, long offset, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.setbit(key, offset, value); } } @Override public Boolean getbit(String key, long offset) { try (Jedis jedis = jedisPool.getResource()) { return jedis.getbit(key, offset); } } @Override public Long setrange(String key, long offset, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.setrange(key, offset, value); } } @Override public String getrange(String key, long startOffset, long endOffset) { try (Jedis jedis = jedisPool.getResource()) { return jedis.getrange(key, startOffset, endOffset); } } @Override public String getSet(String key, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.getSet(key, value); } } @Override public Long setnx(String key, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.setnx(key, value); } } @Override public String setex(String key, int seconds, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.setex(key, seconds, value); } } @Override public String psetex(String key, long milliseconds, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.psetex(key, milliseconds, value); } } @Override public Long decrBy(String key, long integer) { try (Jedis jedis = jedisPool.getResource()) { return jedis.decrBy(key, integer); } } @Override public Long decr(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.decr(key); } } @Override public Long incrBy(String key, long integer) { try (Jedis jedis = jedisPool.getResource()) { return jedis.incrBy(key, integer); } } @Override public Double incrByFloat(String key, double value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.incrByFloat(key, value); } } @Override public Long incr(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.incr(key); } } @Override public Long append(String key, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.append(key, value); } } @Override public String substr(String key, int start, int end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.substr(key, start, end); } } @Override public Long hset(String key, String field, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hset(key, field, value); } } @Override public Long hset(String key, Map hash) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hset(key, hash); } } @Override public String hget(String key, String field) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hget(key, field); } } @Override public Long hsetnx(String key, String field, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hsetnx(key, field, value); } } @Override public String hmset(String key, Map hash) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hmset(key, hash); } } @Override public List hmget(String key, String... fields) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hmget(key, fields); } } @Override public Long hincrBy(String key, String field, long value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hincrBy(key, field, value); } } @Override public Double hincrByFloat(String key, String field, double value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hincrByFloat(key, field, value); } } @Override public Boolean hexists(String key, String field) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hexists(key, field); } } @Override public Long hdel(String key, String... field) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hdel(key, field); } } @Override public Long hlen(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hlen(key); } } @Override public Set hkeys(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hkeys(key); } } @Override public List hvals(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hvals(key); } } @Override public Map hgetAll(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hgetAll(key); } } @Override public Long rpush(String key, String... string) { try (Jedis jedis = jedisPool.getResource()) { return jedis.rpush(key, string); } } @Override public Long lpush(String key, String... string) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lpush(key, string); } } @Override public Long llen(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.llen(key); } } @Override public List lrange(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lrange(key, start, end); } } @Override public String ltrim(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.ltrim(key, start, end); } } @Override public String lindex(String key, long index) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lindex(key, index); } } @Override public String lset(String key, long index, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lset(key, index, value); } } @Override public Long lrem(String key, long count, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lrem(key, count, value); } } @Override public String lpop(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lpop(key); } } @Override public String rpop(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.rpop(key); } } @Override public Long sadd(String key, String... member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sadd(key, member); } } @Override public Set smembers(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.smembers(key); } } @Override public Long srem(String key, String... member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.srem(key, member); } } @Override public String spop(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.spop(key); } } @Override public Set spop(String key, long count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.spop(key, count); } } @Override public Long scard(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.scard(key); } } @Override public Boolean sismember(String key, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sismember(key, member); } } @Override public String srandmember(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.srandmember(key); } } @Override public List srandmember(String key, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.srandmember(key, count); } } @Override public Long strlen(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.strlen(key); } } @Override public Long zadd(String key, double score, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zadd(key, score, member); } } @Override public Long zadd(String key, double score, String member, ZAddParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zadd(key, score, member, params); } } @Override public Long zadd(String key, Map scoreMembers) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zadd(key, scoreMembers); } } @Override public Long zadd(String key, Map scoreMembers, ZAddParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zadd(key, scoreMembers, params); } } @Override public Set zrange(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrange(key, start, end); } } @Override public Long zrem(String key, String... member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrem(key, member); } } @Override public Double zincrby(String key, double score, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zincrby(key, score, member); } } @Override public Double zincrby(String key, double score, String member, ZIncrByParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zincrby(key, score, member, params); } } @Override public Long zrank(String key, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrank(key, member); } } @Override public Long zrevrank(String key, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrank(key, member); } } @Override public Set zrevrange(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrange(key, start, end); } } @Override public Set zrangeWithScores(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeWithScores(key, start, end); } } @Override public Set zrevrangeWithScores(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeWithScores(key, start, end); } } @Override public Long zcard(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zcard(key); } } @Override public Double zscore(String key, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zscore(key, member); } } @Override public Tuple zpopmax(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zpopmax(key); } } @Override public Set zpopmax(String key, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zpopmax(key, count); } } @Override public Tuple zpopmin(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zpopmin(key); } } @Override public Set zpopmin(String key, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zpopmin(key, count); } } @Override public List sort(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sort(key); } } @Override public List sort(String key, SortingParams sortingParameters) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sort(key, sortingParameters); } } @Override public Long zcount(String key, double min, double max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zcount(key, min, max); } } @Override public Long zcount(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zcount(key, min, max); } } @Override public Set zrangeByScore(String key, double min, double max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScore(key, min, max); } } @Override public Set zrangeByScore(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScore(key, min, max); } } @Override public Set zrevrangeByScore(String key, double max, double min) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScore(key, max, min); } } @Override public Set zrangeByScore(String key, double min, double max, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScore(key, min, max, offset, count); } } @Override public Set zrevrangeByScore(String key, String max, String min) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScore(key, max, min); } } @Override public Set zrangeByScore(String key, String min, String max, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScore(key, min, max, offset, count); } } @Override public Set zrevrangeByScore(String key, double max, double min, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScore(key, max, min, offset, count); } } @Override public Set zrangeByScoreWithScores(String key, double min, double max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScoreWithScores(key, min, max); } } @Override public Set zrevrangeByScoreWithScores(String key, double max, double min) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScoreWithScores(key, max, min); } } @Override public Set zrangeByScoreWithScores( String key, double min, double max, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScoreWithScores(key, min, max, offset, count); } } @Override public Set zrevrangeByScore(String key, String max, String min, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScore(key, max, min, offset, count); } } @Override public Set zrangeByScoreWithScores(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScoreWithScores(key, min, max); } } @Override public Set zrevrangeByScoreWithScores(String key, String max, String min) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScoreWithScores(key, max, min); } } @Override public Set zrangeByScoreWithScores( String key, String min, String max, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByScoreWithScores(key, min, max, offset, count); } } @Override public Set zrevrangeByScoreWithScores( String key, double max, double min, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScoreWithScores(key, max, min, offset, count); } } @Override public Set zrevrangeByScoreWithScores( String key, String max, String min, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByScoreWithScores(key, max, min, offset, count); } } @Override public Long zremrangeByRank(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zremrangeByRank(key, start, end); } } @Override public Long zremrangeByScore(String key, double start, double end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zremrangeByScore(key, start, end); } } @Override public Long zremrangeByScore(String key, String start, String end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zremrangeByScore(key, start, end); } } @Override public Long zlexcount(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zlexcount(key, min, max); } } @Override public Set zrangeByLex(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByLex(key, min, max); } } @Override public Set zrangeByLex(String key, String min, String max, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrangeByLex(key, min, max, offset, count); } } @Override public Set zrevrangeByLex(String key, String max, String min) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByLex(key, max, min); } } @Override public Set zrevrangeByLex(String key, String max, String min, int offset, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zrevrangeByLex(key, max, min, offset, count); } } @Override public Long zremrangeByLex(String key, String min, String max) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zremrangeByLex(key, min, max); } } @Override public Long linsert(String key, ListPosition where, String pivot, String value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.linsert(key, where, pivot, value); } } @Override public Long lpushx(String key, String... string) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lpushx(key, string); } } @Override public Long rpushx(String key, String... string) { try (Jedis jedis = jedisPool.getResource()) { return jedis.rpushx(key, string); } } @Override public List blpop(int timeout, String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.blpop(timeout, key); } } @Override public List brpop(int timeout, String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.brpop(timeout, key); } } @Override public Long del(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.del(key); } } @Override public Long unlink(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.unlink(key); } } @Override public String echo(String string) { try (Jedis jedis = jedisPool.getResource()) { return jedis.echo(string); } } @Override public Long move(String key, int dbIndex) { try (Jedis jedis = jedisPool.getResource()) { return jedis.move(key, dbIndex); } } @Override public Long bitcount(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitcount(key); } } @Override public Long bitcount(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitcount(key, start, end); } } @Override public Long bitpos(String key, boolean value) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitpos(key, value); } } @Override public Long bitpos(String key, boolean value, BitPosParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitpos(key, value, params); } } @Override public ScanResult> hscan(String key, String cursor) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hscan(key, cursor); } } @Override public ScanResult> hscan(String key, String cursor, ScanParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hscan(key, cursor, params); } } @Override public ScanResult sscan(String key, String cursor) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sscan(key, cursor); } } @Override public ScanResult sscan(String key, String cursor, ScanParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.sscan(key, cursor, params); } } @Override public ScanResult zscan(String key, String cursor) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zscan(key, cursor); } } @Override public ScanResult zscan(String key, String cursor, ScanParams params) { try (Jedis jedis = jedisPool.getResource()) { return jedis.zscan(key, cursor, params); } } @Override public Long pfadd(String key, String... elements) { try (Jedis jedis = jedisPool.getResource()) { return jedis.pfadd(key, elements); } } @Override public long pfcount(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.pfcount(key); } } @Override public Long geoadd(String key, double longitude, double latitude, String member) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geoadd(key, longitude, latitude, member); } } @Override public Long geoadd(String key, Map memberCoordinateMap) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geoadd(key, memberCoordinateMap); } } @Override public Double geodist(String key, String member1, String member2) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geodist(key, member1, member2); } } @Override public Double geodist(String key, String member1, String member2, GeoUnit unit) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geodist(key, member1, member2, unit); } } @Override public List geohash(String key, String... members) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geohash(key, members); } } @Override public List geopos(String key, String... members) { try (Jedis jedis = jedisPool.getResource()) { return jedis.geopos(key, members); } } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadius(key, longitude, latitude, radius, unit); } } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusReadonly(key, longitude, latitude, radius, unit); } } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadius(key, longitude, latitude, radius, unit, param); } } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusReadonly(key, longitude, latitude, radius, unit, param); } } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusByMember(key, member, radius, unit); } } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusByMemberReadonly(key, member, radius, unit); } } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusByMember(key, member, radius, unit, param); } } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { try (Jedis jedis = jedisPool.getResource()) { return jedis.georadiusByMemberReadonly(key, member, radius, unit, param); } } @Override public List bitfield(String key, String... arguments) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitfield(key, arguments); } } @Override public List bitfieldReadonly(String key, String... arguments) { try (Jedis jedis = jedisPool.getResource()) { return jedis.bitfieldReadonly(key, arguments); } } @Override public Long hstrlen(String key, String field) { try (Jedis jedis = jedisPool.getResource()) { return jedis.hstrlen(key, field); } } @Override public StreamEntryID xadd(String key, StreamEntryID id, Map hash) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xadd(key, id, hash); } } @Override public StreamEntryID xadd( String key, StreamEntryID id, Map hash, long maxLen, boolean approximateLength) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xadd(key, id, hash, maxLen, approximateLength); } } @Override public Long xlen(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xlen(key); } } @Override public List xrange(String key, StreamEntryID start, StreamEntryID end, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xrange(key, start, end, count); } } @Override public List xrevrange( String key, StreamEntryID end, StreamEntryID start, int count) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xrevrange(key, end, start, count); } } @Override public long xack(String key, String group, StreamEntryID... ids) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xack(key, group, ids); } } @Override public String xgroupCreate(String key, String groupname, StreamEntryID id, boolean makeStream) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xgroupCreate(key, groupname, id, makeStream); } } @Override public String xgroupSetID(String key, String groupname, StreamEntryID id) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xgroupSetID(key, groupname, id); } } @Override public long xgroupDestroy(String key, String groupname) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xgroupDestroy(key, groupname); } } @Override public Long xgroupDelConsumer(String key, String groupname, String consumername) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xgroupDelConsumer(key, groupname, consumername); } } @Override public List xpending( String key, String groupname, StreamEntryID start, StreamEntryID end, int count, String consumername) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xpending(key, groupname, start, end, count, consumername); } } @Override public long xdel(String key, StreamEntryID... ids) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xdel(key, ids); } } @Override public long xtrim(String key, long maxLen, boolean approximate) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xtrim(key, maxLen, approximate); } } @Override public List xclaim( String key, String group, String consumername, long minIdleTime, long newIdleTime, int retries, boolean force, StreamEntryID... ids) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xclaim( key, group, consumername, minIdleTime, newIdleTime, retries, force, ids); } } @Override public StreamInfo xinfoStream(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xinfoStream(key); } } @Override public List xinfoGroup(String key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xinfoGroup(key); } } @Override public List xinfoConsumers(String key, String group) { try (Jedis jedis = jedisPool.getResource()) { return jedis.xinfoConsumers(key, group); } } } ================================================ FILE: redis-persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisStandalone.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import redis.clients.jedis.BitPosParams; import redis.clients.jedis.GeoCoordinate; import redis.clients.jedis.GeoRadiusResponse; import redis.clients.jedis.GeoUnit; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.ListPosition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.SortingParams; import redis.clients.jedis.StreamConsumersInfo; import redis.clients.jedis.StreamEntry; import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.StreamGroupInfo; import redis.clients.jedis.StreamInfo; import redis.clients.jedis.StreamPendingEntry; import redis.clients.jedis.Tuple; import redis.clients.jedis.commands.JedisCommands; import redis.clients.jedis.params.GeoRadiusParam; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.params.ZAddParams; import redis.clients.jedis.params.ZIncrByParams; /** A {@link JedisCommands} implementation that delegates to {@link JedisPool}. */ public class JedisStandalone implements JedisCommands { private final JedisPool jedisPool; public JedisStandalone(JedisPool jedisPool) { this.jedisPool = jedisPool; } private R executeInJedis(Function function) { try (Jedis jedis = jedisPool.getResource()) { return function.apply(jedis); } } @Override public String set(String key, String value) { return executeInJedis(jedis -> jedis.set(key, value)); } @Override public String set(String key, String value, SetParams params) { return executeInJedis(jedis -> jedis.set(key, value, params)); } @Override public String get(String key) { return executeInJedis(jedis -> jedis.get(key)); } @Override public Boolean exists(String key) { return executeInJedis(jedis -> jedis.exists(key)); } @Override public Long persist(String key) { return executeInJedis(jedis -> jedis.persist(key)); } @Override public String type(String key) { return executeInJedis(jedis -> jedis.type(key)); } @Override public byte[] dump(String key) { return executeInJedis(jedis -> jedis.dump(key)); } @Override public String restore(String key, int ttl, byte[] serializedValue) { return executeInJedis(jedis -> jedis.restore(key, ttl, serializedValue)); } @Override public String restoreReplace(String key, int ttl, byte[] serializedValue) { return executeInJedis(jedis -> jedis.restoreReplace(key, ttl, serializedValue)); } @Override public Long expire(String key, int seconds) { return executeInJedis(jedis -> jedis.expire(key, seconds)); } @Override public Long pexpire(String key, long milliseconds) { return executeInJedis(jedis -> jedis.pexpire(key, milliseconds)); } @Override public Long expireAt(String key, long unixTime) { return executeInJedis(jedis -> jedis.expireAt(key, unixTime)); } @Override public Long pexpireAt(String key, long millisecondsTimestamp) { return executeInJedis(jedis -> jedis.pexpireAt(key, millisecondsTimestamp)); } @Override public Long ttl(String key) { return executeInJedis(jedis -> jedis.ttl(key)); } @Override public Long pttl(String key) { return executeInJedis(jedis -> jedis.pttl(key)); } @Override public Long touch(String key) { return executeInJedis(jedis -> jedis.touch(key)); } @Override public Boolean setbit(String key, long offset, boolean value) { return executeInJedis(jedis -> jedis.setbit(key, offset, value)); } @Override public Boolean setbit(String key, long offset, String value) { return executeInJedis(jedis -> jedis.setbit(key, offset, value)); } @Override public Boolean getbit(String key, long offset) { return executeInJedis(jedis -> jedis.getbit(key, offset)); } @Override public Long setrange(String key, long offset, String value) { return executeInJedis(jedis -> jedis.setrange(key, offset, value)); } @Override public String getrange(String key, long startOffset, long endOffset) { return executeInJedis(jedis -> jedis.getrange(key, startOffset, endOffset)); } @Override public String getSet(String key, String value) { return executeInJedis(jedis -> jedis.getSet(key, value)); } @Override public Long setnx(String key, String value) { return executeInJedis(jedis -> jedis.setnx(key, value)); } @Override public String setex(String key, int seconds, String value) { return executeInJedis(jedis -> jedis.setex(key, seconds, value)); } @Override public String psetex(String key, long milliseconds, String value) { return executeInJedis(jedis -> jedis.psetex(key, milliseconds, value)); } @Override public Long decrBy(String key, long decrement) { return executeInJedis(jedis -> jedis.decrBy(key, decrement)); } @Override public Long decr(String key) { return executeInJedis(jedis -> jedis.decr(key)); } @Override public Long incrBy(String key, long increment) { return executeInJedis(jedis -> jedis.incrBy(key, increment)); } @Override public Double incrByFloat(String key, double increment) { return executeInJedis(jedis -> jedis.incrByFloat(key, increment)); } @Override public Long incr(String key) { return executeInJedis(jedis -> jedis.incr(key)); } @Override public Long append(String key, String value) { return executeInJedis(jedis -> jedis.append(key, value)); } @Override public String substr(String key, int start, int end) { return executeInJedis(jedis -> jedis.substr(key, start, end)); } @Override public Long hset(String key, String field, String value) { return executeInJedis(jedis -> jedis.hset(key, field, value)); } @Override public Long hset(String key, Map hash) { return executeInJedis(jedis -> jedis.hset(key, hash)); } @Override public String hget(String key, String field) { return executeInJedis(jedis -> jedis.hget(key, field)); } @Override public Long hsetnx(String key, String field, String value) { return executeInJedis(jedis -> jedis.hsetnx(key, field, value)); } @Override public String hmset(String key, Map hash) { return executeInJedis(jedis -> jedis.hmset(key, hash)); } @Override public List hmget(String key, String... fields) { return executeInJedis(jedis -> jedis.hmget(key, fields)); } @Override public Long hincrBy(String key, String field, long value) { return executeInJedis(jedis -> jedis.hincrBy(key, field, value)); } @Override public Double hincrByFloat(String key, String field, double value) { return executeInJedis(jedis -> jedis.hincrByFloat(key, field, value)); } @Override public Boolean hexists(String key, String field) { return executeInJedis(jedis -> jedis.hexists(key, field)); } @Override public Long hdel(String key, String... field) { return executeInJedis(jedis -> jedis.hdel(key, field)); } @Override public Long hlen(String key) { return executeInJedis(jedis -> jedis.hlen(key)); } @Override public Set hkeys(String key) { return executeInJedis(jedis -> jedis.hkeys(key)); } @Override public List hvals(String key) { return executeInJedis(jedis -> jedis.hvals(key)); } @Override public Map hgetAll(String key) { return executeInJedis(jedis -> jedis.hgetAll(key)); } @Override public Long rpush(String key, String... string) { return executeInJedis(jedis -> jedis.rpush(key)); } @Override public Long lpush(String key, String... string) { return executeInJedis(jedis -> jedis.lpush(key, string)); } @Override public Long llen(String key) { return executeInJedis(jedis -> jedis.llen(key)); } @Override public List lrange(String key, long start, long stop) { return executeInJedis(jedis -> jedis.lrange(key, start, stop)); } @Override public String ltrim(String key, long start, long stop) { return executeInJedis(jedis -> jedis.ltrim(key, start, stop)); } @Override public String lindex(String key, long index) { return executeInJedis(jedis -> jedis.lindex(key, index)); } @Override public String lset(String key, long index, String value) { return executeInJedis(jedis -> jedis.lset(key, index, value)); } @Override public Long lrem(String key, long count, String value) { return executeInJedis(jedis -> jedis.lrem(key, count, value)); } @Override public String lpop(String key) { return executeInJedis(jedis -> jedis.lpop(key)); } @Override public String rpop(String key) { return executeInJedis(jedis -> jedis.rpop(key)); } @Override public Long sadd(String key, String... member) { return executeInJedis(jedis -> jedis.sadd(key, member)); } @Override public Set smembers(String key) { return executeInJedis(jedis -> jedis.smembers(key)); } @Override public Long srem(String key, String... member) { return executeInJedis(jedis -> jedis.srem(key, member)); } @Override public String spop(String key) { return executeInJedis(jedis -> jedis.spop(key)); } @Override public Set spop(String key, long count) { return executeInJedis(jedis -> jedis.spop(key, count)); } @Override public Long scard(String key) { return executeInJedis(jedis -> jedis.scard(key)); } @Override public Boolean sismember(String key, String member) { return executeInJedis(jedis -> jedis.sismember(key, member)); } @Override public String srandmember(String key) { return executeInJedis(jedis -> jedis.srandmember(key)); } @Override public List srandmember(String key, int count) { return executeInJedis(jedis -> jedis.srandmember(key, count)); } @Override public Long strlen(String key) { return executeInJedis(jedis -> jedis.strlen(key)); } @Override public Long zadd(String key, double score, String member) { return executeInJedis(jedis -> jedis.zadd(key, score, member)); } @Override public Long zadd(String key, double score, String member, ZAddParams params) { return executeInJedis(jedis -> jedis.zadd(key, score, member, params)); } @Override public Long zadd(String key, Map scoreMembers) { return executeInJedis(jedis -> jedis.zadd(key, scoreMembers)); } @Override public Long zadd(String key, Map scoreMembers, ZAddParams params) { return executeInJedis(jedis -> jedis.zadd(key, scoreMembers, params)); } @Override public Set zrange(String key, long start, long stop) { return executeInJedis(jedis -> jedis.zrange(key, start, stop)); } @Override public Long zrem(String key, String... members) { return executeInJedis(jedis -> jedis.zrem(key, members)); } @Override public Double zincrby(String key, double increment, String member) { return executeInJedis(jedis -> jedis.zincrby(key, increment, member)); } @Override public Double zincrby(String key, double increment, String member, ZIncrByParams params) { return executeInJedis(jedis -> jedis.zincrby(key, increment, member, params)); } @Override public Long zrank(String key, String member) { return executeInJedis(jedis -> jedis.zrank(key, member)); } @Override public Long zrevrank(String key, String member) { return executeInJedis(jedis -> jedis.zrevrank(key, member)); } @Override public Set zrevrange(String key, long start, long stop) { return executeInJedis(jedis -> jedis.zrevrange(key, start, stop)); } @Override public Set zrangeWithScores(String key, long start, long stop) { return executeInJedis(jedis -> jedis.zrangeWithScores(key, start, stop)); } @Override public Set zrevrangeWithScores(String key, long start, long stop) { return executeInJedis(jedis -> jedis.zrevrangeWithScores(key, start, stop)); } @Override public Long zcard(String key) { return executeInJedis(jedis -> jedis.zcard(key)); } @Override public Double zscore(String key, String member) { return executeInJedis(jedis -> jedis.zscore(key, member)); } @Override public Tuple zpopmax(String key) { return executeInJedis(jedis -> jedis.zpopmax(key)); } @Override public Set zpopmax(String key, int count) { return executeInJedis(jedis -> jedis.zpopmax(key, count)); } @Override public Tuple zpopmin(String key) { return executeInJedis(jedis -> jedis.zpopmin(key)); } @Override public Set zpopmin(String key, int count) { return executeInJedis(jedis -> jedis.zpopmin(key, count)); } @Override public List sort(String key) { return executeInJedis(jedis -> jedis.sort(key)); } @Override public List sort(String key, SortingParams sortingParameters) { return executeInJedis(jedis -> jedis.sort(key, sortingParameters)); } @Override public Long zcount(String key, double min, double max) { return executeInJedis(jedis -> jedis.zcount(key, min, max)); } @Override public Long zcount(String key, String min, String max) { return executeInJedis(jedis -> jedis.zcount(key, min, max)); } @Override public Set zrangeByScore(String key, double min, double max) { return executeInJedis(jedis -> jedis.zrangeByScore(key, min, max)); } @Override public Set zrangeByScore(String key, String min, String max) { return executeInJedis(jedis -> jedis.zrangeByScore(key, min, max)); } @Override public Set zrevrangeByScore(String key, double max, double min) { return executeInJedis(jedis -> jedis.zrevrangeByScore(key, max, min)); } @Override public Set zrangeByScore(String key, double min, double max, int offset, int count) { return executeInJedis(jedis -> jedis.zrangeByScore(key, min, max, offset, count)); } @Override public Set zrevrangeByScore(String key, String max, String min) { return executeInJedis(jedis -> jedis.zrevrangeByScore(key, max, min)); } @Override public Set zrangeByScore(String key, String min, String max, int offset, int count) { return executeInJedis(jedis -> jedis.zrangeByScore(key, min, max, offset, count)); } @Override public Set zrevrangeByScore(String key, double max, double min, int offset, int count) { return executeInJedis(jedis -> jedis.zrevrangeByScore(key, max, min, offset, count)); } @Override public Set zrangeByScoreWithScores(String key, double min, double max) { return executeInJedis(jedis -> jedis.zrangeByScoreWithScores(key, min, max)); } @Override public Set zrevrangeByScoreWithScores(String key, double max, double min) { return executeInJedis(jedis -> jedis.zrevrangeByScoreWithScores(key, max, min)); } @Override public Set zrangeByScoreWithScores( String key, double min, double max, int offset, int count) { return executeInJedis(jedis -> jedis.zrangeByScoreWithScores(key, min, max, offset, count)); } @Override public Set zrevrangeByScore(String key, String max, String min, int offset, int count) { return executeInJedis(jedis -> jedis.zrevrangeByScore(key, max, min, offset, count)); } @Override public Set zrangeByScoreWithScores(String key, String min, String max) { return executeInJedis(jedis -> jedis.zrangeByScoreWithScores(key, min, max)); } @Override public Set zrevrangeByScoreWithScores(String key, String max, String min) { return executeInJedis(jedis -> jedis.zrevrangeByScoreWithScores(key, max, min)); } @Override public Set zrangeByScoreWithScores( String key, String min, String max, int offset, int count) { return executeInJedis(jedis -> jedis.zrangeByScoreWithScores(key, min, max, offset, count)); } @Override public Set zrevrangeByScoreWithScores( String key, double max, double min, int offset, int count) { return executeInJedis( jedis -> jedis.zrevrangeByScoreWithScores(key, max, min, offset, count)); } @Override public Set zrevrangeByScoreWithScores( String key, String max, String min, int offset, int count) { return executeInJedis( jedis -> jedis.zrevrangeByScoreWithScores(key, max, min, offset, count)); } @Override public Long zremrangeByRank(String key, long start, long stop) { return executeInJedis(jedis -> jedis.zremrangeByRank(key, start, stop)); } @Override public Long zremrangeByScore(String key, double min, double max) { return executeInJedis(jedis -> jedis.zremrangeByScore(key, min, max)); } @Override public Long zremrangeByScore(String key, String min, String max) { return executeInJedis(jedis -> jedis.zremrangeByScore(key, min, max)); } @Override public Long zlexcount(String key, String min, String max) { return executeInJedis(jedis -> jedis.zlexcount(key, min, max)); } @Override public Set zrangeByLex(String key, String min, String max) { return executeInJedis(jedis -> jedis.zrangeByLex(key, min, max)); } @Override public Set zrangeByLex(String key, String min, String max, int offset, int count) { return executeInJedis(jedis -> jedis.zrangeByLex(key, min, max, offset, count)); } @Override public Set zrevrangeByLex(String key, String max, String min) { return executeInJedis(jedis -> jedis.zrevrangeByLex(key, max, min)); } @Override public Set zrevrangeByLex(String key, String max, String min, int offset, int count) { return executeInJedis(jedis -> jedis.zrevrangeByLex(key, max, min, offset, count)); } @Override public Long zremrangeByLex(String key, String min, String max) { return executeInJedis(jedis -> jedis.zremrangeByLex(key, min, max)); } @Override public Long linsert(String key, ListPosition where, String pivot, String value) { return executeInJedis(jedis -> jedis.linsert(key, where, pivot, value)); } @Override public Long lpushx(String key, String... string) { return executeInJedis(jedis -> jedis.lpushx(key, string)); } @Override public Long rpushx(String key, String... string) { return executeInJedis(jedis -> jedis.rpushx(key, string)); } @Override public List blpop(int timeout, String key) { return executeInJedis(jedis -> jedis.blpop(timeout, key)); } @Override public List brpop(int timeout, String key) { return executeInJedis(jedis -> jedis.brpop(timeout, key)); } @Override public Long del(String key) { return executeInJedis(jedis -> jedis.del(key)); } @Override public Long unlink(String key) { return executeInJedis(jedis -> jedis.unlink(key)); } @Override public String echo(String string) { return executeInJedis(jedis -> jedis.echo(string)); } @Override public Long move(String key, int dbIndex) { return executeInJedis(jedis -> jedis.move(key, dbIndex)); } @Override public Long bitcount(String key) { return executeInJedis(jedis -> jedis.bitcount(key)); } @Override public Long bitcount(String key, long start, long end) { return executeInJedis(jedis -> jedis.bitcount(key, start, end)); } @Override public Long bitpos(String key, boolean value) { return executeInJedis(jedis -> jedis.bitpos(key, value)); } @Override public Long bitpos(String key, boolean value, BitPosParams params) { return executeInJedis(jedis -> jedis.bitpos(key, value, params)); } @Override public ScanResult> hscan(String key, String cursor) { return executeInJedis(jedis -> jedis.hscan(key, cursor)); } @Override public ScanResult> hscan( String key, String cursor, ScanParams params) { return executeInJedis(jedis -> jedis.hscan(key, cursor, params)); } @Override public ScanResult sscan(String key, String cursor) { return executeInJedis(jedis -> jedis.sscan(key, cursor)); } @Override public ScanResult zscan(String key, String cursor) { return executeInJedis(jedis -> jedis.zscan(key, cursor)); } @Override public ScanResult zscan(String key, String cursor, ScanParams params) { return executeInJedis(jedis -> jedis.zscan(key, cursor, params)); } @Override public ScanResult sscan(String key, String cursor, ScanParams params) { return executeInJedis(jedis -> jedis.sscan(key, cursor, params)); } @Override public Long pfadd(String key, String... elements) { return executeInJedis(jedis -> jedis.pfadd(key, elements)); } @Override public long pfcount(String key) { return executeInJedis(jedis -> jedis.pfcount(key)); } @Override public Long geoadd(String key, double longitude, double latitude, String member) { return executeInJedis(jedis -> jedis.geoadd(key, longitude, latitude, member)); } @Override public Long geoadd(String key, Map memberCoordinateMap) { return executeInJedis(jedis -> jedis.geoadd(key, memberCoordinateMap)); } @Override public Double geodist(String key, String member1, String member2) { return executeInJedis(jedis -> jedis.geodist(key, member1, member2)); } @Override public Double geodist(String key, String member1, String member2, GeoUnit unit) { return executeInJedis(jedis -> jedis.geodist(key, member1, member2, unit)); } @Override public List geohash(String key, String... members) { return executeInJedis(jedis -> jedis.geohash(key, members)); } @Override public List geopos(String key, String... members) { return executeInJedis(jedis -> jedis.geopos(key, members)); } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit) { return executeInJedis(jedis -> jedis.georadius(key, longitude, latitude, radius, unit)); } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit) { return executeInJedis( jedis -> jedis.georadiusReadonly(key, longitude, latitude, radius, unit)); } @Override public List georadius( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { return executeInJedis( jedis -> jedis.georadius(key, longitude, latitude, radius, unit, param)); } @Override public List georadiusReadonly( String key, double longitude, double latitude, double radius, GeoUnit unit, GeoRadiusParam param) { return executeInJedis( jedis -> jedis.georadiusReadonly(key, longitude, latitude, radius, unit, param)); } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit) { return executeInJedis(jedis -> jedis.georadiusByMember(key, member, radius, unit)); } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit) { return executeInJedis(jedis -> jedis.georadiusByMemberReadonly(key, member, radius, unit)); } @Override public List georadiusByMember( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { return executeInJedis(jedis -> jedis.georadiusByMember(key, member, radius, unit, param)); } @Override public List georadiusByMemberReadonly( String key, String member, double radius, GeoUnit unit, GeoRadiusParam param) { return executeInJedis( jedis -> jedis.georadiusByMemberReadonly(key, member, radius, unit, param)); } @Override public List bitfield(String key, String... arguments) { return executeInJedis(jedis -> jedis.bitfield(key, arguments)); } @Override public List bitfieldReadonly(String key, String... arguments) { return executeInJedis(jedis -> jedis.bitfieldReadonly(key, arguments)); } @Override public Long hstrlen(String key, String field) { return executeInJedis(jedis -> jedis.hstrlen(key, field)); } @Override public StreamEntryID xadd(String key, StreamEntryID id, Map hash) { return executeInJedis(jedis -> jedis.xadd(key, id, hash)); } @Override public StreamEntryID xadd( String key, StreamEntryID id, Map hash, long maxLen, boolean approximateLength) { return executeInJedis(jedis -> jedis.xadd(key, id, hash, maxLen, approximateLength)); } @Override public Long xlen(String key) { return executeInJedis(jedis -> jedis.xlen(key)); } @Override public List xrange(String key, StreamEntryID start, StreamEntryID end, int count) { return executeInJedis(jedis -> jedis.xrange(key, start, end, count)); } @Override public List xrevrange( String key, StreamEntryID end, StreamEntryID start, int count) { return executeInJedis(jedis -> jedis.xrevrange(key, end, start, count)); } @Override public long xack(String key, String group, StreamEntryID... ids) { return executeInJedis(jedis -> jedis.xack(key, group, ids)); } @Override public String xgroupCreate(String key, String groupname, StreamEntryID id, boolean makeStream) { return executeInJedis(jedis -> jedis.xgroupCreate(key, groupname, id, makeStream)); } @Override public String xgroupSetID(String key, String groupname, StreamEntryID id) { return executeInJedis(jedis -> jedis.xgroupSetID(key, groupname, id)); } @Override public long xgroupDestroy(String key, String groupname) { return executeInJedis(jedis -> jedis.xgroupDestroy(key, groupname)); } @Override public Long xgroupDelConsumer(String key, String groupname, String consumername) { return executeInJedis(jedis -> jedis.hsetnx(key, groupname, consumername)); } @Override public List xpending( String key, String groupname, StreamEntryID start, StreamEntryID end, int count, String consumername) { return executeInJedis( jedis -> jedis.xpending(key, groupname, start, end, count, consumername)); } @Override public long xdel(String key, StreamEntryID... ids) { return executeInJedis(jedis -> jedis.xdel(key, ids)); } @Override public long xtrim(String key, long maxLen, boolean approximate) { return executeInJedis(jedis -> jedis.xtrim(key, maxLen, approximate)); } @Override public List xclaim( String key, String group, String consumername, long minIdleTime, long newIdleTime, int retries, boolean force, StreamEntryID... ids) { return executeInJedis( jedis -> jedis.xclaim( key, group, consumername, minIdleTime, newIdleTime, retries, force, ids)); } @Override public StreamInfo xinfoStream(String key) { return executeInJedis(jedis -> jedis.xinfoStream(key)); } @Override public List xinfoGroup(String key) { return executeInJedis(jedis -> jedis.xinfoGroup(key)); } @Override public List xinfoConsumers(String key, String group) { return executeInJedis(jedis -> jedis.xinfoConsumers(key, group)); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/config/utils/RedisQueuesShardingStrategyProviderTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.config.utils; import java.util.Collections; import org.junit.Test; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.dynoqueue.RedisQueuesShardingStrategyProvider; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.ShardSupplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class RedisQueuesShardingStrategyProviderTest { @Test public void testStrategy() { ShardSupplier shardSupplier = mock(ShardSupplier.class); doReturn("current").when(shardSupplier).getCurrentShard(); RedisQueuesShardingStrategyProvider.LocalOnlyStrategy strat = new RedisQueuesShardingStrategyProvider.LocalOnlyStrategy(shardSupplier); assertEquals("current", strat.getNextShard(Collections.emptyList(), new Message("a", "b"))); } @Test public void testProvider() { ShardSupplier shardSupplier = mock(ShardSupplier.class); RedisProperties properties = mock(RedisProperties.class); when(properties.getQueueShardingStrategy()).thenReturn("localOnly"); RedisQueuesShardingStrategyProvider stratProvider = new RedisQueuesShardingStrategyProvider(shardSupplier, properties); assertTrue( stratProvider.get() instanceof RedisQueuesShardingStrategyProvider.LocalOnlyStrategy); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/BaseDynoDAOTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class BaseDynoDAOTest { @Mock private JedisProxy jedisProxy; @Mock private ObjectMapper objectMapper; private RedisProperties properties; private ConductorProperties conductorProperties; private BaseDynoDAO baseDynoDAO; @Before public void setUp() { properties = mock(RedisProperties.class); conductorProperties = mock(ConductorProperties.class); this.baseDynoDAO = new BaseDynoDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Test public void testNsKey() { assertEquals("", baseDynoDAO.nsKey()); String[] keys = {"key1", "key2"}; assertEquals("key1.key2", baseDynoDAO.nsKey(keys)); when(properties.getWorkflowNamespacePrefix()).thenReturn("test"); assertEquals("test", baseDynoDAO.nsKey()); assertEquals("test.key1.key2", baseDynoDAO.nsKey(keys)); when(conductorProperties.getStack()).thenReturn("stack"); assertEquals("test.stack.key1.key2", baseDynoDAO.nsKey(keys)); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/DynoQueueDAOTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.dynoqueue.RedisQueuesShardingStrategyProvider; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; import redis.clients.jedis.commands.JedisCommands; import static com.netflix.conductor.redis.dynoqueue.RedisQueuesShardingStrategyProvider.LOCAL_ONLY_STRATEGY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class DynoQueueDAOTest { private QueueDAO queueDAO; @Before public void init() { RedisProperties properties = mock(RedisProperties.class); when(properties.getQueueShardingStrategy()).thenReturn(LOCAL_ONLY_STRATEGY); JedisCommands jedisMock = new JedisMock(); ShardSupplier shardSupplier = new ShardSupplier() { @Override public Set getQueueShards() { return new HashSet<>(Collections.singletonList("a")); } @Override public String getCurrentShard() { return "a"; } @Override public String getShardForHost(Host host) { return "a"; } }; ShardingStrategy shardingStrategy = new RedisQueuesShardingStrategyProvider(shardSupplier, properties).get(); RedisQueues redisQueues = new RedisQueues( jedisMock, jedisMock, "", shardSupplier, 60_000, 60_000, shardingStrategy); queueDAO = new DynoQueueDAO(redisQueues); } @Rule public ExpectedException expected = ExpectedException.none(); @Test public void test() { String queueName = "TestQueue"; long offsetTimeInSecond = 0; for (int i = 0; i < 10; i++) { String messageId = "msg" + i; queueDAO.push(queueName, messageId, offsetTimeInSecond); } int size = queueDAO.getSize(queueName); assertEquals(10, size); Map details = queueDAO.queuesDetail(); assertEquals(1, details.size()); assertEquals(10L, details.get(queueName).longValue()); for (int i = 0; i < 10; i++) { String messageId = "msg" + i; queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond); } List popped = queueDAO.pop(queueName, 10, 100); assertNotNull(popped); assertEquals(10, popped.size()); Map>> verbose = queueDAO.queuesDetailVerbose(); assertEquals(1, verbose.size()); long shardSize = verbose.get(queueName).get("a").get("size"); long unackedSize = verbose.get(queueName).get("a").get("uacked"); assertEquals(0, shardSize); assertEquals(10, unackedSize); popped.forEach(messageId -> queueDAO.ack(queueName, messageId)); verbose = queueDAO.queuesDetailVerbose(); assertEquals(1, verbose.size()); shardSize = verbose.get(queueName).get("a").get("size"); unackedSize = verbose.get(queueName).get("a").get("uacked"); assertEquals(0, shardSize); assertEquals(0, unackedSize); popped = queueDAO.pop(queueName, 10, 100); assertNotNull(popped); assertEquals(0, popped.size()); for (int i = 0; i < 10; i++) { String messageId = "msg" + i; queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond); } size = queueDAO.getSize(queueName); assertEquals(10, size); for (int i = 0; i < 10; i++) { String messageId = "msg" + i; queueDAO.remove(queueName, messageId); } size = queueDAO.getSize(queueName); assertEquals(0, size); for (int i = 0; i < 10; i++) { String messageId = "msg" + i; queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond); } queueDAO.flush(queueName); size = queueDAO.getSize(queueName); assertEquals(0, size); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAOTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.List; import java.util.UUID; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.events.EventHandler.Action; import com.netflix.conductor.common.metadata.events.EventHandler.Action.Type; import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import redis.clients.jedis.commands.JedisCommands; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class RedisEventHandlerDAOTest { private RedisEventHandlerDAO redisEventHandlerDAO; @Autowired private ObjectMapper objectMapper; @Before public void init() { ConductorProperties conductorProperties = mock(ConductorProperties.class); RedisProperties properties = mock(RedisProperties.class); JedisCommands jedisMock = new JedisMock(); JedisProxy jedisProxy = new JedisProxy(jedisMock); redisEventHandlerDAO = new RedisEventHandlerDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Test public void testEventHandlers() { String event1 = "SQS::arn:account090:sqstest1"; String event2 = "SQS::arn:account090:sqstest2"; EventHandler eventHandler = new EventHandler(); eventHandler.setName(UUID.randomUUID().toString()); eventHandler.setActive(false); Action action = new Action(); action.setAction(Type.start_workflow); action.setStart_workflow(new StartWorkflow()); action.getStart_workflow().setName("test_workflow"); eventHandler.getActions().add(action); eventHandler.setEvent(event1); redisEventHandlerDAO.addEventHandler(eventHandler); List allEventHandlers = redisEventHandlerDAO.getAllEventHandlers(); assertNotNull(allEventHandlers); assertEquals(1, allEventHandlers.size()); assertEquals(eventHandler.getName(), allEventHandlers.get(0).getName()); assertEquals(eventHandler.getEvent(), allEventHandlers.get(0).getEvent()); List byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event1, true); assertNotNull(byEvents); assertEquals(0, byEvents.size()); // event is marked as in-active eventHandler.setActive(true); eventHandler.setEvent(event2); redisEventHandlerDAO.updateEventHandler(eventHandler); allEventHandlers = redisEventHandlerDAO.getAllEventHandlers(); assertNotNull(allEventHandlers); assertEquals(1, allEventHandlers.size()); byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event1, true); assertNotNull(byEvents); assertEquals(0, byEvents.size()); byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event2, true); assertNotNull(byEvents); assertEquals(1, byEvents.size()); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/RedisExecutionDAOTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.time.Duration; import java.util.Collections; import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.ExecutionDAOTest; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import redis.clients.jedis.commands.JedisCommands; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class RedisExecutionDAOTest extends ExecutionDAOTest { private RedisExecutionDAO executionDAO; @Autowired private ObjectMapper objectMapper; @Before public void init() { ConductorProperties conductorProperties = mock(ConductorProperties.class); RedisProperties properties = mock(RedisProperties.class); when(properties.getEventExecutionPersistenceTTL()).thenReturn(Duration.ofSeconds(5)); JedisCommands jedisMock = new JedisMock(); JedisProxy jedisProxy = new JedisProxy(jedisMock); executionDAO = new RedisExecutionDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Test public void testCorrelateTaskToWorkflowInDS() { String workflowId = "workflowId"; String taskId = "taskId1"; String taskDefName = "task1"; TaskDef def = new TaskDef(); def.setName("task1"); def.setConcurrentExecLimit(1); TaskModel task = new TaskModel(); task.setTaskId(taskId); task.setWorkflowInstanceId(workflowId); task.setReferenceTaskName("ref_name"); task.setTaskDefName(taskDefName); task.setTaskType(taskDefName); task.setStatus(TaskModel.Status.IN_PROGRESS); List tasks = executionDAO.createTasks(Collections.singletonList(task)); assertNotNull(tasks); assertEquals(1, tasks.size()); executionDAO.correlateTaskToWorkflowInDS(taskId, workflowId); tasks = executionDAO.getTasksForWorkflow(workflowId); assertNotNull(tasks); assertEquals(workflowId, tasks.get(0).getWorkflowInstanceId()); assertEquals(taskId, tasks.get(0).getTaskId()); } @Override protected ExecutionDAO getExecutionDAO() { return executionDAO; } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/RedisMetadataDAOTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskDef.RetryLogic; import com.netflix.conductor.common.metadata.tasks.TaskDef.TimeoutPolicy; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import redis.clients.jedis.commands.JedisCommands; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class RedisMetadataDAOTest { private RedisMetadataDAO redisMetadataDAO; @Autowired private ObjectMapper objectMapper; @Before public void init() { ConductorProperties conductorProperties = mock(ConductorProperties.class); RedisProperties properties = mock(RedisProperties.class); when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); JedisCommands jedisMock = new JedisMock(); JedisProxy jedisProxy = new JedisProxy(jedisMock); redisMetadataDAO = new RedisMetadataDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Test(expected = ConflictException.class) public void testDup() { WorkflowDef def = new WorkflowDef(); def.setName("testDup"); def.setVersion(1); redisMetadataDAO.createWorkflowDef(def); redisMetadataDAO.createWorkflowDef(def); } @Test public void testWorkflowDefOperations() { WorkflowDef def = new WorkflowDef(); def.setName("test"); def.setVersion(1); def.setDescription("description"); def.setCreatedBy("unit_test"); def.setCreateTime(1L); def.setOwnerApp("ownerApp"); def.setUpdatedBy("unit_test2"); def.setUpdateTime(2L); redisMetadataDAO.createWorkflowDef(def); List all = redisMetadataDAO.getAllWorkflowDefs(); assertNotNull(all); assertEquals(1, all.size()); assertEquals("test", all.get(0).getName()); assertEquals(1, all.get(0).getVersion()); WorkflowDef found = redisMetadataDAO.getWorkflowDef("test", 1).get(); assertEquals(def, found); def.setVersion(2); redisMetadataDAO.createWorkflowDef(def); all = redisMetadataDAO.getAllWorkflowDefs(); assertNotNull(all); assertEquals(2, all.size()); assertEquals("test", all.get(0).getName()); assertEquals(1, all.get(0).getVersion()); found = redisMetadataDAO.getLatestWorkflowDef(def.getName()).get(); assertEquals(def.getName(), found.getName()); assertEquals(def.getVersion(), found.getVersion()); assertEquals(2, found.getVersion()); all = redisMetadataDAO.getAllVersions(def.getName()); assertNotNull(all); assertEquals(2, all.size()); assertEquals("test", all.get(0).getName()); assertEquals("test", all.get(1).getName()); assertEquals(1, all.get(0).getVersion()); assertEquals(2, all.get(1).getVersion()); def.setDescription("updated"); redisMetadataDAO.updateWorkflowDef(def); found = redisMetadataDAO.getWorkflowDef(def.getName(), def.getVersion()).get(); assertEquals(def.getDescription(), found.getDescription()); List allnames = redisMetadataDAO.findAll(); assertNotNull(allnames); assertEquals(1, allnames.size()); assertEquals(def.getName(), allnames.get(0)); redisMetadataDAO.removeWorkflowDef("test", 1); Optional deleted = redisMetadataDAO.getWorkflowDef("test", 1); assertFalse(deleted.isPresent()); redisMetadataDAO.removeWorkflowDef("test", 2); Optional latestDef = redisMetadataDAO.getLatestWorkflowDef("test"); assertFalse(latestDef.isPresent()); WorkflowDef[] workflowDefsArray = new WorkflowDef[3]; for (int i = 1; i <= 3; i++) { workflowDefsArray[i - 1] = new WorkflowDef(); workflowDefsArray[i - 1].setName("test"); workflowDefsArray[i - 1].setVersion(i); workflowDefsArray[i - 1].setDescription("description"); workflowDefsArray[i - 1].setCreatedBy("unit_test"); workflowDefsArray[i - 1].setCreateTime(1L); workflowDefsArray[i - 1].setOwnerApp("ownerApp"); workflowDefsArray[i - 1].setUpdatedBy("unit_test2"); workflowDefsArray[i - 1].setUpdateTime(2L); redisMetadataDAO.createWorkflowDef(workflowDefsArray[i - 1]); } redisMetadataDAO.removeWorkflowDef("test", 1); redisMetadataDAO.removeWorkflowDef("test", 2); WorkflowDef workflow = redisMetadataDAO.getLatestWorkflowDef("test").get(); assertEquals(workflow.getVersion(), 3); } @Test public void testGetAllWorkflowDefsLatestVersions() { WorkflowDef def = new WorkflowDef(); def.setName("test1"); def.setVersion(1); def.setDescription("description"); def.setCreatedBy("unit_test"); def.setCreateTime(1L); def.setOwnerApp("ownerApp"); def.setUpdatedBy("unit_test2"); def.setUpdateTime(2L); redisMetadataDAO.createWorkflowDef(def); def.setName("test2"); redisMetadataDAO.createWorkflowDef(def); def.setVersion(2); redisMetadataDAO.createWorkflowDef(def); def.setName("test3"); def.setVersion(1); redisMetadataDAO.createWorkflowDef(def); def.setVersion(2); redisMetadataDAO.createWorkflowDef(def); def.setVersion(3); redisMetadataDAO.createWorkflowDef(def); // Placed the values in a map because they might not be stored in order of defName. // To test, needed to confirm that the versions are correct for the definitions. Map allMap = redisMetadataDAO.getAllWorkflowDefsLatestVersions().stream() .collect(Collectors.toMap(WorkflowDef::getName, Function.identity())); assertNotNull(allMap); assertEquals(3, allMap.size()); assertEquals(1, allMap.get("test1").getVersion()); assertEquals(2, allMap.get("test2").getVersion()); assertEquals(3, allMap.get("test3").getVersion()); } @Test(expected = NotFoundException.class) public void removeInvalidWorkflowDef() { redisMetadataDAO.removeWorkflowDef("hello", 1); } @Test public void testTaskDefOperations() { TaskDef def = new TaskDef("taskA"); def.setDescription("description"); def.setCreatedBy("unit_test"); def.setCreateTime(1L); def.setInputKeys(Arrays.asList("a", "b", "c")); def.setOutputKeys(Arrays.asList("01", "o2")); def.setOwnerApp("ownerApp"); def.setRetryCount(3); def.setRetryDelaySeconds(100); def.setRetryLogic(RetryLogic.FIXED); def.setTimeoutPolicy(TimeoutPolicy.ALERT_ONLY); def.setUpdatedBy("unit_test2"); def.setUpdateTime(2L); def.setRateLimitPerFrequency(50); def.setRateLimitFrequencyInSeconds(1); redisMetadataDAO.createTaskDef(def); TaskDef found = redisMetadataDAO.getTaskDef(def.getName()); assertEquals(def, found); def.setDescription("updated description"); redisMetadataDAO.updateTaskDef(def); found = redisMetadataDAO.getTaskDef(def.getName()); assertEquals(def, found); assertEquals("updated description", found.getDescription()); for (int i = 0; i < 9; i++) { TaskDef tdf = new TaskDef("taskA" + i); redisMetadataDAO.createTaskDef(tdf); } List all = redisMetadataDAO.getAllTaskDefs(); assertNotNull(all); assertEquals(10, all.size()); Set allnames = all.stream().map(TaskDef::getName).collect(Collectors.toSet()); assertEquals(10, allnames.size()); List sorted = allnames.stream().sorted().collect(Collectors.toList()); assertEquals(def.getName(), sorted.get(0)); for (int i = 0; i < 9; i++) { assertEquals(def.getName() + i, sorted.get(i + 1)); } for (int i = 0; i < 9; i++) { redisMetadataDAO.removeTaskDef(def.getName() + i); } all = redisMetadataDAO.getAllTaskDefs(); assertNotNull(all); assertEquals(1, all.size()); assertEquals(def.getName(), all.get(0).getName()); } @Test(expected = NotFoundException.class) public void testRemoveTaskDef() { redisMetadataDAO.removeTaskDef("test" + UUID.randomUUID()); } @Test public void testDefaultsAreSetForResponseTimeout() { TaskDef def = new TaskDef("taskA"); def.setDescription("description"); def.setCreatedBy("unit_test"); def.setCreateTime(1L); def.setInputKeys(Arrays.asList("a", "b", "c")); def.setOutputKeys(Arrays.asList("01", "o2")); def.setOwnerApp("ownerApp"); def.setRetryCount(3); def.setRetryDelaySeconds(100); def.setRetryLogic(RetryLogic.FIXED); def.setTimeoutPolicy(TimeoutPolicy.ALERT_ONLY); def.setUpdatedBy("unit_test2"); def.setUpdateTime(2L); def.setRateLimitPerFrequency(50); def.setRateLimitFrequencyInSeconds(1); def.setResponseTimeoutSeconds(0); redisMetadataDAO.createTaskDef(def); TaskDef found = redisMetadataDAO.getTaskDef(def.getName()); assertEquals(found.getResponseTimeoutSeconds(), 3600); found.setTimeoutSeconds(200); found.setResponseTimeoutSeconds(0); redisMetadataDAO.updateTaskDef(found); TaskDef foundNew = redisMetadataDAO.getTaskDef(def.getName()); assertEquals(foundNew.getResponseTimeoutSeconds(), 199); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/RedisPollDataDAOTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import org.junit.Before; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.dao.PollDataDAO; import com.netflix.conductor.dao.PollDataDAOTest; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import redis.clients.jedis.commands.JedisCommands; import static org.mockito.Mockito.mock; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class RedisPollDataDAOTest extends PollDataDAOTest { private PollDataDAO redisPollDataDAO; @Autowired private ObjectMapper objectMapper; @Before public void init() { ConductorProperties conductorProperties = mock(ConductorProperties.class); RedisProperties properties = mock(RedisProperties.class); JedisCommands jedisMock = new JedisMock(); JedisProxy jedisProxy = new JedisProxy(jedisMock); redisPollDataDAO = new RedisPollDataDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Override protected PollDataDAO getPollDataDAO() { return redisPollDataDAO; } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/dao/RedisRateLimitDAOTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.dao; import java.util.UUID; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.config.TestObjectMapperConfiguration; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.jedis.JedisMock; import com.netflix.conductor.redis.jedis.JedisProxy; import com.fasterxml.jackson.databind.ObjectMapper; import redis.clients.jedis.commands.JedisCommands; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) @RunWith(SpringRunner.class) public class RedisRateLimitDAOTest { private RedisRateLimitingDAO rateLimitingDao; @Autowired private ObjectMapper objectMapper; @Before public void init() { ConductorProperties conductorProperties = mock(ConductorProperties.class); RedisProperties properties = mock(RedisProperties.class); JedisCommands jedisMock = new JedisMock(); JedisProxy jedisProxy = new JedisProxy(jedisMock); rateLimitingDao = new RedisRateLimitingDAO(jedisProxy, objectMapper, conductorProperties, properties); } @Test public void testExceedsRateLimitWhenNoRateLimitSet() { TaskDef taskDef = new TaskDef("TestTaskDefinition"); TaskModel task = new TaskModel(); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName(taskDef.getName()); assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); } @Test public void testExceedsRateLimitWithinLimit() { TaskDef taskDef = new TaskDef("TestTaskDefinition"); taskDef.setRateLimitFrequencyInSeconds(60); taskDef.setRateLimitPerFrequency(20); TaskModel task = new TaskModel(); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName(taskDef.getName()); assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); } @Test public void testExceedsRateLimitOutOfLimit() { TaskDef taskDef = new TaskDef("TestTaskDefinition"); taskDef.setRateLimitFrequencyInSeconds(60); taskDef.setRateLimitPerFrequency(1); TaskModel task = new TaskModel(); task.setTaskId(UUID.randomUUID().toString()); task.setTaskDefName(taskDef.getName()); assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); assertTrue(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/jedis/ConfigurationHostSupplierTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.List; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.redis.config.RedisProperties; import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; import com.netflix.dyno.connectionpool.Host; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ConfigurationHostSupplierTest { private RedisProperties properties; private ConfigurationHostSupplier configurationHostSupplier; @Before public void setUp() { properties = mock(RedisProperties.class); configurationHostSupplier = new ConfigurationHostSupplier(properties); } @Test public void getHost() { when(properties.getHosts()).thenReturn("dyno1:8102:us-east-1c"); List hosts = configurationHostSupplier.getHosts(); assertEquals(1, hosts.size()); Host firstHost = hosts.get(0); assertEquals("dyno1", firstHost.getHostName()); assertEquals(8102, firstHost.getPort()); assertEquals("us-east-1c", firstHost.getRack()); assertTrue(firstHost.isUp()); } @Test public void getMultipleHosts() { when(properties.getHosts()).thenReturn("dyno1:8102:us-east-1c;dyno2:8103:us-east-1c"); List hosts = configurationHostSupplier.getHosts(); assertEquals(2, hosts.size()); Host firstHost = hosts.get(0); assertEquals("dyno1", firstHost.getHostName()); assertEquals(8102, firstHost.getPort()); assertEquals("us-east-1c", firstHost.getRack()); assertTrue(firstHost.isUp()); Host secondHost = hosts.get(1); assertEquals("dyno2", secondHost.getHostName()); assertEquals(8103, secondHost.getPort()); assertEquals("us-east-1c", secondHost.getRack()); assertTrue(secondHost.isUp()); } @Test public void getAuthenticatedHost() { when(properties.getHosts()).thenReturn("redis1:6432:us-east-1c:password"); List hosts = configurationHostSupplier.getHosts(); assertEquals(1, hosts.size()); Host firstHost = hosts.get(0); assertEquals("redis1", firstHost.getHostName()); assertEquals(6432, firstHost.getPort()); assertEquals("us-east-1c", firstHost.getRack()); assertEquals("password", firstHost.getPassword()); assertTrue(firstHost.isUp()); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/jedis/JedisClusterTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.AbstractMap; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import org.junit.Test; import org.mockito.Mockito; import redis.clients.jedis.GeoUnit; import redis.clients.jedis.ListPosition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.SortingParams; import redis.clients.jedis.params.GeoRadiusParam; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.params.ZAddParams; import redis.clients.jedis.params.ZIncrByParams; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class JedisClusterTest { private final redis.clients.jedis.JedisCluster mockCluster = mock(redis.clients.jedis.JedisCluster.class); private final JedisCluster jedisCluster = new JedisCluster(mockCluster); @Test public void testSet() { jedisCluster.set("key", "value"); jedisCluster.set("key", "value", SetParams.setParams()); } @Test public void testGet() { jedisCluster.get("key"); } @Test public void testExists() { jedisCluster.exists("key"); } @Test public void testPersist() { jedisCluster.persist("key"); } @Test public void testType() { jedisCluster.type("key"); } @Test public void testExpire() { jedisCluster.expire("key", 1337); } @Test public void testPexpire() { jedisCluster.pexpire("key", 1337); } @Test public void testExpireAt() { jedisCluster.expireAt("key", 1337); } @Test public void testPexpireAt() { jedisCluster.pexpireAt("key", 1337); } @Test public void testTtl() { jedisCluster.ttl("key"); } @Test public void testPttl() { jedisCluster.pttl("key"); } @Test public void testSetbit() { jedisCluster.setbit("key", 1337, "value"); jedisCluster.setbit("key", 1337, true); } @Test public void testGetbit() { jedisCluster.getbit("key", 1337); } @Test public void testSetrange() { jedisCluster.setrange("key", 1337, "value"); } @Test public void testGetrange() { jedisCluster.getrange("key", 1337, 1338); } @Test public void testGetSet() { jedisCluster.getSet("key", "value"); } @Test public void testSetnx() { jedisCluster.setnx("test", "value"); } @Test public void testSetex() { jedisCluster.setex("key", 1337, "value"); } @Test public void testPsetex() { jedisCluster.psetex("key", 1337, "value"); } @Test public void testDecrBy() { jedisCluster.decrBy("key", 1337); } @Test public void testDecr() { jedisCluster.decr("key"); } @Test public void testIncrBy() { jedisCluster.incrBy("key", 1337); } @Test public void testIncrByFloat() { jedisCluster.incrByFloat("key", 1337); } @Test public void testIncr() { jedisCluster.incr("key"); } @Test public void testAppend() { jedisCluster.append("key", "value"); } @Test public void testSubstr() { jedisCluster.substr("key", 1337, 1338); } @Test public void testHset() { jedisCluster.hset("key", "field", "value"); } @Test public void testHget() { jedisCluster.hget("key", "field"); } @Test public void testHsetnx() { jedisCluster.hsetnx("key", "field", "value"); } @Test public void testHmset() { jedisCluster.hmset("key", new HashMap<>()); } @Test public void testHmget() { jedisCluster.hmget("key", "fields"); } @Test public void testHincrBy() { jedisCluster.hincrBy("key", "field", 1337); } @Test public void testHincrByFloat() { jedisCluster.hincrByFloat("key", "field", 1337); } @Test public void testHexists() { jedisCluster.hexists("key", "field"); } @Test public void testHdel() { jedisCluster.hdel("key", "field"); } @Test public void testHlen() { jedisCluster.hlen("key"); } @Test public void testHkeys() { jedisCluster.hkeys("key"); } @Test public void testHvals() { jedisCluster.hvals("key"); } @Test public void testGgetAll() { jedisCluster.hgetAll("key"); } @Test public void testRpush() { jedisCluster.rpush("key", "string"); } @Test public void testLpush() { jedisCluster.lpush("key", "string"); } @Test public void testLlen() { jedisCluster.llen("key"); } @Test public void testLrange() { jedisCluster.lrange("key", 1337, 1338); } @Test public void testLtrim() { jedisCluster.ltrim("key", 1337, 1338); } @Test public void testLindex() { jedisCluster.lindex("key", 1337); } @Test public void testLset() { jedisCluster.lset("key", 1337, "value"); } @Test public void testLrem() { jedisCluster.lrem("key", 1337, "value"); } @Test public void testLpop() { jedisCluster.lpop("key"); } @Test public void testRpop() { jedisCluster.rpop("key"); } @Test public void testSadd() { jedisCluster.sadd("key", "member"); } @Test public void testSmembers() { jedisCluster.smembers("key"); } @Test public void testSrem() { jedisCluster.srem("key", "member"); } @Test public void testSpop() { jedisCluster.spop("key"); jedisCluster.spop("key", 1337); } @Test public void testScard() { jedisCluster.scard("key"); } @Test public void testSismember() { jedisCluster.sismember("key", "member"); } @Test public void testSrandmember() { jedisCluster.srandmember("key"); jedisCluster.srandmember("key", 1337); } @Test public void testStrlen() { jedisCluster.strlen("key"); } @Test public void testZadd() { jedisCluster.zadd("key", new HashMap<>()); jedisCluster.zadd("key", new HashMap<>(), ZAddParams.zAddParams()); jedisCluster.zadd("key", 1337, "members"); jedisCluster.zadd("key", 1337, "members", ZAddParams.zAddParams()); } @Test public void testZrange() { jedisCluster.zrange("key", 1337, 1338); } @Test public void testZrem() { jedisCluster.zrem("key", "member"); } @Test public void testZincrby() { jedisCluster.zincrby("key", 1337, "member"); jedisCluster.zincrby("key", 1337, "member", ZIncrByParams.zIncrByParams()); } @Test public void testZrank() { jedisCluster.zrank("key", "member"); } @Test public void testZrevrank() { jedisCluster.zrevrank("key", "member"); } @Test public void testZrevrange() { jedisCluster.zrevrange("key", 1337, 1338); } @Test public void testZrangeWithScores() { jedisCluster.zrangeWithScores("key", 1337, 1338); } @Test public void testZrevrangeWithScores() { jedisCluster.zrevrangeWithScores("key", 1337, 1338); } @Test public void testZcard() { jedisCluster.zcard("key"); } @Test public void testZscore() { jedisCluster.zscore("key", "member"); } @Test public void testSort() { jedisCluster.sort("key"); jedisCluster.sort("key", new SortingParams()); } @Test public void testZcount() { jedisCluster.zcount("key", "min", "max"); jedisCluster.zcount("key", 1337, 1338); } @Test public void testZrangeByScore() { jedisCluster.zrangeByScore("key", "min", "max"); jedisCluster.zrangeByScore("key", 1337, 1338); jedisCluster.zrangeByScore("key", "min", "max", 1337, 1338); jedisCluster.zrangeByScore("key", 1337, 1338, 1339, 1340); } @Test public void testZrevrangeByScore() { jedisCluster.zrevrangeByScore("key", "max", "min"); jedisCluster.zrevrangeByScore("key", 1337, 1338); jedisCluster.zrevrangeByScore("key", "max", "min", 1337, 1338); jedisCluster.zrevrangeByScore("key", 1337, 1338, 1339, 1340); } @Test public void testZrangeByScoreWithScores() { jedisCluster.zrangeByScoreWithScores("key", "min", "max"); jedisCluster.zrangeByScoreWithScores("key", "min", "max", 1337, 1338); jedisCluster.zrangeByScoreWithScores("key", 1337, 1338); jedisCluster.zrangeByScoreWithScores("key", 1337, 1338, 1339, 1340); } @Test public void testZrevrangeByScoreWithScores() { jedisCluster.zrevrangeByScoreWithScores("key", "max", "min"); jedisCluster.zrevrangeByScoreWithScores("key", "max", "min", 1337, 1338); jedisCluster.zrevrangeByScoreWithScores("key", 1337, 1338); jedisCluster.zrevrangeByScoreWithScores("key", 1337, 1338, 1339, 1340); } @Test public void testZremrangeByRank() { jedisCluster.zremrangeByRank("key", 1337, 1338); } @Test public void testZremrangeByScore() { jedisCluster.zremrangeByScore("key", "start", "end"); jedisCluster.zremrangeByScore("key", 1337, 1338); } @Test public void testZlexcount() { jedisCluster.zlexcount("key", "min", "max"); } @Test public void testZrangeByLex() { jedisCluster.zrangeByLex("key", "min", "max"); jedisCluster.zrangeByLex("key", "min", "max", 1337, 1338); } @Test public void testZrevrangeByLex() { jedisCluster.zrevrangeByLex("key", "max", "min"); jedisCluster.zrevrangeByLex("key", "max", "min", 1337, 1338); } @Test public void testZremrangeByLex() { jedisCluster.zremrangeByLex("key", "min", "max"); } @Test public void testLinsert() { jedisCluster.linsert("key", ListPosition.AFTER, "pivot", "value"); } @Test public void testLpushx() { jedisCluster.lpushx("key", "string"); } @Test public void testRpushx() { jedisCluster.rpushx("key", "string"); } @Test public void testBlpop() { jedisCluster.blpop(1337, "arg"); } @Test public void testBrpop() { jedisCluster.brpop(1337, "arg"); } @Test public void testDel() { jedisCluster.del("key"); } @Test public void testEcho() { jedisCluster.echo("string"); } @Test(expected = UnsupportedOperationException.class) public void testMove() { jedisCluster.move("key", 1337); } @Test public void testBitcount() { jedisCluster.bitcount("key"); jedisCluster.bitcount("key", 1337, 1338); } @Test(expected = UnsupportedOperationException.class) public void testBitpos() { jedisCluster.bitpos("key", true); } @Test public void testHscan() { jedisCluster.hscan("key", "cursor"); ScanResult> scanResult = new ScanResult<>( "cursor".getBytes(), Arrays.asList( new AbstractMap.SimpleEntry<>("key1".getBytes(), "val1".getBytes()), new AbstractMap.SimpleEntry<>( "key2".getBytes(), "val2".getBytes()))); when(mockCluster.hscan(Mockito.any(), Mockito.any(), Mockito.any(ScanParams.class))) .thenReturn(scanResult); ScanResult> result = jedisCluster.hscan("key", "cursor", new ScanParams()); assertEquals("cursor", result.getCursor()); assertEquals(2, result.getResult().size()); assertEquals("val1", result.getResult().get(0).getValue()); } @Test public void testSscan() { jedisCluster.sscan("key", "cursor"); ScanResult scanResult = new ScanResult<>( "sscursor".getBytes(), Arrays.asList("val1".getBytes(), "val2".getBytes())); when(mockCluster.sscan(Mockito.any(), Mockito.any(), Mockito.any(ScanParams.class))) .thenReturn(scanResult); ScanResult result = jedisCluster.sscan("key", "cursor", new ScanParams()); assertEquals("sscursor", result.getCursor()); assertEquals(2, result.getResult().size()); assertEquals("val1", result.getResult().get(0)); } @Test public void testZscan() { jedisCluster.zscan("key", "cursor"); jedisCluster.zscan("key", "cursor", new ScanParams()); } @Test public void testPfadd() { jedisCluster.pfadd("key", "elements"); } @Test public void testPfcount() { jedisCluster.pfcount("key"); } @Test public void testGeoadd() { jedisCluster.geoadd("key", new HashMap<>()); jedisCluster.geoadd("key", 1337, 1338, "member"); } @Test public void testGeodist() { jedisCluster.geodist("key", "member1", "member2"); jedisCluster.geodist("key", "member1", "member2", GeoUnit.KM); } @Test public void testGeohash() { jedisCluster.geohash("key", "members"); } @Test public void testGeopos() { jedisCluster.geopos("key", "members"); } @Test public void testGeoradius() { jedisCluster.georadius("key", 1337, 1338, 32, GeoUnit.KM); jedisCluster.georadius("key", 1337, 1338, 32, GeoUnit.KM, GeoRadiusParam.geoRadiusParam()); } @Test public void testGeoradiusByMember() { jedisCluster.georadiusByMember("key", "member", 1337, GeoUnit.KM); jedisCluster.georadiusByMember( "key", "member", 1337, GeoUnit.KM, GeoRadiusParam.geoRadiusParam()); } @Test public void testBitfield() { jedisCluster.bitfield("key", "arguments"); } } ================================================ FILE: redis-persistence/src/test/java/com/netflix/conductor/redis/jedis/JedisSentinelTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.redis.jedis; import java.util.HashMap; import org.junit.Before; import org.junit.Test; import redis.clients.jedis.GeoUnit; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisSentinelPool; import redis.clients.jedis.ListPosition; import redis.clients.jedis.ScanParams; import redis.clients.jedis.SortingParams; import redis.clients.jedis.params.GeoRadiusParam; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.params.ZAddParams; import redis.clients.jedis.params.ZIncrByParams; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class JedisSentinelTest { private final Jedis jedis = mock(Jedis.class); private final JedisSentinelPool jedisPool = mock(JedisSentinelPool.class); private final JedisSentinel jedisSentinel = new JedisSentinel(jedisPool); @Before public void init() { when(this.jedisPool.getResource()).thenReturn(this.jedis); } @Test public void testSet() { jedisSentinel.set("key", "value"); jedisSentinel.set("key", "value", SetParams.setParams()); } @Test public void testGet() { jedisSentinel.get("key"); } @Test public void testExists() { jedisSentinel.exists("key"); } @Test public void testPersist() { jedisSentinel.persist("key"); } @Test public void testType() { jedisSentinel.type("key"); } @Test public void testExpire() { jedisSentinel.expire("key", 1337); } @Test public void testPexpire() { jedisSentinel.pexpire("key", 1337); } @Test public void testExpireAt() { jedisSentinel.expireAt("key", 1337); } @Test public void testPexpireAt() { jedisSentinel.pexpireAt("key", 1337); } @Test public void testTtl() { jedisSentinel.ttl("key"); } @Test public void testPttl() { jedisSentinel.pttl("key"); } @Test public void testSetbit() { jedisSentinel.setbit("key", 1337, "value"); jedisSentinel.setbit("key", 1337, true); } @Test public void testGetbit() { jedisSentinel.getbit("key", 1337); } @Test public void testSetrange() { jedisSentinel.setrange("key", 1337, "value"); } @Test public void testGetrange() { jedisSentinel.getrange("key", 1337, 1338); } @Test public void testGetSet() { jedisSentinel.getSet("key", "value"); } @Test public void testSetnx() { jedisSentinel.setnx("test", "value"); } @Test public void testSetex() { jedisSentinel.setex("key", 1337, "value"); } @Test public void testPsetex() { jedisSentinel.psetex("key", 1337, "value"); } @Test public void testDecrBy() { jedisSentinel.decrBy("key", 1337); } @Test public void testDecr() { jedisSentinel.decr("key"); } @Test public void testIncrBy() { jedisSentinel.incrBy("key", 1337); } @Test public void testIncrByFloat() { jedisSentinel.incrByFloat("key", 1337); } @Test public void testIncr() { jedisSentinel.incr("key"); } @Test public void testAppend() { jedisSentinel.append("key", "value"); } @Test public void testSubstr() { jedisSentinel.substr("key", 1337, 1338); } @Test public void testHset() { jedisSentinel.hset("key", "field", "value"); } @Test public void testHget() { jedisSentinel.hget("key", "field"); } @Test public void testHsetnx() { jedisSentinel.hsetnx("key", "field", "value"); } @Test public void testHmset() { jedisSentinel.hmset("key", new HashMap<>()); } @Test public void testHmget() { jedisSentinel.hmget("key", "fields"); } @Test public void testHincrBy() { jedisSentinel.hincrBy("key", "field", 1337); } @Test public void testHincrByFloat() { jedisSentinel.hincrByFloat("key", "field", 1337); } @Test public void testHexists() { jedisSentinel.hexists("key", "field"); } @Test public void testHdel() { jedisSentinel.hdel("key", "field"); } @Test public void testHlen() { jedisSentinel.hlen("key"); } @Test public void testHkeys() { jedisSentinel.hkeys("key"); } @Test public void testHvals() { jedisSentinel.hvals("key"); } @Test public void testGgetAll() { jedisSentinel.hgetAll("key"); } @Test public void testRpush() { jedisSentinel.rpush("key", "string"); } @Test public void testLpush() { jedisSentinel.lpush("key", "string"); } @Test public void testLlen() { jedisSentinel.llen("key"); } @Test public void testLrange() { jedisSentinel.lrange("key", 1337, 1338); } @Test public void testLtrim() { jedisSentinel.ltrim("key", 1337, 1338); } @Test public void testLindex() { jedisSentinel.lindex("key", 1337); } @Test public void testLset() { jedisSentinel.lset("key", 1337, "value"); } @Test public void testLrem() { jedisSentinel.lrem("key", 1337, "value"); } @Test public void testLpop() { jedisSentinel.lpop("key"); } @Test public void testRpop() { jedisSentinel.rpop("key"); } @Test public void testSadd() { jedisSentinel.sadd("key", "member"); } @Test public void testSmembers() { jedisSentinel.smembers("key"); } @Test public void testSrem() { jedisSentinel.srem("key", "member"); } @Test public void testSpop() { jedisSentinel.spop("key"); jedisSentinel.spop("key", 1337); } @Test public void testScard() { jedisSentinel.scard("key"); } @Test public void testSismember() { jedisSentinel.sismember("key", "member"); } @Test public void testSrandmember() { jedisSentinel.srandmember("key"); jedisSentinel.srandmember("key", 1337); } @Test public void testStrlen() { jedisSentinel.strlen("key"); } @Test public void testZadd() { jedisSentinel.zadd("key", new HashMap<>()); jedisSentinel.zadd("key", new HashMap<>(), ZAddParams.zAddParams()); jedisSentinel.zadd("key", 1337, "members"); jedisSentinel.zadd("key", 1337, "members", ZAddParams.zAddParams()); } @Test public void testZrange() { jedisSentinel.zrange("key", 1337, 1338); } @Test public void testZrem() { jedisSentinel.zrem("key", "member"); } @Test public void testZincrby() { jedisSentinel.zincrby("key", 1337, "member"); jedisSentinel.zincrby("key", 1337, "member", ZIncrByParams.zIncrByParams()); } @Test public void testZrank() { jedisSentinel.zrank("key", "member"); } @Test public void testZrevrank() { jedisSentinel.zrevrank("key", "member"); } @Test public void testZrevrange() { jedisSentinel.zrevrange("key", 1337, 1338); } @Test public void testZrangeWithScores() { jedisSentinel.zrangeWithScores("key", 1337, 1338); } @Test public void testZrevrangeWithScores() { jedisSentinel.zrevrangeWithScores("key", 1337, 1338); } @Test public void testZcard() { jedisSentinel.zcard("key"); } @Test public void testZscore() { jedisSentinel.zscore("key", "member"); } @Test public void testSort() { jedisSentinel.sort("key"); jedisSentinel.sort("key", new SortingParams()); } @Test public void testZcount() { jedisSentinel.zcount("key", "min", "max"); jedisSentinel.zcount("key", 1337, 1338); } @Test public void testZrangeByScore() { jedisSentinel.zrangeByScore("key", "min", "max"); jedisSentinel.zrangeByScore("key", 1337, 1338); jedisSentinel.zrangeByScore("key", "min", "max", 1337, 1338); jedisSentinel.zrangeByScore("key", 1337, 1338, 1339, 1340); } @Test public void testZrevrangeByScore() { jedisSentinel.zrevrangeByScore("key", "max", "min"); jedisSentinel.zrevrangeByScore("key", 1337, 1338); jedisSentinel.zrevrangeByScore("key", "max", "min", 1337, 1338); jedisSentinel.zrevrangeByScore("key", 1337, 1338, 1339, 1340); } @Test public void testZrangeByScoreWithScores() { jedisSentinel.zrangeByScoreWithScores("key", "min", "max"); jedisSentinel.zrangeByScoreWithScores("key", "min", "max", 1337, 1338); jedisSentinel.zrangeByScoreWithScores("key", 1337, 1338); jedisSentinel.zrangeByScoreWithScores("key", 1337, 1338, 1339, 1340); } @Test public void testZrevrangeByScoreWithScores() { jedisSentinel.zrevrangeByScoreWithScores("key", "max", "min"); jedisSentinel.zrevrangeByScoreWithScores("key", "max", "min", 1337, 1338); jedisSentinel.zrevrangeByScoreWithScores("key", 1337, 1338); jedisSentinel.zrevrangeByScoreWithScores("key", 1337, 1338, 1339, 1340); } @Test public void testZremrangeByRank() { jedisSentinel.zremrangeByRank("key", 1337, 1338); } @Test public void testZremrangeByScore() { jedisSentinel.zremrangeByScore("key", "start", "end"); jedisSentinel.zremrangeByScore("key", 1337, 1338); } @Test public void testZlexcount() { jedisSentinel.zlexcount("key", "min", "max"); } @Test public void testZrangeByLex() { jedisSentinel.zrangeByLex("key", "min", "max"); jedisSentinel.zrangeByLex("key", "min", "max", 1337, 1338); } @Test public void testZrevrangeByLex() { jedisSentinel.zrevrangeByLex("key", "max", "min"); jedisSentinel.zrevrangeByLex("key", "max", "min", 1337, 1338); } @Test public void testZremrangeByLex() { jedisSentinel.zremrangeByLex("key", "min", "max"); } @Test public void testLinsert() { jedisSentinel.linsert("key", ListPosition.AFTER, "pivot", "value"); } @Test public void testLpushx() { jedisSentinel.lpushx("key", "string"); } @Test public void testRpushx() { jedisSentinel.rpushx("key", "string"); } @Test public void testBlpop() { jedisSentinel.blpop(1337, "arg"); } @Test public void testBrpop() { jedisSentinel.brpop(1337, "arg"); } @Test public void testDel() { jedisSentinel.del("key"); } @Test public void testEcho() { jedisSentinel.echo("string"); } @Test public void testMove() { jedisSentinel.move("key", 1337); } @Test public void testBitcount() { jedisSentinel.bitcount("key"); jedisSentinel.bitcount("key", 1337, 1338); } @Test public void testBitpos() { jedisSentinel.bitpos("key", true); } @Test public void testHscan() { jedisSentinel.hscan("key", "cursor"); jedisSentinel.hscan("key", "cursor", new ScanParams()); } @Test public void testSscan() { jedisSentinel.sscan("key", "cursor"); jedisSentinel.sscan("key", "cursor", new ScanParams()); } @Test public void testZscan() { jedisSentinel.zscan("key", "cursor"); jedisSentinel.zscan("key", "cursor", new ScanParams()); } @Test public void testPfadd() { jedisSentinel.pfadd("key", "elements"); } @Test public void testPfcount() { jedisSentinel.pfcount("key"); } @Test public void testGeoadd() { jedisSentinel.geoadd("key", new HashMap<>()); jedisSentinel.geoadd("key", 1337, 1338, "member"); } @Test public void testGeodist() { jedisSentinel.geodist("key", "member1", "member2"); jedisSentinel.geodist("key", "member1", "member2", GeoUnit.KM); } @Test public void testGeohash() { jedisSentinel.geohash("key", "members"); } @Test public void testGeopos() { jedisSentinel.geopos("key", "members"); } @Test public void testGeoradius() { jedisSentinel.georadius("key", 1337, 1338, 32, GeoUnit.KM); jedisSentinel.georadius("key", 1337, 1338, 32, GeoUnit.KM, GeoRadiusParam.geoRadiusParam()); } @Test public void testGeoradiusByMember() { jedisSentinel.georadiusByMember("key", "member", 1337, GeoUnit.KM); jedisSentinel.georadiusByMember( "key", "member", 1337, GeoUnit.KM, GeoRadiusParam.geoRadiusParam()); } @Test public void testBitfield() { jedisSentinel.bitfield("key", "arguments"); } } ================================================ FILE: rest/build.gradle ================================================ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') implementation 'org.springframework.boot:spring-boot-starter-web' implementation "com.netflix.runtime:health-api:${revHealth}" implementation "org.springdoc:springdoc-openapi-ui:${revOpenapi}" } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/config/RequestMappingConstants.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.config; public interface RequestMappingConstants { String API_PREFIX = "/api/"; String ADMIN = API_PREFIX + "admin"; String EVENT = API_PREFIX + "event"; String METADATA = API_PREFIX + "metadata"; String QUEUE = API_PREFIX + "queue"; String TASKS = API_PREFIX + "tasks"; String WORKFLOW_BULK = API_PREFIX + "workflow/bulk"; String WORKFLOW = API_PREFIX + "workflow"; } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/config/RestConfiguration.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.config; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.http.MediaType.TEXT_PLAIN; @Configuration public class RestConfiguration implements WebMvcConfigurer { /** * Disable all 3 (Accept header, url parameter, path extension) strategies of content * negotiation and only allow application/json and text/plain types. *
    * *

    Any "mapping" that is annotated with produces=TEXT_PLAIN_VALUE will be sent * as text/plain all others as application/json.
    * More details on Spring MVC content negotiation can be found at https://spring.io/blog/2013/05/11/content-negotiation-using-spring-mvc *
    */ @Override public void configureContentNegotiation(ContentNegotiationConfigurer configurer) { configurer .favorParameter(false) .favorPathExtension(false) .ignoreAcceptHeader(true) .defaultContentType(APPLICATION_JSON, TEXT_PLAIN); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/AdminResource.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import java.util.Map; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.service.AdminService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.ADMIN; import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; @RestController @RequestMapping(ADMIN) public class AdminResource { private final AdminService adminService; public AdminResource(AdminService adminService) { this.adminService = adminService; } @Operation(summary = "Get all the configuration parameters") @GetMapping("/config") public Map getAllConfig() { return adminService.getAllConfig(); } @GetMapping("/task/{tasktype}") @Operation(summary = "Get the list of pending tasks for a given task type") public List view( @PathVariable("tasktype") String taskType, @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "count", defaultValue = "100", required = false) int count) { return adminService.getListOfPendingTask(taskType, start, count); } @PostMapping(value = "/sweep/requeue/{workflowId}", produces = TEXT_PLAIN_VALUE) @Operation(summary = "Queue up all the running workflows for sweep") public String requeueSweep(@PathVariable("workflowId") String workflowId) { return adminService.requeueSweep(workflowId); } @PostMapping(value = "/consistency/verifyAndRepair/{workflowId}", produces = TEXT_PLAIN_VALUE) @Operation(summary = "Verify and repair workflow consistency") public String verifyAndRepairWorkflowConsistency( @PathVariable("workflowId") String workflowId) { return String.valueOf(adminService.verifyAndRepairWorkflowConsistency(workflowId)); } @GetMapping("/queues") @Operation(summary = "Get registered queues") public Map getEventQueues( @RequestParam(value = "verbose", defaultValue = "false", required = false) boolean verbose) { return adminService.getEventQueues(verbose); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/ApplicationExceptionMapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.HashMap; import java.util.Map; import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import com.netflix.conductor.common.validation.ErrorResponse; import com.netflix.conductor.core.exception.ConflictException; import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.exception.TransientException; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.metrics.Monitors; import com.fasterxml.jackson.databind.exc.InvalidFormatException; @RestControllerAdvice @Order(ValidationExceptionMapper.ORDER + 1) public class ApplicationExceptionMapper { private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationExceptionMapper.class); private final String host = Utils.getServerId(); private static final Map, HttpStatus> EXCEPTION_STATUS_MAP = new HashMap<>(); static { EXCEPTION_STATUS_MAP.put(NotFoundException.class, HttpStatus.NOT_FOUND); EXCEPTION_STATUS_MAP.put(ConflictException.class, HttpStatus.CONFLICT); EXCEPTION_STATUS_MAP.put(IllegalArgumentException.class, HttpStatus.BAD_REQUEST); EXCEPTION_STATUS_MAP.put(InvalidFormatException.class, HttpStatus.INTERNAL_SERVER_ERROR); } @ExceptionHandler(Throwable.class) public ResponseEntity handleAll(HttpServletRequest request, Throwable th) { logException(request, th); HttpStatus status = EXCEPTION_STATUS_MAP.getOrDefault(th.getClass(), HttpStatus.INTERNAL_SERVER_ERROR); ErrorResponse errorResponse = new ErrorResponse(); errorResponse.setInstance(host); errorResponse.setStatus(status.value()); errorResponse.setMessage(th.getMessage()); errorResponse.setRetryable( th instanceof TransientException); // set it to true for TransientException Monitors.error("error", String.valueOf(status.value())); return new ResponseEntity<>(errorResponse, status); } private void logException(HttpServletRequest request, Throwable exception) { LOGGER.error( "Error {} url: '{}'", exception.getClass().getSimpleName(), request.getRequestURI(), exception); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/EventResource.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.service.EventService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.EVENT; @RestController @RequestMapping(EVENT) public class EventResource { private final EventService eventService; public EventResource(EventService eventService) { this.eventService = eventService; } @PostMapping @Operation(summary = "Add a new event handler.") public void addEventHandler(@RequestBody EventHandler eventHandler) { eventService.addEventHandler(eventHandler); } @PutMapping @Operation(summary = "Update an existing event handler.") public void updateEventHandler(@RequestBody EventHandler eventHandler) { eventService.updateEventHandler(eventHandler); } @DeleteMapping("/{name}") @Operation(summary = "Remove an event handler") public void removeEventHandlerStatus(@PathVariable("name") String name) { eventService.removeEventHandlerStatus(name); } @GetMapping @Operation(summary = "Get all the event handlers") public List getEventHandlers() { return eventService.getEventHandlers(); } @GetMapping("/{event}") @Operation(summary = "Get event handlers for a given event") public List getEventHandlersForEvent( @PathVariable("event") String event, @RequestParam(value = "activeOnly", defaultValue = "true", required = false) boolean activeOnly) { return eventService.getEventHandlersForEvent(event, activeOnly); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/HealthCheckResource.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.Collections; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import com.netflix.runtime.health.api.HealthCheckStatus; @RestController @RequestMapping("/health") public class HealthCheckResource { // SBMTODO: Move this Spring boot health check @GetMapping public HealthCheckStatus doCheck() throws Exception { return HealthCheckStatus.create(true, Collections.emptyList()); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/MetadataResource.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import java.util.Map; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDefSummary; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.service.MetadataService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.METADATA; @RestController @RequestMapping(value = METADATA) public class MetadataResource { private final MetadataService metadataService; public MetadataResource(MetadataService metadataService) { this.metadataService = metadataService; } @PostMapping("/workflow") @Operation(summary = "Create a new workflow definition") public void create(@RequestBody WorkflowDef workflowDef) { metadataService.registerWorkflowDef(workflowDef); } @PostMapping("/workflow/validate") @Operation(summary = "Validates a new workflow definition") public void validate(@RequestBody WorkflowDef workflowDef) { metadataService.validateWorkflowDef(workflowDef); } @PutMapping("/workflow") @Operation(summary = "Create or update workflow definition") public BulkResponse update(@RequestBody List workflowDefs) { return metadataService.updateWorkflowDef(workflowDefs); } @Operation(summary = "Retrieves workflow definition along with blueprint") @GetMapping("/workflow/{name}") public WorkflowDef get( @PathVariable("name") String name, @RequestParam(value = "version", required = false) Integer version) { return metadataService.getWorkflowDef(name, version); } @Operation(summary = "Retrieves all workflow definition along with blueprint") @GetMapping("/workflow") public List getAll() { return metadataService.getWorkflowDefs(); } @Operation(summary = "Returns workflow names and versions only (no definition bodies)") @GetMapping("/workflow/names-and-versions") public Map> getWorkflowNamesAndVersions() { return metadataService.getWorkflowNamesAndVersions(); } @Operation(summary = "Returns only the latest version of all workflow definitions") @GetMapping("/workflow/latest-versions") public List getAllWorkflowsWithLatestVersions() { return metadataService.getWorkflowDefsLatestVersions(); } @DeleteMapping("/workflow/{name}/{version}") @Operation( summary = "Removes workflow definition. It does not remove workflows associated with the definition.") public void unregisterWorkflowDef( @PathVariable("name") String name, @PathVariable("version") Integer version) { metadataService.unregisterWorkflowDef(name, version); } @PostMapping("/taskdefs") @Operation(summary = "Create new task definition(s)") public void registerTaskDef(@RequestBody List taskDefs) { metadataService.registerTaskDef(taskDefs); } @PutMapping("/taskdefs") @Operation(summary = "Update an existing task") public void registerTaskDef(@RequestBody TaskDef taskDef) { metadataService.updateTaskDef(taskDef); } @GetMapping(value = "/taskdefs") @Operation(summary = "Gets all task definition") public List getTaskDefs() { return metadataService.getTaskDefs(); } @GetMapping("/taskdefs/{tasktype}") @Operation(summary = "Gets the task definition") public TaskDef getTaskDef(@PathVariable("tasktype") String taskType) { return metadataService.getTaskDef(taskType); } @DeleteMapping("/taskdefs/{tasktype}") @Operation(summary = "Remove a task definition") public void unregisterTaskDef(@PathVariable("tasktype") String taskType) { metadataService.unregisterTaskDef(taskType); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/QueueAdminResource.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.Map; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.core.events.queue.DefaultEventQueueProcessor; import com.netflix.conductor.model.TaskModel.Status; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.QUEUE; @RestController @RequestMapping(QUEUE) public class QueueAdminResource { private final DefaultEventQueueProcessor defaultEventQueueProcessor; public QueueAdminResource(DefaultEventQueueProcessor defaultEventQueueProcessor) { this.defaultEventQueueProcessor = defaultEventQueueProcessor; } @Operation(summary = "Get the queue length") @GetMapping(value = "/size") public Map size() { return defaultEventQueueProcessor.size(); } @Operation(summary = "Get Queue Names") @GetMapping(value = "/") public Map names() { return defaultEventQueueProcessor.queues(); } @Operation(summary = "Publish a message in queue to mark a wait task as completed.") @PostMapping(value = "/update/{workflowId}/{taskRefName}/{status}") public void update( @PathVariable("workflowId") String workflowId, @PathVariable("taskRefName") String taskRefName, @PathVariable("status") Status status, @RequestBody Map output) throws Exception { defaultEventQueueProcessor.updateByTaskRefName(workflowId, taskRefName, output, status); } @Operation(summary = "Publish a message in queue to mark a wait task (by taskId) as completed.") @PostMapping("/update/{workflowId}/task/{taskId}/{status}") public void updateByTaskId( @PathVariable("workflowId") String workflowId, @PathVariable("taskId") String taskId, @PathVariable("status") Status status, @RequestBody Map output) throws Exception { defaultEventQueueProcessor.updateByTaskId(workflowId, taskId, output, status); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/TaskResource.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import java.util.Map; import java.util.Optional; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.service.TaskService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.TASKS; import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; @RestController @RequestMapping(value = TASKS) public class TaskResource { private final TaskService taskService; public TaskResource(TaskService taskService) { this.taskService = taskService; } @GetMapping("/poll/{tasktype}") @Operation(summary = "Poll for a task of a certain type") public ResponseEntity poll( @PathVariable("tasktype") String taskType, @RequestParam(value = "workerid", required = false) String workerId, @RequestParam(value = "domain", required = false) String domain) { // for backwards compatibility with 2.x client which expects a 204 when no Task is found return Optional.ofNullable(taskService.poll(taskType, workerId, domain)) .map(ResponseEntity::ok) .orElse(ResponseEntity.noContent().build()); } @GetMapping("/poll/batch/{tasktype}") @Operation(summary = "Batch poll for a task of a certain type") public ResponseEntity> batchPoll( @PathVariable("tasktype") String taskType, @RequestParam(value = "workerid", required = false) String workerId, @RequestParam(value = "domain", required = false) String domain, @RequestParam(value = "count", defaultValue = "1") int count, @RequestParam(value = "timeout", defaultValue = "100") int timeout) { // for backwards compatibility with 2.x client which expects a 204 when no Task is found return Optional.ofNullable( taskService.batchPoll(taskType, workerId, domain, count, timeout)) .map(ResponseEntity::ok) .orElse(ResponseEntity.noContent().build()); } @PostMapping(produces = TEXT_PLAIN_VALUE) @Operation(summary = "Update a task") public String updateTask(@RequestBody TaskResult taskResult) { return taskService.updateTask(taskResult); } @PostMapping("/{taskId}/log") @Operation(summary = "Log Task Execution Details") public void log(@PathVariable("taskId") String taskId, @RequestBody String log) { taskService.log(taskId, log); } @GetMapping("/{taskId}/log") @Operation(summary = "Get Task Execution Logs") public List getTaskLogs(@PathVariable("taskId") String taskId) { return taskService.getTaskLogs(taskId); } @GetMapping("/{taskId}") @Operation(summary = "Get task by Id") public ResponseEntity getTask(@PathVariable("taskId") String taskId) { // for backwards compatibility with 2.x client which expects a 204 when no Task is found return Optional.ofNullable(taskService.getTask(taskId)) .map(ResponseEntity::ok) .orElse(ResponseEntity.noContent().build()); } @GetMapping("/queue/sizes") @Operation(summary = "Deprecated. Please use /tasks/queue/size endpoint") @Deprecated public Map size( @RequestParam(value = "taskType", required = false) List taskTypes) { return taskService.getTaskQueueSizes(taskTypes); } @GetMapping("/queue/size") @Operation(summary = "Get queue size for a task type.") public Integer taskDepth( @RequestParam("taskType") String taskType, @RequestParam(value = "domain", required = false) String domain, @RequestParam(value = "isolationGroupId", required = false) String isolationGroupId, @RequestParam(value = "executionNamespace", required = false) String executionNamespace) { return taskService.getTaskQueueSize(taskType, domain, executionNamespace, isolationGroupId); } @GetMapping("/queue/all/verbose") @Operation(summary = "Get the details about each queue") public Map>> allVerbose() { return taskService.allVerbose(); } @GetMapping("/queue/all") @Operation(summary = "Get the details about each queue") public Map all() { return taskService.getAllQueueDetails(); } @GetMapping("/queue/polldata") @Operation(summary = "Get the last poll data for a given task type") public List getPollData(@RequestParam("taskType") String taskType) { return taskService.getPollData(taskType); } @GetMapping("/queue/polldata/all") @Operation(summary = "Get the last poll data for all task types") public List getAllPollData() { return taskService.getAllPollData(); } @PostMapping(value = "/queue/requeue/{taskType}", produces = TEXT_PLAIN_VALUE) @Operation(summary = "Requeue pending tasks") public String requeuePendingTask(@PathVariable("taskType") String taskType) { return taskService.requeuePendingTask(taskType); } @Operation( summary = "Search for tasks based in payload and other parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC") @GetMapping(value = "/search") public SearchResult search( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return taskService.search(start, size, sort, freeText, query); } @Operation( summary = "Search for tasks based in payload and other parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC") @GetMapping(value = "/search-v2") public SearchResult searchV2( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return taskService.searchV2(start, size, sort, freeText, query); } @Operation(summary = "Get the external uri where the task payload is to be stored") @GetMapping({"/externalstoragelocation", "external-storage-location"}) public ExternalStorageLocation getExternalStorageLocation( @RequestParam("path") String path, @RequestParam("operation") String operation, @RequestParam("payloadType") String payloadType) { return taskService.getExternalStorageLocation(path, operation, payloadType); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/ValidationExceptionMapper.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; import javax.validation.ValidationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import com.netflix.conductor.common.validation.ErrorResponse; import com.netflix.conductor.common.validation.ValidationError; import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.metrics.Monitors; /** This class converts Hibernate {@link ValidationException} into http response. */ @RestControllerAdvice @Order(ValidationExceptionMapper.ORDER) public class ValidationExceptionMapper { private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationExceptionMapper.class); public static final int ORDER = Ordered.HIGHEST_PRECEDENCE; private final String host = Utils.getServerId(); @ExceptionHandler(ValidationException.class) public ResponseEntity toResponse( HttpServletRequest request, ValidationException exception) { logException(request, exception); HttpStatus httpStatus; if (exception instanceof ConstraintViolationException) { httpStatus = HttpStatus.BAD_REQUEST; } else { httpStatus = HttpStatus.INTERNAL_SERVER_ERROR; Monitors.error("error", "error"); } return new ResponseEntity<>(toErrorResponse(exception), httpStatus); } private ErrorResponse toErrorResponse(ValidationException ve) { if (ve instanceof ConstraintViolationException) { return constraintViolationExceptionToErrorResponse((ConstraintViolationException) ve); } else { ErrorResponse result = new ErrorResponse(); result.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); result.setMessage(ve.getMessage()); result.setInstance(host); return result; } } private ErrorResponse constraintViolationExceptionToErrorResponse( ConstraintViolationException exception) { ErrorResponse errorResponse = new ErrorResponse(); errorResponse.setStatus(HttpStatus.BAD_REQUEST.value()); errorResponse.setMessage("Validation failed, check below errors for detail."); List validationErrors = new ArrayList<>(); exception .getConstraintViolations() .forEach( e -> validationErrors.add( new ValidationError( getViolationPath(e), e.getMessage(), getViolationInvalidValue(e.getInvalidValue())))); errorResponse.setValidationErrors(validationErrors); return errorResponse; } private String getViolationPath(final ConstraintViolation violation) { final String propertyPath = violation.getPropertyPath().toString(); return !"".equals(propertyPath) ? propertyPath : ""; } private String getViolationInvalidValue(final Object invalidValue) { if (invalidValue == null) { return null; } if (invalidValue.getClass().isArray()) { if (invalidValue instanceof Object[]) { // not helpful to return object array, skip it. return null; } else if (invalidValue instanceof boolean[]) { return Arrays.toString((boolean[]) invalidValue); } else if (invalidValue instanceof byte[]) { return Arrays.toString((byte[]) invalidValue); } else if (invalidValue instanceof char[]) { return Arrays.toString((char[]) invalidValue); } else if (invalidValue instanceof double[]) { return Arrays.toString((double[]) invalidValue); } else if (invalidValue instanceof float[]) { return Arrays.toString((float[]) invalidValue); } else if (invalidValue instanceof int[]) { return Arrays.toString((int[]) invalidValue); } else if (invalidValue instanceof long[]) { return Arrays.toString((long[]) invalidValue); } else if (invalidValue instanceof short[]) { return Arrays.toString((short[]) invalidValue); } } // It is only helpful to return invalid value of primitive types if (invalidValue.getClass().getName().startsWith("java.lang.")) { return invalidValue.toString(); } return null; } private void logException(HttpServletRequest request, ValidationException exception) { LOGGER.error( "Error {} url: '{}'", exception.getClass().getSimpleName(), request.getRequestURI(), exception); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/WorkflowBulkResource.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.model.BulkResponse; import com.netflix.conductor.service.WorkflowBulkService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.WORKFLOW_BULK; /** Synchronous Bulk APIs to process the workflows in batches */ @RestController @RequestMapping(WORKFLOW_BULK) public class WorkflowBulkResource { private final WorkflowBulkService workflowBulkService; public WorkflowBulkResource(WorkflowBulkService workflowBulkService) { this.workflowBulkService = workflowBulkService; } /** * Pause the list of workflows. * * @param workflowIds - list of workflow Ids to perform pause operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ @PutMapping("/pause") @Operation(summary = "Pause the list of workflows") public BulkResponse pauseWorkflow(@RequestBody List workflowIds) { return workflowBulkService.pauseWorkflow(workflowIds); } /** * Resume the list of workflows. * * @param workflowIds - list of workflow Ids to perform resume operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ @PutMapping("/resume") @Operation(summary = "Resume the list of workflows") public BulkResponse resumeWorkflow(@RequestBody List workflowIds) { return workflowBulkService.resumeWorkflow(workflowIds); } /** * Restart the list of workflows. * * @param workflowIds - list of workflow Ids to perform restart operation on * @param useLatestDefinitions if true, use latest workflow and task definitions upon restart * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ @PostMapping("/restart") @Operation(summary = "Restart the list of completed workflow") public BulkResponse restart( @RequestBody List workflowIds, @RequestParam(value = "useLatestDefinitions", defaultValue = "false", required = false) boolean useLatestDefinitions) { return workflowBulkService.restart(workflowIds, useLatestDefinitions); } /** * Retry the last failed task for each workflow from the list. * * @param workflowIds - list of workflow Ids to perform retry operation on * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ @PostMapping("/retry") @Operation(summary = "Retry the last failed task for each workflow from the list") public BulkResponse retry(@RequestBody List workflowIds) { return workflowBulkService.retry(workflowIds); } /** * Terminate workflows execution. * * @param workflowIds - list of workflow Ids to perform terminate operation on * @param reason - description to be specified for the terminated workflow for future * references. * @return bulk response object containing a list of succeeded workflows and a list of failed * ones with errors */ @PostMapping("/terminate") @Operation(summary = "Terminate workflows execution") public BulkResponse terminate( @RequestBody List workflowIds, @RequestParam(value = "reason", required = false) String reason) { return workflowBulkService.terminate(workflowIds, reason); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/controllers/WorkflowResource.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.List; import java.util.Map; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.run.*; import com.netflix.conductor.service.WorkflowService; import com.netflix.conductor.service.WorkflowTestService; import io.swagger.v3.oas.annotations.Operation; import static com.netflix.conductor.rest.config.RequestMappingConstants.WORKFLOW; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; @RestController @RequestMapping(WORKFLOW) public class WorkflowResource { private final WorkflowService workflowService; private final WorkflowTestService workflowTestService; public WorkflowResource( WorkflowService workflowService, WorkflowTestService workflowTestService) { this.workflowService = workflowService; this.workflowTestService = workflowTestService; } @PostMapping(produces = TEXT_PLAIN_VALUE) @Operation( summary = "Start a new workflow with StartWorkflowRequest, which allows task to be executed in a domain") public String startWorkflow(@RequestBody StartWorkflowRequest request) { return workflowService.startWorkflow(request); } @PostMapping(value = "/{name}", produces = TEXT_PLAIN_VALUE) @Operation( summary = "Start a new workflow. Returns the ID of the workflow instance that can be later used for tracking") public String startWorkflow( @PathVariable("name") String name, @RequestParam(value = "version", required = false) Integer version, @RequestParam(value = "correlationId", required = false) String correlationId, @RequestParam(value = "priority", defaultValue = "0", required = false) int priority, @RequestBody Map input) { return workflowService.startWorkflow(name, version, correlationId, priority, input); } @GetMapping("/{name}/correlated/{correlationId}") @Operation(summary = "Lists workflows for the given correlation id") public List getWorkflows( @PathVariable("name") String name, @PathVariable("correlationId") String correlationId, @RequestParam(value = "includeClosed", defaultValue = "false", required = false) boolean includeClosed, @RequestParam(value = "includeTasks", defaultValue = "false", required = false) boolean includeTasks) { return workflowService.getWorkflows(name, correlationId, includeClosed, includeTasks); } @PostMapping(value = "/{name}/correlated") @Operation(summary = "Lists workflows for the given correlation id list") public Map> getWorkflows( @PathVariable("name") String name, @RequestParam(value = "includeClosed", defaultValue = "false", required = false) boolean includeClosed, @RequestParam(value = "includeTasks", defaultValue = "false", required = false) boolean includeTasks, @RequestBody List correlationIds) { return workflowService.getWorkflows(name, includeClosed, includeTasks, correlationIds); } @GetMapping("/{workflowId}") @Operation(summary = "Gets the workflow by workflow id") public Workflow getExecutionStatus( @PathVariable("workflowId") String workflowId, @RequestParam(value = "includeTasks", defaultValue = "true", required = false) boolean includeTasks) { return workflowService.getExecutionStatus(workflowId, includeTasks); } @DeleteMapping("/{workflowId}/remove") @Operation(summary = "Removes the workflow from the system") public void delete( @PathVariable("workflowId") String workflowId, @RequestParam(value = "archiveWorkflow", defaultValue = "true", required = false) boolean archiveWorkflow) { workflowService.deleteWorkflow(workflowId, archiveWorkflow); } @GetMapping("/running/{name}") @Operation(summary = "Retrieve all the running workflows") public List getRunningWorkflow( @PathVariable("name") String workflowName, @RequestParam(value = "version", defaultValue = "1", required = false) int version, @RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) { return workflowService.getRunningWorkflows(workflowName, version, startTime, endTime); } @PutMapping("/decide/{workflowId}") @Operation(summary = "Starts the decision task for a workflow") public void decide(@PathVariable("workflowId") String workflowId) { workflowService.decideWorkflow(workflowId); } @PutMapping("/{workflowId}/pause") @Operation(summary = "Pauses the workflow") public void pauseWorkflow(@PathVariable("workflowId") String workflowId) { workflowService.pauseWorkflow(workflowId); } @PutMapping("/{workflowId}/resume") @Operation(summary = "Resumes the workflow") public void resumeWorkflow(@PathVariable("workflowId") String workflowId) { workflowService.resumeWorkflow(workflowId); } @PutMapping("/{workflowId}/skiptask/{taskReferenceName}") @Operation(summary = "Skips a given task from a current running workflow") public void skipTaskFromWorkflow( @PathVariable("workflowId") String workflowId, @PathVariable("taskReferenceName") String taskReferenceName, SkipTaskRequest skipTaskRequest) { workflowService.skipTaskFromWorkflow(workflowId, taskReferenceName, skipTaskRequest); } @PostMapping(value = "/{workflowId}/rerun", produces = TEXT_PLAIN_VALUE) @Operation(summary = "Reruns the workflow from a specific task") public String rerun( @PathVariable("workflowId") String workflowId, @RequestBody RerunWorkflowRequest request) { return workflowService.rerunWorkflow(workflowId, request); } @PostMapping("/{workflowId}/restart") @Operation(summary = "Restarts a completed workflow") @ResponseStatus( value = HttpStatus.NO_CONTENT) // for backwards compatibility with 2.x client which // expects a 204 for this request public void restart( @PathVariable("workflowId") String workflowId, @RequestParam(value = "useLatestDefinitions", defaultValue = "false", required = false) boolean useLatestDefinitions) { workflowService.restartWorkflow(workflowId, useLatestDefinitions); } @PostMapping("/{workflowId}/retry") @Operation(summary = "Retries the last failed task") @ResponseStatus( value = HttpStatus.NO_CONTENT) // for backwards compatibility with 2.x client which // expects a 204 for this request public void retry( @PathVariable("workflowId") String workflowId, @RequestParam( value = "resumeSubworkflowTasks", defaultValue = "false", required = false) boolean resumeSubworkflowTasks) { workflowService.retryWorkflow(workflowId, resumeSubworkflowTasks); } @PostMapping("/{workflowId}/resetcallbacks") @Operation(summary = "Resets callback times of all non-terminal SIMPLE tasks to 0") @ResponseStatus( value = HttpStatus.NO_CONTENT) // for backwards compatibility with 2.x client which // expects a 204 for this request public void resetWorkflow(@PathVariable("workflowId") String workflowId) { workflowService.resetWorkflow(workflowId); } @DeleteMapping("/{workflowId}") @Operation(summary = "Terminate workflow execution") public void terminate( @PathVariable("workflowId") String workflowId, @RequestParam(value = "reason", required = false) String reason) { workflowService.terminateWorkflow(workflowId, reason); } @Operation( summary = "Search for workflows based on payload and other parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC.") @GetMapping(value = "/search") public SearchResult search( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return workflowService.searchWorkflows(start, size, sort, freeText, query); } @Operation( summary = "Search for workflows based on payload and other parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC.") @GetMapping(value = "/search-v2") public SearchResult searchV2( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return workflowService.searchWorkflowsV2(start, size, sort, freeText, query); } @Operation( summary = "Search for workflows based on task parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC") @GetMapping(value = "/search-by-tasks") public SearchResult searchWorkflowsByTasks( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return workflowService.searchWorkflowsByTasks(start, size, sort, freeText, query); } @Operation( summary = "Search for workflows based on task parameters", description = "use sort options as sort=:ASC|DESC e.g. sort=name&sort=workflowId:DESC." + " If order is not specified, defaults to ASC") @GetMapping(value = "/search-by-tasks-v2") public SearchResult searchWorkflowsByTasksV2( @RequestParam(value = "start", defaultValue = "0", required = false) int start, @RequestParam(value = "size", defaultValue = "100", required = false) int size, @RequestParam(value = "sort", required = false) String sort, @RequestParam(value = "freeText", defaultValue = "*", required = false) String freeText, @RequestParam(value = "query", required = false) String query) { return workflowService.searchWorkflowsByTasksV2(start, size, sort, freeText, query); } @Operation( summary = "Get the uri and path of the external storage where the workflow payload is to be stored") @GetMapping({"/externalstoragelocation", "external-storage-location"}) public ExternalStorageLocation getExternalStorageLocation( @RequestParam("path") String path, @RequestParam("operation") String operation, @RequestParam("payloadType") String payloadType) { return workflowService.getExternalStorageLocation(path, operation, payloadType); } @PostMapping(value = "test", produces = APPLICATION_JSON_VALUE) @Operation(summary = "Test workflow execution using mock data") public Workflow testWorkflow(@RequestBody WorkflowTestRequest request) { return workflowTestService.testWorkflow(request); } } ================================================ FILE: rest/src/main/java/com/netflix/conductor/rest/startup/KitchenSinkInitializer.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.startup; import java.io.IOException; import java.io.InputStreamReader; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.event.EventListener; import org.springframework.core.io.Resource; import org.springframework.http.HttpEntity; import org.springframework.stereotype.Component; import org.springframework.util.FileCopyUtils; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; import com.netflix.conductor.common.metadata.tasks.TaskDef; import static org.springframework.http.HttpHeaders.CONTENT_TYPE; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @Component public class KitchenSinkInitializer { private static final Logger LOGGER = LoggerFactory.getLogger(KitchenSinkInitializer.class); private final RestTemplate restTemplate; @Value("${loadSample:false}") private boolean loadSamples; @Value("${server.port:8080}") private int port; @Value("classpath:./kitchensink/kitchensink.json") private Resource kitchenSink; @Value("classpath:./kitchensink/sub_flow_1.json") private Resource subFlow; @Value("classpath:./kitchensink/kitchenSink-ephemeralWorkflowWithStoredTasks.json") private Resource ephemeralWorkflowWithStoredTasks; @Value("classpath:./kitchensink/kitchenSink-ephemeralWorkflowWithEphemeralTasks.json") private Resource ephemeralWorkflowWithEphemeralTasks; public KitchenSinkInitializer(RestTemplateBuilder restTemplateBuilder) { this.restTemplate = restTemplateBuilder.build(); } @EventListener(ApplicationReadyEvent.class) public void setupKitchenSink() { try { if (loadSamples) { LOGGER.info("Loading Kitchen Sink examples"); createKitchenSink(); } } catch (Exception e) { LOGGER.error("Error initializing kitchen sink", e); } } private void createKitchenSink() throws Exception { List taskDefs = new LinkedList<>(); TaskDef taskDef; for (int i = 0; i < 40; i++) { taskDef = new TaskDef("task_" + i, "task_" + i, 1, 0); taskDef.setOwnerEmail("example@email.com"); taskDefs.add(taskDef); } taskDef = new TaskDef("search_elasticsearch", "search_elasticsearch", 1, 0); taskDef.setOwnerEmail("example@email.com"); taskDefs.add(taskDef); restTemplate.postForEntity(url("/api/metadata/taskdefs"), taskDefs, Object.class); /* * Kitchensink example (stored workflow with stored tasks) */ MultiValueMap headers = new LinkedMultiValueMap<>(); headers.add(CONTENT_TYPE, APPLICATION_JSON_VALUE); HttpEntity request = new HttpEntity<>(readToString(kitchenSink), headers); restTemplate.postForEntity(url("/api/metadata/workflow/"), request, Map.class); request = new HttpEntity<>(readToString(subFlow), headers); restTemplate.postForEntity(url("/api/metadata/workflow/"), request, Map.class); restTemplate.postForEntity( url("/api/workflow/kitchensink"), Collections.singletonMap("task2Name", "task_5"), String.class); LOGGER.info("Kitchen sink workflow is created!"); /* * Kitchensink example with ephemeral workflow and stored tasks */ request = new HttpEntity<>(readToString(ephemeralWorkflowWithStoredTasks), headers); restTemplate.postForEntity(url("/api/workflow/"), request, String.class); LOGGER.info("Ephemeral Kitchen sink workflow with stored tasks is created!"); /* * Kitchensink example with ephemeral workflow and ephemeral tasks */ request = new HttpEntity<>(readToString(ephemeralWorkflowWithEphemeralTasks), headers); restTemplate.postForEntity(url("/api/workflow/"), request, String.class); LOGGER.info("Ephemeral Kitchen sink workflow with ephemeral tasks is created!"); } private String readToString(Resource resource) throws IOException { return FileCopyUtils.copyToString(new InputStreamReader(resource.getInputStream())); } private String url(String path) { return "http://localhost:" + port + path; } } ================================================ FILE: rest/src/main/resources/kitchensink/kitchenSink-ephemeralWorkflowWithEphemeralTasks.json ================================================ { "name": "kitchenSink-ephemeralWorkflowWithEphemeralTasks", "workflowDef": { "name": "ephemeralKitchenSinkEphemeralTasks", "description": "Kitchensink ephemeral workflow with ephemeral tasks", "version": 1, "tasks": [ { "name": "task_10001", "taskReferenceName": "task_10001", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE", "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "task_10001", "description": "task_10001", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "event_task", "taskReferenceName": "event_0", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "EVENT", "sink": "conductor" }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${task_2.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_10004", "taskReferenceName": "task_10004", "inputParameters": { "mod": "${task_2.output.mod}", "oddEven": "${task_2.output.oddEven}" }, "type": "SIMPLE", "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "task_10004", "description": "task_10004", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${task_10004.output.dynamicTasks}", "input": "${task_10004.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input" }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN" } ], "1": [ { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_100010", "taskReferenceName": "task_100010", "type": "SIMPLE", "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "task_100010", "description": "task_100010", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${task_10001.output.mod}", "oddEven": "${task_10001.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], [ { "name": "task_100011", "taskReferenceName": "task_100011", "type": "SIMPLE", "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "task_100011", "description": "task_100011", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${task_10001.output.mod}", "oddEven": "${task_10001.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ] ] }, { "name": "join", "taskReferenceName": "join2", "type": "JOIN", "joinOn": [ "wf3", "wf4" ] } ] } }, { "name": "search_elasticsearch", "taskReferenceName": "get_es_1", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_100030", "taskReferenceName": "task_100030", "inputParameters": { "statuses": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "type": "SIMPLE", "taskDefinition": { "ownerApp": null, "createTime": null, "updateTime": null, "createdBy": null, "updatedBy": null, "name": "task_100030", "description": "task_100030", "retryCount": 1, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "concurrentExecLimit": null, "inputTemplate": {} } } ], "outputParameters": { "statues": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "schemaVersion": 2, "ownerEmail": "example@email.com" }, "input": { "task2Name": "task_10005" } } ================================================ FILE: rest/src/main/resources/kitchensink/kitchenSink-ephemeralWorkflowWithStoredTasks.json ================================================ { "name": "kitchenSink-ephemeralWorkflowWithStoredTasks", "workflowDef": { "name": "ephemeralKitchenSinkStoredTasks", "description": "kitchensink workflow definition", "version": 1, "tasks": [ { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE" }, { "name": "event_task", "taskReferenceName": "event_0", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "EVENT", "sink": "conductor" }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${task_2.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_4", "taskReferenceName": "task_4", "inputParameters": { "mod": "${task_2.output.mod}", "oddEven": "${task_2.output.oddEven}" }, "type": "SIMPLE" }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${task_4.output.dynamicTasks}", "input": "${task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input" }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN" } ], "1": [ { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_10", "taskReferenceName": "task_10", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], [ { "name": "task_11", "taskReferenceName": "task_11", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ] ] }, { "name": "join", "taskReferenceName": "join2", "type": "JOIN", "joinOn": [ "wf3", "wf4" ] } ] } }, { "name": "search_elasticsearch", "taskReferenceName": "get_es_1", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_30", "taskReferenceName": "task_30", "inputParameters": { "statuses": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "type": "SIMPLE" } ], "outputParameters": { "statues": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "schemaVersion": 2, "ownerEmail": "example@email.com" }, "input": { "task2Name": "task_5" } } ================================================ FILE: rest/src/main/resources/kitchensink/kitchensink.json ================================================ { "name": "kitchensink", "description": "kitchensink workflow", "version": 1, "tasks": [ { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE" }, { "name": "event_task", "taskReferenceName": "event_0", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "EVENT", "sink": "conductor" }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "${workflow.input.task2Name}" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute" }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "${task_2.output.oddEven}" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_4", "taskReferenceName": "task_4", "inputParameters": { "mod": "${task_2.output.mod}", "oddEven": "${task_2.output.oddEven}" }, "type": "SIMPLE" }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "${task_4.output.dynamicTasks}", "input": "${task_4.output.inputs}" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input" }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN" } ], "1": [ { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_10", "taskReferenceName": "task_10", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], [ { "name": "task_11", "taskReferenceName": "task_11", "type": "SIMPLE" }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "${task_1.output.mod}", "oddEven": "${task_1.output.oddEven}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ] ] }, { "name": "join", "taskReferenceName": "join2", "type": "JOIN", "joinOn": [ "wf3", "wf4" ] } ] } }, { "name": "search_elasticsearch", "taskReferenceName": "get_es_1", "inputParameters": { "http_request": { "uri": "http://localhost:9200/conductor/_search?size=10", "method": "GET" } }, "type": "HTTP" }, { "name": "task_30", "taskReferenceName": "task_30", "inputParameters": { "statuses": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "type": "SIMPLE" } ], "outputParameters": { "statues": "${get_es_1.output..status}", "workflowIds": "${get_es_1.output..workflowId}" }, "ownerEmail": "example@email.com", "schemaVersion": 2 } ================================================ FILE: rest/src/main/resources/kitchensink/sub_flow_1.json ================================================ { "name": "sub_flow_1", "description": "A Simple sub-workflow with 2 tasks", "version": 1, "tasks": [ { "name": "task_5", "taskReferenceName": "task_5", "inputParameters": {}, "type": "SIMPLE" }, { "name": "task_6", "taskReferenceName": "task_6", "type": "SIMPLE" } ], "outputParameters": {}, "schemaVersion": 2, "ownerEmail": "example@email.com" } ================================================ FILE: rest/src/main/resources/kitchensink/wf1.json ================================================ { "createTime": 1477681181098, "updateTime": 1478835878290, "name": "main_workflow", "description": "Kitchensink workflow", "version": 1, "tasks": [ { "name": "task_1", "taskReferenceName": "task_1", "inputParameters": { "mod": "workflow.input.mod", "oddEven": "workflow.input.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "dyntask", "taskReferenceName": "task_2", "inputParameters": { "taskToExecute": "workflow.input.task2Name" }, "type": "DYNAMIC", "dynamicTaskNameParam": "taskToExecute", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_3", "taskReferenceName": "task_3", "inputParameters": { "mod": "task_2.output.mod", "oddEven": "task_2.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "oddEvenDecision", "taskReferenceName": "oddEvenDecision", "inputParameters": { "oddEven": "task_3.output.oddEven" }, "type": "DECISION", "caseValueParam": "oddEven", "decisionCases": { "0": [ { "name": "task_4", "taskReferenceName": "task_4", "inputParameters": { "mod": "task_3.output.mod", "oddEven": "task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "dynamic_fanout", "taskReferenceName": "fanout1", "inputParameters": { "dynamicTasks": "task_4.output.dynamicTasks", "input": "task_4.output.inputs" }, "type": "FORK_JOIN_DYNAMIC", "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "input", "startDelay": 0, "callbackFromWorker": true }, { "name": "dynamic_join", "taskReferenceName": "join1", "type": "JOIN", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_5", "taskReferenceName": "task_5", "inputParameters": { "mod": "task_4.output.mod", "oddEven": "task_4.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_6", "taskReferenceName": "task_6", "inputParameters": { "mod": "task_5.output.mod", "oddEven": "task_5.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], "1": [ { "name": "task_7", "taskReferenceName": "task_7", "inputParameters": { "mod": "task_3.output.mod", "oddEven": "task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_8", "taskReferenceName": "task_8", "inputParameters": { "mod": "task_7.output.mod", "oddEven": "task_7.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_9", "taskReferenceName": "task_9", "inputParameters": { "mod": "task_8.output.mod", "oddEven": "task_8.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "modDecision", "taskReferenceName": "modDecision", "inputParameters": { "mod": "task_8.output.mod" }, "type": "DECISION", "caseValueParam": "mod", "decisionCases": { "0": [ { "name": "task_12", "taskReferenceName": "task_12", "inputParameters": { "mod": "task_9.output.mod", "oddEven": "task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_13", "taskReferenceName": "task_13", "inputParameters": { "mod": "task_12.output.mod", "oddEven": "task_12.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "sub_workflow_x", "taskReferenceName": "wf1", "inputParameters": { "mod": "task_12.output.mod", "oddEven": "task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "callbackFromWorker": true, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "1": [ { "name": "task_15", "taskReferenceName": "task_15", "inputParameters": { "mod": "task_9.output.mod", "oddEven": "task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_16", "taskReferenceName": "task_16", "inputParameters": { "mod": "task_15.output.mod", "oddEven": "task_15.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "sub_workflow_x", "taskReferenceName": "wf2", "inputParameters": { "mod": "task_12.output.mod", "oddEven": "task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "callbackFromWorker": true, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } } ], "4": [ { "name": "task_18", "taskReferenceName": "task_18", "inputParameters": { "mod": "task_9.output.mod", "oddEven": "task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_19", "taskReferenceName": "task_19", "inputParameters": { "mod": "task_18.output.mod", "oddEven": "task_18.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], "5": [ { "name": "task_21", "taskReferenceName": "task_21", "inputParameters": { "mod": "task_9.output.mod", "oddEven": "task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "sub_workflow_x", "taskReferenceName": "wf3", "inputParameters": { "mod": "task_12.output.mod", "oddEven": "task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "callbackFromWorker": true, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "task_22", "taskReferenceName": "task_22", "inputParameters": { "mod": "task_21.output.mod", "oddEven": "task_21.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ] }, "defaultCase": [ { "name": "task_24", "taskReferenceName": "task_24", "inputParameters": { "mod": "task_9.output.mod", "oddEven": "task_9.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "sub_workflow_x", "taskReferenceName": "wf4", "inputParameters": { "mod": "task_12.output.mod", "oddEven": "task_12.output.oddEven" }, "type": "SUB_WORKFLOW", "startDelay": 0, "callbackFromWorker": true, "subWorkflowParam": { "name": "sub_flow_1", "version": 1 } }, { "name": "task_25", "taskReferenceName": "task_25", "inputParameters": { "mod": "task_24.output.mod", "oddEven": "task_24.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], "startDelay": 0, "callbackFromWorker": true } ] }, "startDelay": 0, "callbackFromWorker": true }, { "name": "task_28", "taskReferenceName": "task_28", "inputParameters": { "mod": "task_3.output.mod", "oddEven": "task_3.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_29", "taskReferenceName": "task_29", "inputParameters": { "mod": "task_28.output.mod", "oddEven": "task_28.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_30", "taskReferenceName": "task_30", "inputParameters": { "mod": "task_29.output.mod", "oddEven": "task_29.output.oddEven" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], "schemaVersion": 1 } ================================================ FILE: rest/src/main/resources/kitchensink/wf2.json ================================================ { "createTime": 1477681181098, "updateTime": 1478837752600, "name": "sub_flow_1", "description": "sub workflow", "version": 1, "tasks": [ { "name": "task_5", "taskReferenceName": "task_5", "inputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_28", "taskReferenceName": "task_28", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "fork_join", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "task_10", "taskReferenceName": "task_10", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_11", "taskReferenceName": "task_11", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], [ { "name": "task_20", "taskReferenceName": "task_20", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true }, { "name": "task_21", "taskReferenceName": "task_21", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ] ], "startDelay": 0, "callbackFromWorker": true }, { "name": "join", "taskReferenceName": "join", "type": "JOIN", "startDelay": 0, "joinOn": [ "task_21", "task_11" ], "callbackFromWorker": true }, { "name": "task_30", "taskReferenceName": "task_30", "type": "SIMPLE", "startDelay": 0, "callbackFromWorker": true } ], "outputParameters": { "mod": "${workflow.input.mod}", "oddEven": "${workflow.input.oddEven}" }, "schemaVersion": 2 } ================================================ FILE: rest/src/main/resources/static/index.html ================================================ Netflix Conductor

    Conductor Logo


    ================================================ FILE: rest/src/test/java/com/netflix/conductor/rest/controllers/AdminResourceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.service.AdminService; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class AdminResourceTest { @Mock private AdminService mockAdminService; @Mock private AdminResource adminResource; @Before public void before() { this.mockAdminService = mock(AdminService.class); this.adminResource = new AdminResource(mockAdminService); } @Test public void testGetAllConfig() { Map configs = new HashMap<>(); configs.put("config1", "test"); when(mockAdminService.getAllConfig()).thenReturn(configs); assertEquals(configs, adminResource.getAllConfig()); } @Test public void testView() { Task task = new Task(); task.setReferenceTaskName("test"); List listOfTask = new ArrayList<>(); listOfTask.add(task); when(mockAdminService.getListOfPendingTask(anyString(), anyInt(), anyInt())) .thenReturn(listOfTask); assertEquals(listOfTask, adminResource.view("testTask", 0, 100)); } @Test public void testRequeueSweep() { String workflowId = "w123"; when(mockAdminService.requeueSweep(anyString())).thenReturn(workflowId); assertEquals(workflowId, adminResource.requeueSweep(workflowId)); } @Test public void testGetEventQueues() { adminResource.getEventQueues(false); verify(mockAdminService, times(1)).getEventQueues(anyBoolean()); } } ================================================ FILE: rest/src/test/java/com/netflix/conductor/rest/controllers/EventResourceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.List; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.service.EventService; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class EventResourceTest { private EventResource eventResource; @Mock private EventService mockEventService; @Before public void setUp() { this.mockEventService = mock(EventService.class); this.eventResource = new EventResource(this.mockEventService); } @Test public void testAddEventHandler() { EventHandler eventHandler = new EventHandler(); eventResource.addEventHandler(eventHandler); verify(mockEventService, times(1)).addEventHandler(any(EventHandler.class)); } @Test public void testUpdateEventHandler() { EventHandler eventHandler = new EventHandler(); eventResource.updateEventHandler(eventHandler); verify(mockEventService, times(1)).updateEventHandler(any(EventHandler.class)); } @Test public void testRemoveEventHandlerStatus() { eventResource.removeEventHandlerStatus("testEvent"); verify(mockEventService, times(1)).removeEventHandlerStatus(anyString()); } @Test public void testGetEventHandlersForEvent() { EventHandler eventHandler = new EventHandler(); eventResource.addEventHandler(eventHandler); List listOfEventHandler = new ArrayList<>(); listOfEventHandler.add(eventHandler); when(mockEventService.getEventHandlersForEvent(anyString(), anyBoolean())) .thenReturn(listOfEventHandler); assertEquals(listOfEventHandler, eventResource.getEventHandlersForEvent("testEvent", true)); } @Test public void testGetEventHandlers() { EventHandler eventHandler = new EventHandler(); eventResource.addEventHandler(eventHandler); List listOfEventHandler = new ArrayList<>(); listOfEventHandler.add(eventHandler); when(mockEventService.getEventHandlers()).thenReturn(listOfEventHandler); assertEquals(listOfEventHandler, eventResource.getEventHandlers()); } } ================================================ FILE: rest/src/test/java/com/netflix/conductor/rest/controllers/MetadataResourceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.List; import org.junit.Before; import org.junit.Test; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.service.MetadataService; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyList; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class MetadataResourceTest { private MetadataResource metadataResource; private MetadataService mockMetadataService; @Before public void before() { this.mockMetadataService = mock(MetadataService.class); this.metadataResource = new MetadataResource(this.mockMetadataService); } @Test public void testCreateWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); metadataResource.create(workflowDef); verify(mockMetadataService, times(1)).registerWorkflowDef(any(WorkflowDef.class)); } @Test public void testValidateWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); metadataResource.validate(workflowDef); verify(mockMetadataService, times(1)).validateWorkflowDef(any(WorkflowDef.class)); } @Test public void testUpdateWorkflow() { WorkflowDef workflowDef = new WorkflowDef(); List listOfWorkflowDef = new ArrayList<>(); listOfWorkflowDef.add(workflowDef); metadataResource.update(listOfWorkflowDef); verify(mockMetadataService, times(1)).updateWorkflowDef(anyList()); } @Test public void testGetWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test"); workflowDef.setVersion(1); workflowDef.setDescription("test"); when(mockMetadataService.getWorkflowDef(anyString(), any())).thenReturn(workflowDef); assertEquals(workflowDef, metadataResource.get("test", 1)); } @Test public void testGetAllWorkflowDef() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test"); workflowDef.setVersion(1); workflowDef.setDescription("test"); List listOfWorkflowDef = new ArrayList<>(); listOfWorkflowDef.add(workflowDef); when(mockMetadataService.getWorkflowDefs()).thenReturn(listOfWorkflowDef); assertEquals(listOfWorkflowDef, metadataResource.getAll()); } @Test public void testGetAllWorkflowDefLatestVersions() { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test"); workflowDef.setVersion(1); workflowDef.setDescription("test"); List listOfWorkflowDef = new ArrayList<>(); listOfWorkflowDef.add(workflowDef); when(mockMetadataService.getWorkflowDefsLatestVersions()).thenReturn(listOfWorkflowDef); assertEquals(listOfWorkflowDef, metadataResource.getAllWorkflowsWithLatestVersions()); } @Test public void testUnregisterWorkflowDef() throws Exception { metadataResource.unregisterWorkflowDef("test", 1); verify(mockMetadataService, times(1)).unregisterWorkflowDef(anyString(), any()); } @Test public void testRegisterListOfTaskDef() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setDescription("desc"); List listOfTaskDefs = new ArrayList<>(); listOfTaskDefs.add(taskDef); metadataResource.registerTaskDef(listOfTaskDefs); verify(mockMetadataService, times(1)).registerTaskDef(listOfTaskDefs); } @Test public void testRegisterTaskDef() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setDescription("desc"); metadataResource.registerTaskDef(taskDef); verify(mockMetadataService, times(1)).updateTaskDef(taskDef); } @Test public void testGetAllTaskDefs() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setDescription("desc"); List listOfTaskDefs = new ArrayList<>(); listOfTaskDefs.add(taskDef); when(mockMetadataService.getTaskDefs()).thenReturn(listOfTaskDefs); assertEquals(listOfTaskDefs, metadataResource.getTaskDefs()); } @Test public void testGetTaskDef() { TaskDef taskDef = new TaskDef(); taskDef.setName("test"); taskDef.setDescription("desc"); when(mockMetadataService.getTaskDef(anyString())).thenReturn(taskDef); assertEquals(taskDef, metadataResource.getTaskDef("test")); } @Test public void testUnregisterTaskDef() { metadataResource.unregisterTaskDef("test"); verify(mockMetadataService, times(1)).unregisterTaskDef(anyString()); } } ================================================ FILE: rest/src/test/java/com/netflix/conductor/rest/controllers/TaskResourceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.springframework.http.ResponseEntity; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskExecLog; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.service.TaskService; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TaskResourceTest { private TaskService mockTaskService; private TaskResource taskResource; @Before public void before() { this.mockTaskService = mock(TaskService.class); this.taskResource = new TaskResource(this.mockTaskService); } @Test public void testPoll() { Task task = new Task(); task.setTaskType("SIMPLE"); task.setWorkerId("123"); task.setDomain("test"); when(mockTaskService.poll(anyString(), anyString(), anyString())).thenReturn(task); assertEquals(ResponseEntity.ok(task), taskResource.poll("SIMPLE", "123", "test")); } @Test public void testBatchPoll() { Task task = new Task(); task.setTaskType("SIMPLE"); task.setWorkerId("123"); task.setDomain("test"); List listOfTasks = new ArrayList<>(); listOfTasks.add(task); when(mockTaskService.batchPoll(anyString(), anyString(), anyString(), anyInt(), anyInt())) .thenReturn(listOfTasks); assertEquals( ResponseEntity.ok(listOfTasks), taskResource.batchPoll("SIMPLE", "123", "test", 1, 100)); } @Test public void testUpdateTask() { TaskResult taskResult = new TaskResult(); taskResult.setStatus(TaskResult.Status.COMPLETED); taskResult.setTaskId("123"); when(mockTaskService.updateTask(any(TaskResult.class))).thenReturn("123"); assertEquals("123", taskResource.updateTask(taskResult)); } @Test public void testLog() { taskResource.log("123", "test log"); verify(mockTaskService, times(1)).log(anyString(), anyString()); } @Test public void testGetTaskLogs() { List listOfLogs = new ArrayList<>(); listOfLogs.add(new TaskExecLog("test log")); when(mockTaskService.getTaskLogs(anyString())).thenReturn(listOfLogs); assertEquals(listOfLogs, taskResource.getTaskLogs("123")); } @Test public void testGetTask() { Task task = new Task(); task.setTaskType("SIMPLE"); task.setWorkerId("123"); task.setDomain("test"); task.setStatus(Task.Status.IN_PROGRESS); when(mockTaskService.getTask(anyString())).thenReturn(task); ResponseEntity entity = taskResource.getTask("123"); assertNotNull(entity); assertEquals(task, entity.getBody()); } @Test public void testSize() { Map map = new HashMap<>(); map.put("test1", 1); map.put("test2", 2); List list = new ArrayList<>(); list.add("test1"); list.add("test2"); when(mockTaskService.getTaskQueueSizes(anyList())).thenReturn(map); assertEquals(map, taskResource.size(list)); } @Test public void testAllVerbose() { Map map = new HashMap<>(); map.put("queue1", 1L); map.put("queue2", 2L); Map> mapOfMap = new HashMap<>(); mapOfMap.put("queue", map); Map>> queueSizeMap = new HashMap<>(); queueSizeMap.put("queue", mapOfMap); when(mockTaskService.allVerbose()).thenReturn(queueSizeMap); assertEquals(queueSizeMap, taskResource.allVerbose()); } @Test public void testQueueDetails() { Map map = new HashMap<>(); map.put("queue1", 1L); map.put("queue2", 2L); when(mockTaskService.getAllQueueDetails()).thenReturn(map); assertEquals(map, taskResource.all()); } @Test public void testGetPollData() { PollData pollData = new PollData("queue", "test", "w123", 100); List listOfPollData = new ArrayList<>(); listOfPollData.add(pollData); when(mockTaskService.getPollData(anyString())).thenReturn(listOfPollData); assertEquals(listOfPollData, taskResource.getPollData("w123")); } @Test public void testGetAllPollData() { PollData pollData = new PollData("queue", "test", "w123", 100); List listOfPollData = new ArrayList<>(); listOfPollData.add(pollData); when(mockTaskService.getAllPollData()).thenReturn(listOfPollData); assertEquals(listOfPollData, taskResource.getAllPollData()); } @Test public void testRequeueTaskType() { when(mockTaskService.requeuePendingTask(anyString())).thenReturn("1"); assertEquals("1", taskResource.requeuePendingTask("SIMPLE")); } @Test public void testSearch() { Task task = new Task(); task.setTaskType("SIMPLE"); task.setWorkerId("123"); task.setDomain("test"); task.setStatus(Task.Status.IN_PROGRESS); TaskSummary taskSummary = new TaskSummary(task); List listOfTaskSummary = Collections.singletonList(taskSummary); SearchResult searchResult = new SearchResult<>(100, listOfTaskSummary); when(mockTaskService.search(0, 100, "asc", "*", "*")).thenReturn(searchResult); assertEquals(searchResult, taskResource.search(0, 100, "asc", "*", "*")); } @Test public void testSearchV2() { Task task = new Task(); task.setTaskType("SIMPLE"); task.setWorkerId("123"); task.setDomain("test"); task.setStatus(Task.Status.IN_PROGRESS); List listOfTasks = Collections.singletonList(task); SearchResult searchResult = new SearchResult<>(100, listOfTasks); when(mockTaskService.searchV2(0, 100, "asc", "*", "*")).thenReturn(searchResult); assertEquals(searchResult, taskResource.searchV2(0, 100, "asc", "*", "*")); } @Test public void testGetExternalStorageLocation() { ExternalStorageLocation externalStorageLocation = mock(ExternalStorageLocation.class); when(mockTaskService.getExternalStorageLocation("path", "operation", "payloadType")) .thenReturn(externalStorageLocation); assertEquals( externalStorageLocation, taskResource.getExternalStorageLocation("path", "operation", "payloadType")); } } ================================================ FILE: rest/src/test/java/com/netflix/conductor/rest/controllers/WorkflowResourceTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.rest.controllers; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.service.WorkflowService; import com.netflix.conductor.service.WorkflowTestService; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class WorkflowResourceTest { @Mock private WorkflowService mockWorkflowService; @Mock private WorkflowTestService mockWorkflowTestService; private WorkflowResource workflowResource; @Before public void before() { this.mockWorkflowService = mock(WorkflowService.class); this.mockWorkflowTestService = mock(WorkflowTestService.class); this.workflowResource = new WorkflowResource(this.mockWorkflowService, this.mockWorkflowTestService); } @Test public void testStartWorkflow() { StartWorkflowRequest startWorkflowRequest = new StartWorkflowRequest(); startWorkflowRequest.setName("w123"); Map input = new HashMap<>(); input.put("1", "abc"); startWorkflowRequest.setInput(input); String workflowID = "w112"; when(mockWorkflowService.startWorkflow(any(StartWorkflowRequest.class))) .thenReturn(workflowID); assertEquals("w112", workflowResource.startWorkflow(startWorkflowRequest)); } @Test public void testStartWorkflowParam() { Map input = new HashMap<>(); input.put("1", "abc"); String workflowID = "w112"; when(mockWorkflowService.startWorkflow( anyString(), anyInt(), anyString(), anyInt(), anyMap())) .thenReturn(workflowID); assertEquals("w112", workflowResource.startWorkflow("test1", 1, "c123", 0, input)); } @Test public void getWorkflows() { Workflow workflow = new Workflow(); workflow.setCorrelationId("123"); ArrayList listOfWorkflows = new ArrayList<>() { { add(workflow); } }; when(mockWorkflowService.getWorkflows(anyString(), anyString(), anyBoolean(), anyBoolean())) .thenReturn(listOfWorkflows); assertEquals(listOfWorkflows, workflowResource.getWorkflows("test1", "123", true, true)); } @Test public void testGetWorklfowsMultipleCorrelationId() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); List workflowArrayList = new ArrayList<>() { { add(workflow); } }; List correlationIdList = new ArrayList<>() { { add("c123"); } }; Map> workflowMap = new HashMap<>(); workflowMap.put("c123", workflowArrayList); when(mockWorkflowService.getWorkflows(anyString(), anyBoolean(), anyBoolean(), anyList())) .thenReturn(workflowMap); assertEquals( workflowMap, workflowResource.getWorkflows("test", true, true, correlationIdList)); } @Test public void testGetExecutionStatus() { Workflow workflow = new Workflow(); workflow.setCorrelationId("c123"); when(mockWorkflowService.getExecutionStatus(anyString(), anyBoolean())) .thenReturn(workflow); assertEquals(workflow, workflowResource.getExecutionStatus("w123", true)); } @Test public void testDelete() { workflowResource.delete("w123", true); verify(mockWorkflowService, times(1)).deleteWorkflow(anyString(), anyBoolean()); } @Test public void testGetRunningWorkflow() { List listOfWorklfows = new ArrayList<>() { { add("w123"); } }; when(mockWorkflowService.getRunningWorkflows(anyString(), anyInt(), anyLong(), anyLong())) .thenReturn(listOfWorklfows); assertEquals(listOfWorklfows, workflowResource.getRunningWorkflow("w123", 1, 12L, 13L)); } @Test public void testDecide() { workflowResource.decide("w123"); verify(mockWorkflowService, times(1)).decideWorkflow(anyString()); } @Test public void testPauseWorkflow() { workflowResource.pauseWorkflow("w123"); verify(mockWorkflowService, times(1)).pauseWorkflow(anyString()); } @Test public void testResumeWorkflow() { workflowResource.resumeWorkflow("test"); verify(mockWorkflowService, times(1)).resumeWorkflow(anyString()); } @Test public void testSkipTaskFromWorkflow() { workflowResource.skipTaskFromWorkflow("test", "testTask", null); verify(mockWorkflowService, times(1)) .skipTaskFromWorkflow(anyString(), anyString(), isNull()); } @Test public void testRerun() { RerunWorkflowRequest request = new RerunWorkflowRequest(); workflowResource.rerun("test", request); verify(mockWorkflowService, times(1)) .rerunWorkflow(anyString(), any(RerunWorkflowRequest.class)); } @Test public void restart() { workflowResource.restart("w123", false); verify(mockWorkflowService, times(1)).restartWorkflow(anyString(), anyBoolean()); } @Test public void testRetry() { workflowResource.retry("w123", false); verify(mockWorkflowService, times(1)).retryWorkflow(anyString(), anyBoolean()); } @Test public void testResetWorkflow() { workflowResource.resetWorkflow("w123"); verify(mockWorkflowService, times(1)).resetWorkflow(anyString()); } @Test public void testTerminate() { workflowResource.terminate("w123", "test"); verify(mockWorkflowService, times(1)).terminateWorkflow(anyString(), anyString()); } @Test public void testSearch() { workflowResource.search(0, 100, "asc", "*", "*"); verify(mockWorkflowService, times(1)) .searchWorkflows(anyInt(), anyInt(), anyString(), anyString(), anyString()); } @Test public void testSearchV2() { workflowResource.searchV2(0, 100, "asc", "*", "*"); verify(mockWorkflowService).searchWorkflowsV2(0, 100, "asc", "*", "*"); } @Test public void testSearchWorkflowsByTasks() { workflowResource.searchWorkflowsByTasks(0, 100, "asc", "*", "*"); verify(mockWorkflowService, times(1)) .searchWorkflowsByTasks(anyInt(), anyInt(), anyString(), anyString(), anyString()); } @Test public void testSearchWorkflowsByTasksV2() { workflowResource.searchWorkflowsByTasksV2(0, 100, "asc", "*", "*"); verify(mockWorkflowService).searchWorkflowsByTasksV2(0, 100, "asc", "*", "*"); } @Test public void testGetExternalStorageLocation() { workflowResource.getExternalStorageLocation("path", "operation", "payloadType"); verify(mockWorkflowService).getExternalStorageLocation("path", "operation", "payloadType"); } } ================================================ FILE: server/build.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ plugins { id 'org.springframework.boot' } dependencies { implementation project(':conductor-rest') implementation project(':conductor-core') implementation project(':conductor-redis-persistence') implementation project(':conductor-cassandra-persistence') implementation project(':conductor-es6-persistence') implementation project(':conductor-grpc-server') implementation project(':conductor-redis-lock') implementation project(':conductor-redis-concurrency-limit') implementation project(':conductor-http-task') implementation project(':conductor-json-jq-task') implementation project(':conductor-awss3-storage') implementation project(':conductor-awssqs-event-queue') implementation 'org.springframework.boot:spring-boot-starter' implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.retry:spring-retry' implementation 'org.springframework.boot:spring-boot-starter-log4j2' implementation 'org.apache.logging.log4j:log4j-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation "io.orkes.queues:orkes-conductor-queues:${revOrkesQueues}" implementation "org.springdoc:springdoc-openapi-ui:${revOpenapi}" runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" testImplementation project(':conductor-rest') testImplementation project(':conductor-common') testImplementation "io.grpc:grpc-testing:${revGrpc}" testImplementation "com.google.protobuf:protobuf-java:${revProtoBuf}" testImplementation "io.grpc:grpc-protobuf:${revGrpc}" testImplementation "io.grpc:grpc-stub:${revGrpc}" } jar { enabled = true } bootJar { mainClass = 'com.netflix.conductor.Conductor' classifier = 'boot' } // https://docs.spring.io/spring-boot/docs/current/gradle-plugin/reference/htmlsingle/#integrating-with-actuator.build-info // This will configure a BuildInfo task named bootBuildInfo springBoot { buildInfo() } compileJava.dependsOn bootBuildInfo ================================================ FILE: server/src/main/java/com/netflix/conductor/Conductor.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor; import java.io.IOException; import java.util.Properties; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.context.annotation.ComponentScan; import org.springframework.core.io.FileSystemResource; // Prevents from the datasource beans to be loaded, AS they are needed only for specific databases. // In case that SQL database is selected this class will be imported back in the appropriate // database persistence module. @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @ComponentScan(basePackages = {"com.netflix.conductor", "io.orkes.conductor"}) public class Conductor { private static final Logger log = LoggerFactory.getLogger(Conductor.class); public static void main(String[] args) throws IOException { loadExternalConfig(); SpringApplication.run(Conductor.class, args); } /** * Reads properties from the location specified in CONDUCTOR_CONFIG_FILE and sets * them as system properties so they override the default properties. * *

    Spring Boot property hierarchy is documented here, * https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config * * @throws IOException if file can't be read. */ private static void loadExternalConfig() throws IOException { String configFile = System.getProperty("CONDUCTOR_CONFIG_FILE"); if (StringUtils.isNotBlank(configFile)) { FileSystemResource resource = new FileSystemResource(configFile); if (resource.exists()) { Properties properties = new Properties(); properties.load(resource.getInputStream()); properties.forEach( (key, value) -> System.setProperty((String) key, (String) value)); log.info("Loaded {} properties from {}", properties.size(), configFile); } else { log.warn("Ignoring {} since it does not exist", configFile); } } } } ================================================ FILE: server/src/main/resources/META-INF/additional-spring-configuration-metadata.json ================================================ { "properties": [ { "name": "conductor.db.type", "type": "java.lang.String", "description": "The type of database to be used while running the Conductor application." }, { "name": "conductor.indexing.enabled", "type": "java.lang.Boolean", "description": "Enable indexing to elasticsearch. If set to false, a no-op implementation will be used." }, { "name": "conductor.grpc-server.enabled", "type": "java.lang.Boolean", "description": "Enable the gRPC server." } ], "hints": [ { "name": "conductor.db.type", "values": [ { "value": "memory", "description": "Use in-memory redis as the database implementation." }, { "value": "cassandra", "description": "Use cassandra as the database implementation." }, { "value": "mysql", "description": "Use MySQL as the database implementation." }, { "value": "postgres", "description": "Use Postgres as the database implementation." }, { "value": "dynomite", "description": "Use Dynomite as the database implementation." }, { "value": "redis_cluster", "description": "Use Redis Cluster configuration as the database implementation." }, { "value": "redis_sentinel", "description": "Use Redis Sentinel configuration as the database implementation." }, { "value": "redis_standalone", "description": "Use Redis Standalone configuration as the database implementation." } ] } ] } ================================================ FILE: server/src/main/resources/application.properties ================================================ # # Copyright 2021 Netflix, Inc. #

    # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at #

    # http://www.apache.org/licenses/LICENSE-2.0 #

    # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. # spring.application.name=conductor springdoc.api-docs.path=/api-docs loadSample=true conductor.db.type=memory conductor.queue.type=redis_standalone conductor.indexing.enabled=false #Redis configuration details. #format is host:port:rack separated by semicolon #Auth is supported. Password is taken from host[0]. format: host:port:rack:password #conductor.redis.hosts=host1:port:rack;host2:port:rack:host3:port:rack conductor.redis.hosts=localhost:6379:us-east-1c #namespace for the keys stored in Dynomite/Redis conductor.redis.workflowNamespacePrefix= #namespace prefix for the dyno queues conductor.redis.queueNamespacePrefix= #no. of threads allocated to dyno-queues queues.dynomite.threads=10 # By default with dynomite, we want the repair service enabled conductor.workflow-repair-service.enabled=true #non-quorum port used to connect to local redis. Used by dyno-queues conductor.redis.queuesNonQuorumPort=22122 # For a single node dynomite or redis server, make sure the value below is set to same as rack specified in the "workflow.dynomite.cluster.hosts" property. conductor.redis.availabilityZone=us-east-1c #conductor.redis.maxIdleConnections=8 #conductor.redis.minIdleConnections=5 #conductor.redis.minEvictableIdleTimeMillis = 1800000 #conductor.redis.timeBetweenEvictionRunsMillis = -1L #conductor.redis.testWhileIdle = false #conductor.redis.numTestsPerEvictionRun = 3 #Transport address to elasticsearch conductor.elasticsearch.url=localhost:9300 #Name of the elasticsearch cluster conductor.elasticsearch.indexName=conductor #Elasticsearch major release version. conductor.elasticsearch.version=6 #conductor.elasticsearch.version=7 # Default event queue type to listen on for wait task conductor.default-event-queue.type=sqs #zookeeper # conductor.zookeeper-lock.connectionString=host1.2181,host2:2181,host3:2181 # conductor.zookeeper-lock.sessionTimeoutMs # conductor.zookeeper-lock.connectionTimeoutMs # conductor.zookeeper-lock.namespace #disable locking during workflow execution conductor.app.workflow-execution-lock-enabled=false conductor.workflow-execution-lock.type=noop_lock #Redis cluster settings for locking module # conductor.redis-lock.serverType=single #Comma separated list of server nodes # conductor.redis-lock.serverAddress=redis://127.0.0.1:6379 #Redis sentinel master name # conductor.redis-lock.serverMasterName=master # conductor.redis-lock.namespace #Following properties set for using AMQP events and tasks with conductor: #(To enable support of AMQP queues) #conductor.event-queues.amqp.enabled=true # Here are the settings with default values: #conductor.event-queues.amqp.hosts= #conductor.event-queues.amqp.username= #conductor.event-queues.amqp.password= #conductor.event-queues.amqp.virtualHost=/ #conductor.event-queues.amqp.port=5672 #conductor.event-queues.amqp.useNio=false #conductor.event-queues.amqp.batchSize=1 #conductor.event-queues.amqp.pollTimeDuration=100ms #conductor.event-queues.amqp.queueType=classic #conductor.event-queues.amqp.sequentialMsgProcessing=true #conductor.event-queues.amqp.connectionTimeoutInMilliSecs=180000 #conductor.event-queues.amqp.networkRecoveryIntervalInMilliSecs=5000 #conductor.event-queues.amqp.requestHeartbeatTimeoutInSecs=30 #conductor.event-queues.amqp.handshakeTimeoutInMilliSecs=180000 #conductor.event-queues.amqp.maxChannelCount=5000 #conductor.event-queues.amqp.limit=50 #conductor.event-queues.amqp.duration=1000 #conductor.event-queues.amqp.retryType=REGULARINTERVALS #conductor.event-queues.amqp.useExchange=true( exchange or queue) #conductor.event-queues.amqp.listenerQueuePrefix=myqueue # Use durable queue ? #conductor.event-queues.amqp.durable=false # Use exclusive queue ? #conductor.event-queues.amqp.exclusive=false # Enable support of priorities on queue. Set the max priority on message. # Setting is ignored if the value is lower or equals to 0 #conductor.event-queues.amqp.maxPriority=-1 # To enable Workflow/Task Summary Input/Output JSON Serialization, use the following: # conductor.app.summary-input-output-json-serialization.enabled=true # Additional modules for metrics collection exposed to Prometheus (optional) # conductor.metrics-prometheus.enabled=true # management.endpoints.web.exposure.include=prometheus # Additional modules for metrics collection exposed to Datadog (optional) management.metrics.export.datadog.enabled=${conductor.metrics-datadog.enabled:false} management.metrics.export.datadog.api-key=${conductor.metrics-datadog.api-key:} ================================================ FILE: server/src/main/resources/banner.txt ================================================ ______ ______ .__ __. _______ __ __ ______ .___________. ______ .______ / | / __ \ | \ | | | \ | | | | / || | / __ \ | _ \ | ,----'| | | | | \| | | .--. || | | | | ,----'`---| |----`| | | | | |_) | | | | | | | | . ` | | | | || | | | | | | | | | | | | / | `----.| `--' | | |\ | | '--' || `--' | | `----. | | | `--' | | |\ \----. \______| \______/ |__| \__| |_______/ \______/ \______| |__| \______/ | _| `._____| ${application.formatted-version} :::Spring Boot:::${spring-boot.formatted-version} ================================================ FILE: server/src/main/resources/log4j2.xml ================================================ ================================================ FILE: server/src/test/java/com/netflix/conductor/common/config/ConductorObjectMapperTest.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.common.config; import java.io.IOException; import java.io.StringWriter; import java.util.HashMap; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.Any; import com.google.protobuf.Struct; import com.google.protobuf.Value; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** * Tests the customized {@link ObjectMapper} that is used by {@link com.netflix.conductor.Conductor} * application. */ @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) @RunWith(SpringRunner.class) @TestPropertySource(properties = "conductor.queue.type=") public class ConductorObjectMapperTest { @Autowired ObjectMapper objectMapper; @Test public void testSimpleMapping() throws IOException { assertTrue(objectMapper.canSerialize(Any.class)); Struct struct1 = Struct.newBuilder() .putFields( "some-key", Value.newBuilder().setStringValue("some-value").build()) .build(); Any source = Any.pack(struct1); StringWriter buf = new StringWriter(); objectMapper.writer().writeValue(buf, source); Any dest = objectMapper.reader().forType(Any.class).readValue(buf.toString()); assertEquals(source.getTypeUrl(), dest.getTypeUrl()); Struct struct2 = dest.unpack(Struct.class); assertTrue(struct2.containsFields("some-key")); assertEquals( struct1.getFieldsOrThrow("some-key").getStringValue(), struct2.getFieldsOrThrow("some-key").getStringValue()); } @Test public void testNullOnWrite() throws JsonProcessingException { Map data = new HashMap<>(); data.put("someKey", null); data.put("someId", "abc123"); String result = objectMapper.writeValueAsString(data); assertTrue(result.contains("null")); } @Test public void testWorkflowSerDe() throws IOException { WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("testDef"); workflowDef.setVersion(2); Workflow workflow = new Workflow(); workflow.setWorkflowDefinition(workflowDef); workflow.setWorkflowId("test-workflow-id"); workflow.setStatus(Workflow.WorkflowStatus.RUNNING); workflow.setStartTime(10L); workflow.setInput(null); Map data = new HashMap<>(); data.put("someKey", null); data.put("someId", "abc123"); workflow.setOutput(data); String workflowPayload = objectMapper.writeValueAsString(workflow); Workflow workflow1 = objectMapper.readValue(workflowPayload, Workflow.class); assertTrue(workflow1.getOutput().containsKey("someKey")); assertNull(workflow1.getOutput().get("someKey")); assertNotNull(workflow1.getInput()); } } ================================================ FILE: settings.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ plugins { id "com.gradle.enterprise" version "3.11.1" } gradleEnterprise { buildScan { termsOfServiceUrl = "https://gradle.com/terms-of-service" termsOfServiceAgree = "yes" publishAlways() buildScanPublished { scan -> file("buildscan.log") << "${new Date()} - ${scan.buildScanUri}\n" } } } rootProject.name = 'conductor' include 'annotations' include 'annotations-processor' include 'server' include 'common' include 'core' include 'client' include 'client-spring' include 'cassandra-persistence' include 'redis-persistence' include 'es6-persistence' include 'redis-lock' include 'awss3-storage' include 'awssqs-event-queue' include 'redis-concurrency-limit' include 'json-jq-task' include 'http-task' include 'rest' include 'grpc' include 'grpc-server' include 'grpc-client' include 'java-sdk' include 'test-harness' rootProject.children.each {it.name="conductor-${it.name}"} ================================================ FILE: springboot-bom-overrides.gradle ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ // Contains overrides for Spring Boot Dependency Management plugin // Dependency version override properties can be found at https://docs.spring.io/spring-boot/docs/2.7.3/reference/htmlsingle/#appendix.dependency-versions.properties // Conductor's default is ES6, but SB brings in ES7 ext['elasticsearch.version'] = revElasticSearch6 // SB brings groovy 3.0.x which is not compatible with Spock ext['groovy.version'] = revGroovy ================================================ FILE: test-harness/build.gradle ================================================ apply plugin: 'groovy' dependencies { testImplementation project(':conductor-server') testImplementation project(':conductor-common') testImplementation project(':conductor-rest') testImplementation project(':conductor-core') testImplementation project(':conductor-redis-persistence') testImplementation project(':conductor-cassandra-persistence') testImplementation project(':conductor-es6-persistence') testImplementation project(':conductor-grpc-server') testImplementation project(':conductor-client') testImplementation project(':conductor-grpc-client') testImplementation project(':conductor-json-jq-task') testImplementation project(':conductor-http-task') testImplementation "org.springframework.retry:spring-retry" testImplementation "com.fasterxml.jackson.core:jackson-databind:${revFasterXml}" testImplementation "com.fasterxml.jackson.core:jackson-core:${revFasterXml}" testImplementation "org.apache.commons:commons-lang3" testImplementation "com.google.protobuf:protobuf-java:${revProtoBuf}" testImplementation "com.google.guava:guava:${revGuava}" testImplementation "org.springframework:spring-web" testImplementation "redis.clients:jedis:${revJedis}" testImplementation "com.netflix.dyno-queues:dyno-queues-redis:${revDynoQueues}" testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" testImplementation "org.elasticsearch.client:elasticsearch-rest-client" testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation('junit:junit:4.13.2') testImplementation "org.junit.vintage:junit-vintage-engine" testImplementation "javax.ws.rs:javax.ws.rs-api:${revJAXRS}" testImplementation "org.glassfish.jersey.core:jersey-common:${revJerseyCommon}" } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.base import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Primary import org.springframework.test.context.TestPropertySource import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.redis.dao.DynoQueueDAO import com.netflix.conductor.redis.jedis.JedisMock import com.netflix.dyno.connectionpool.Host import com.netflix.dyno.queues.ShardSupplier import com.netflix.dyno.queues.redis.RedisQueues import redis.clients.jedis.commands.JedisCommands import spock.mock.DetachedMockFactory @TestPropertySource(properties = [ "conductor.system-task-workers.enabled=false", "conductor.workflow-repair-service.enabled=true", "conductor.workflow-reconciler.enabled=false", "conductor.integ-test.queue-spy.enabled=true" ]) abstract class AbstractResiliencySpecification extends AbstractSpecification { @Configuration static class TestQueueConfiguration { @Primary @Bean @ConditionalOnProperty(name = "conductor.integ-test.queue-spy.enabled", havingValue = "true") QueueDAO SpyQueueDAO() { DetachedMockFactory detachedMockFactory = new DetachedMockFactory() JedisCommands jedisMock = new JedisMock() ShardSupplier shardSupplier = new ShardSupplier() { @Override Set getQueueShards() { return new HashSet<>(Collections.singletonList("a")) } @Override String getCurrentShard() { return "a" } @Override String getShardForHost(Host host) { return "a" } } RedisQueues redisQueues = new RedisQueues(jedisMock, jedisMock, "mockedQueues", shardSupplier, 60000, 120000) DynoQueueDAO dynoQueueDAO = new DynoQueueDAO(redisQueues) return detachedMockFactory.Spy(dynoQueueDAO) } } @Autowired QueueDAO queueDAO } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.base import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.TestPropertySource import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.WorkflowExecutor import com.netflix.conductor.core.operation.StartWorkflowOperation import com.netflix.conductor.core.reconciliation.WorkflowSweeper import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.netflix.conductor.test.util.WorkflowTestUtil import spock.lang.Specification @SpringBootTest @TestPropertySource(locations = "classpath:application-integrationtest.properties") abstract class AbstractSpecification extends Specification { @Autowired ExecutionService workflowExecutionService @Autowired MetadataService metadataService @Autowired WorkflowExecutor workflowExecutor @Autowired WorkflowTestUtil workflowTestUtil @Autowired WorkflowSweeper workflowSweeper @Autowired AsyncSystemTaskExecutor asyncSystemTaskExecutor @Autowired StartWorkflowOperation startWorkflowOperation def cleanup() { workflowTestUtil.clearWorkflows() } void sweep(String workflowId) { workflowSweeper.sweep(workflowId) } protected String startWorkflow(String name, Integer version, String correlationId, Map workflowInput, String workflowInputPath) { StartWorkflowInput input = new StartWorkflowInput(name: name, version: version, correlationId: correlationId, workflowInput: workflowInput, externalInputPayloadStoragePath: workflowInputPath) startWorkflowOperation.execute(input) } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/DecisionTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import spock.lang.Unroll import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class DecisionTaskSpec extends AbstractSpecification { @Autowired Join joinTask @Shared def DECISION_WF = "DecisionWorkflow" @Shared def FORK_JOIN_DECISION_WF = "ForkConditionalTest" @Shared def COND_TASK_WF = "ConditionalTaskWF" def setup() { //initialization code for each feature workflowTestUtil.registerWorkflows('simple_decision_task_integration_test.json', 'decision_and_fork_join_integration_test.json', 'conditional_task_workflow_integration_test.json') } def "Test simple decision workflow"() { given: "Workflow an input of a workflow with decision task" Map input = new HashMap() input['param1'] = 'p1' input['param2'] = 'p2' input['case'] = 'c' when: "A decision workflow is started with the workflow input" def workflowInstanceId = startWorkflow(DECISION_WF, 1, 'decision_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DECISION' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'DECISION' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_20' tasks[3].status == Task.Status.SCHEDULED } when: "the task 'integration_task_20' is polled and completed" def polledAndCompletedTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask20Try1) and: "verify that the 'integration_task_20' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[3].taskType == 'integration_task_20' tasks[3].status == Task.Status.COMPLETED } } def "Test a workflow that has a decision task that leads to a fork join"() { given: "Workflow an input of a workflow with decision task" Map input = new HashMap() input['param1'] = 'p1' input['param2'] = 'p2' input['case'] = 'c' when: "A decision workflow is started with the workflow input" def workflowInstanceId = startWorkflow(FORK_JOIN_DECISION_WF, 1, 'decision_forkjoin', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS } when: "the tasks 'integration_task_1' and 'integration_task_10' are polled and completed" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("joinTask").taskId def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') def polledAndCompletedTask10Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_10', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) verifyPolledAndAcknowledgedTask(polledAndCompletedTask10Try1) and: "verify that the 'integration_task_1' and 'integration_task_10' are COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_20' tasks[6].status == Task.Status.SCHEDULED } when: "the task 'integration_task_20' is polled and completed" def polledAndCompletedTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task1.integration.worker') and: "the workflow is evaluated" sweep(workflowInstanceId) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask20Try1) when: "JOIN task is polled and executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that JOIN is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_20' tasks[6].status == Task.Status.COMPLETED } } def "Test default case condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the default case is executed" Map input = new HashMap() input['param1'] = 'xxx' input['param2'] = 'two' when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, 'conditional_default', input, null) then: "verify that the workflow is running and the default condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DECISION' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['caseOutput'] == ['xxx'] tasks[1].taskType == 'integration_task_10' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_10' is polled and completed" def polledAndCompletedTask10Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_10', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask10Try1) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_10' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'DECISION' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['caseOutput'] == ['null'] } } @Unroll def "Test case 'nested' and '#caseValue' condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the 'nested' and '#caseValue' decision tree is executed" Map input = new HashMap() input['param1'] = 'nested' input['param2'] = caseValue when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, workflowCorrelationId, input, null) then: "verify that the workflow is running and the 'nested' and '#caseValue' condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'DECISION' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['caseOutput'] == ['nested'] tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[1].outputData['caseOutput'] == [caseValue] tasks[2].taskType == expectedTaskName tasks[2].status == Task.Status.SCHEDULED } when: "the task '#expectedTaskName' is polled and completed" def polledAndCompletedTaskTry1 = workflowTestUtil.pollAndCompleteTask(expectedTaskName, 'task.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTaskTry1) and: with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[2].taskType == expectedTaskName tasks[2].status == endTaskStatus tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[3].outputData['caseOutput'] == ['null'] } where: caseValue | expectedTaskName | workflowCorrelationId || endTaskStatus 'two' | 'integration_task_2' | 'conditional_nested_two' || Task.Status.COMPLETED 'one' | 'integration_task_1' | 'conditional_nested_one' || Task.Status.COMPLETED } def "Test 'three' case condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the default case is executed" Map input = new HashMap() input['param1'] = 'three' input['param2'] = 'two' input['finalCase'] = 'notify' when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, 'conditional_three', input, null) then: "verify that the workflow is running and the 'three' condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DECISION' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['caseOutput'] == ['three'] tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_3' is polled and completed" def polledAndCompletedTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask3Try1) and: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'DECISION' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['caseOutput'] == ['notify'] tasks[3].taskType == 'integration_task_4' tasks[3].status == Task.Status.SCHEDULED } when: "the task 'integration_task_4' is polled and completed" def polledAndCompletedTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask4Try1) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'DECISION' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['caseOutput'] == ['notify'] tasks[3].taskType == 'integration_task_4' tasks[3].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/DoWhileSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.common.utils.TaskUtils import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.test.base.AbstractSpecification import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class DoWhileSpec extends AbstractSpecification { @Autowired Join joinTask @Autowired SubWorkflow subWorkflowTask def setup() { workflowTestUtil.registerWorkflows('do_while_integration_test.json', 'do_while_multiple_integration_test.json', 'do_while_as_subtask_integration_test.json', 'simple_one_task_sub_workflow_integration_test.json', 'do_while_iteration_fix_test.json', 'do_while_sub_workflow_integration_test.json', 'do_while_five_loop_over_integration_test.json', 'do_while_system_tasks.json', 'do_while_with_decision_task.json', 'do_while_set_variable_fix.json') } def "Test workflow with 2 iterations of five tasks"() { given: "Number of iterations of the loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 2 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("do_while_five_loop_over_integration_test", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[1].iteration == 1 tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].status == Task.Status.COMPLETED tasks[2].iteration == 1 tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.SCHEDULED tasks[3].iteration == 1 } when: "Polling and completing first task" Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JSON_JQ_TRANSFORM' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED } when: "Polling and completing second task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 9 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JSON_JQ_TRANSFORM' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'LAMBDA' tasks[6].status == Task.Status.COMPLETED tasks[6].iteration == 2 tasks[7].taskType == 'JSON_JQ_TRANSFORM' tasks[7].status == Task.Status.COMPLETED tasks[7].iteration == 2 tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.SCHEDULED tasks[8].iteration == 2 } when: "Polling and completing first task" polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 2) when: "Polling and completing second task" polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 12 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JSON_JQ_TRANSFORM' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'LAMBDA' tasks[6].status == Task.Status.COMPLETED tasks[6].iteration == 2 tasks[7].taskType == 'JSON_JQ_TRANSFORM' tasks[7].status == Task.Status.COMPLETED tasks[7].iteration == 2 tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.COMPLETED tasks[8].iteration == 2 tasks[9].taskType == 'JSON_JQ_TRANSFORM' tasks[9].status == Task.Status.COMPLETED tasks[9].iteration == 2 tasks[10].taskType == 'integration_task_2' tasks[10].status == Task.Status.COMPLETED tasks[10].iteration == 2 tasks[11].taskType == 'integration_task_3' tasks[11].status == Task.Status.SCHEDULED tasks[11].iteration == 0 // this is outside DO_WHILE } when: "Polling and completing last task" polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 12 tasks[11].taskType == 'integration_task_3' tasks[11].status == Task.Status.COMPLETED tasks[11].iteration == 0 } } def "Test workflow with 2 iterations of 3 system tasks"() { given: "Number of iterations of the loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 2 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("do_while_system_tasks", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[1].iteration == 1 tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].status == Task.Status.COMPLETED tasks[2].iteration == 1 tasks[3].taskType == 'JSON_JQ_TRANSFORM' tasks[3].status == Task.Status.COMPLETED tasks[3].iteration == 1 tasks[4].taskType == 'LAMBDA' tasks[4].status == Task.Status.COMPLETED tasks[4].iteration == 2 tasks[5].taskType == 'JSON_JQ_TRANSFORM' tasks[5].status == Task.Status.COMPLETED tasks[5].iteration == 2 tasks[6].taskType == 'JSON_JQ_TRANSFORM' tasks[6].status == Task.Status.COMPLETED tasks[6].iteration == 2 tasks[7].taskType == 'integration_task_1' tasks[7].status == Task.Status.SCHEDULED tasks[7].iteration == 0 // outside the loop } when: "Polling and completing first task" Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 8 tasks[7].taskType == 'integration_task_1' tasks[7].status == Task.Status.COMPLETED tasks[7].iteration == 0 // outside the loop } } def "Test workflow with a single iteration Do While task"() { given: "Number of iterations of the loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 1 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("Do_While_Workflow", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.SCHEDULED } when: "Polling and completing first task" Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing second task" def joinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__1").taskId Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinId) then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED } } def "Test workflow with a single iteration Do While task with Sub workflow"() { given: "Number of iterations of the loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 1 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("Do_While_Sub_Workflow", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.SCHEDULED } when: "Polling and completing first task" Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing second task" def joinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__1").taskId Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinId) then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.SCHEDULED } when: "the sub workflow is started by issuing a system task call" def parentWorkflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = parentWorkflow.getTaskByRefName('st1__1').taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) then: "verify that the sub workflow task is in a IN PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.IN_PROGRESS } when: "sub workflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowInstanceId = workflow.getTaskByRefName('st1__1').subWorkflowId then: "verify that the sub workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.SCHEDULED } when: "the 'simple_task_in_sub_wf' belonging to the sub workflow is polled and completed" def polledAndCompletedSubWorkflowTask = workflowTestUtil.pollAndCompleteTask('simple_task_in_sub_wf', 'subworkflow.task.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndCompletedSubWorkflowTask) and: "verify that the sub workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'simple_task_in_sub_wf' } and: "the parent workflow is swept" sweep(workflowInstanceId) and: "verify that the workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.COMPLETED } } def "Test workflow with multiple Do While tasks with multiple iterations"() { given: "Number of iterations of the first loop is set to 2 and second loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 2 workflowInput['loop2'] = 1 when: "A workflow with multiple do while tasks with multiple iterations is started" def workflowInstanceId = startWorkflow("Do_While_Multiple", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.SCHEDULED } when: "Polling and completing first task" Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing second task" def join1Id = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__1").taskId Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, join1Id) then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0' tasks[6].status == Task.Status.SCHEDULED } when: "Polling and completing second iteration of first task" Tuple polledAndCompletedSecondIterationTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedSecondIterationTask0, [:]) verifyTaskIteration(polledAndCompletedSecondIterationTask0[0] as Task, 2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 11 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'FORK' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.SCHEDULED tasks[9].taskType == 'integration_task_2' tasks[9].status == Task.Status.SCHEDULED tasks[10].taskType == 'JOIN' tasks[10].status == Task.Status.IN_PROGRESS } when: "Polling and completing second iteration of second task" def join2Id = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__2").taskId Tuple polledAndCompletedSecondIterationTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedSecondIterationTask1) verifyTaskIteration(polledAndCompletedSecondIterationTask1[0] as Task, 2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 11 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'FORK' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'integration_task_2' tasks[9].status == Task.Status.SCHEDULED tasks[10].taskType == 'JOIN' tasks[10].status == Task.Status.IN_PROGRESS } when: "Polling and completing second iteration of third task" Tuple polledAndCompletedSecondIterationTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, join2Id) then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedSecondIterationTask2) verifyTaskIteration(polledAndCompletedSecondIterationTask2[0] as Task, 2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 13 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'FORK' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'integration_task_2' tasks[9].status == Task.Status.COMPLETED tasks[10].taskType == 'JOIN' tasks[10].status == Task.Status.COMPLETED tasks[11].taskType == 'DO_WHILE' tasks[11].status == Task.Status.IN_PROGRESS tasks[12].taskType == 'integration_task_3' tasks[12].status == Task.Status.SCHEDULED } when: "Polling and completing task within the second do while" Tuple polledAndCompletedIntegrationTask3 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedIntegrationTask3) verifyTaskIteration(polledAndCompletedIntegrationTask3[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 13 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_1' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_2' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'FORK' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'integration_task_1' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'integration_task_2' tasks[9].status == Task.Status.COMPLETED tasks[10].taskType == 'JOIN' tasks[10].status == Task.Status.COMPLETED tasks[11].taskType == 'DO_WHILE' tasks[11].status == Task.Status.COMPLETED tasks[12].taskType == 'integration_task_3' tasks[12].status == Task.Status.COMPLETED } } def "Test retrying a failed do while workflow"() { setup: "Update the task definition with no retries" def taskName = 'integration_task_0' def persistedTaskDefinition = workflowTestUtil.getPersistedTaskDefinition(taskName).get() def modifiedTaskDefinition = new TaskDef(persistedTaskDefinition.name, persistedTaskDefinition.description, persistedTaskDefinition.ownerEmail, 0, persistedTaskDefinition.timeoutSeconds, persistedTaskDefinition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTaskDefinition) when: "A do while workflow is started" def workflowInput = new HashMap() workflowInput['loop'] = 1 def workflowInstanceId = startWorkflow("Do_While_Workflow", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.SCHEDULED } when: "Polling and failing first task" Tuple polledAndFailedTask0 = workflowTestUtil.pollAndFailTask('integration_task_0', 'integration.test.worker', "induced..failure") then: "Verify that the task was polled and acknowledged and workflow is in failed state" verifyPolledAndAcknowledgedTask(polledAndFailedTask0) verifyTaskIteration(polledAndFailedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.CANCELED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED } when: "The workflow is retried" workflowExecutor.retry(workflowInstanceId, false) then: "Verify that workflow is running" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.SCHEDULED } when: "Polling and completing first task" Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS } when: "Polling and completing second task" def joinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__1").taskId Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinId) then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.COMPLETED } cleanup: "Reset the task definition" metadataService.updateTaskDef(persistedTaskDefinition) } def "Test auto retrying a failed do while workflow"() { setup: "Update the task definition with retryCount to 1 and retryDelaySeconds to 0" def taskName = 'integration_task_0' def persistedTaskDefinition = workflowTestUtil.getPersistedTaskDefinition(taskName).get() def modifiedTaskDefinition = new TaskDef(persistedTaskDefinition.name, persistedTaskDefinition.description, persistedTaskDefinition.ownerEmail, 1, persistedTaskDefinition.timeoutSeconds, persistedTaskDefinition.responseTimeoutSeconds) modifiedTaskDefinition.setRetryDelaySeconds(0) metadataService.updateTaskDef(modifiedTaskDefinition) when: "A do while workflow is started" def workflowInput = new HashMap() workflowInput['loop'] = 1 def workflowInstanceId = startWorkflow("Do_While_Workflow", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.SCHEDULED } when: "Polling and failing first task" Tuple polledAndFailedTask0 = workflowTestUtil.pollAndFailTask('integration_task_0', 'integration.test.worker', "induced..failure") then: "Verify that the task was polled and acknowledged and retried task was generated and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndFailedTask0) verifyTaskIteration(polledAndFailedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.SCHEDULED tasks[2].retryCount == 1 tasks[2].retriedTaskId == tasks[1].taskId } when: "Polling and completing first task" Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS } when: "Polling and completing second task" def joinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join__1").taskId Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinId) then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) verifyTaskIteration(polledAndCompletedTask2[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'FORK' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_1' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.COMPLETED } cleanup: "Reset the task definition" metadataService.updateTaskDef(persistedTaskDefinition) } def "Test workflow with a iteration Do While task as subtask of a forkjoin task"() { given: "Number of iterations of the loop is set to 1" def workflowInput = new HashMap() workflowInput['loop'] = 1 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("Do_While_SubTask", 1, "looptest", workflowInput, null) then: "Verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DO_WHILE' tasks[1].status == Task.Status.IN_PROGRESS tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'integration_task_0' tasks[4].status == Task.Status.SCHEDULED } when: "Polling and completing first task in DO While" def joinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join").taskId Tuple polledAndCompletedTask0 = workflowTestUtil.pollAndCompleteTask('integration_task_0', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask0) verifyTaskIteration(polledAndCompletedTask0[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DO_WHILE' tasks[1].status == Task.Status.IN_PROGRESS tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'integration_task_0' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_1' tasks[5].status == Task.Status.SCHEDULED } when: "Polling and completing second task in DO While" Tuple polledAndCompletedTask1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'integration.test.worker') then: "Verify that the task was polled and acknowledged and workflow is in running state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1) verifyTaskIteration(polledAndCompletedTask1[0] as Task, 1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DO_WHILE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'integration_task_0' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_1' tasks[5].status == Task.Status.COMPLETED } when: "Polling and completing third task" Tuple polledAndCompletedTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'integration.test.worker') and: "the workflow is evaluated" sweep(workflowInstanceId) and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinId) then: "Verify that the task was polled and acknowledged and workflow is in completed state" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DO_WHILE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_0' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_1' tasks[5].status == Task.Status.COMPLETED } } def "Test workflow with Do While task contains loop over task that use iteration in script expression"() { given: "Number of iterations of the loop is set to 2" def workflowInput = new HashMap() workflowInput['loop'] = 2 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("Do_While_Workflow_Iteration_Fix", 1, "looptest", workflowInput, null) then: "Verify that the workflow has competed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'LAMBDA' tasks[1].status == Task.Status.COMPLETED tasks[1].outputData.get("result") == 0 tasks[2].taskType == 'LAMBDA' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData.get("result") == 1 } } def "Test workflow with Do While task contains set variable task"() { given: "The loop condition is set to use set variable" def workflowInput = new HashMap() workflowInput['value'] = 2 when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("do_while_Set_variable_fix", 1, "looptest", workflowInput, null) then: "Verify that the workflow has competed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SET_VARIABLE' tasks[1].status == Task.Status.COMPLETED tasks[1].inputData.get("value") == "0" } } def "Test workflow with Do While task contains decision task"() { given: "The loop condition is set to use set variable" def workflowInput = new HashMap() def array = new ArrayList() array.add(1); array.add(2); workflowInput['list'] = array when: "A do_while workflow is started" def workflowInstanceId = startWorkflow("DO_While_with_Decision_task", 1, "looptest", workflowInput, null) then: "Verify that the loop over task is waiting for the wait task to get completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[1].taskType == 'INLINE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'WAIT' tasks[3].status == Task.Status.IN_PROGRESS } when: "The wait task is completed" def waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[3] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) then: "Verify that the next iteration is scheduled and workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.IN_PROGRESS tasks[0].iteration == 2 tasks[1].taskType == 'INLINE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'WAIT' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'INLINE' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'INLINE' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'SWITCH' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'WAIT' tasks[7].status == Task.Status.IN_PROGRESS } when: "The wait task is completed" waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[7] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) then: "Verify that the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 9 tasks[0].taskType == 'DO_WHILE' tasks[0].status == Task.Status.COMPLETED tasks[0].iteration == 2 tasks[1].taskType == 'INLINE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'WAIT' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'INLINE' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'INLINE' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'SWITCH' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'WAIT' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'INLINE' tasks[8].status == Task.Status.COMPLETED } } void verifyTaskIteration(Task task, int iteration) { assert task.getReferenceTaskName().endsWith(TaskUtils.getLoopOverTaskRefNameSuffix(task.getIteration())) assert task.iteration == iteration } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/DynamicForkJoinSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared class DynamicForkJoinSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired Join joinTask @Autowired SubWorkflow subWorkflowTask @Shared def DYNAMIC_FORK_JOIN_WF = "DynamicFanInOutTest" def setup() { workflowTestUtil.registerWorkflows('dynamic_fork_join_integration_test.json', 'simple_workflow_3_integration_test.json') } def "Test dynamic fork join success flow"() { when: " a dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_workflow', [:], null) then: "verify that the workflow has been successfully started and the first task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: " the first task is 'integration_task_1' output has a list of dynamic tasks" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def dynamicTasksInput = ['xdt1': ['k1': 'v1'], 'xdt2': ['k2': 'v2']] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': dynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "Poll and complete 'integration_task_2' and 'integration_task_3'" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("dynamicfanouttask_join").taskId def pollAndCompleteTask2Try = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker', ['ok1': 'ov1']) def pollAndCompleteTask3Try = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.worker', ['ok1': 'ov1']) and: "workflow is evaluated by the reconciler" sweep(workflowInstanceId) then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try, ['k1': 'v1']) workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask3Try, ['k2': 'v2']) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that the workflow has progressed and the 'integration_task_2' and 'integration_task_3' are complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.COMPLETED tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[4].outputData['xdt1']['ok1'] == 'ov1' tasks[4].outputData['xdt2']['ok1'] == 'ov1' tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_4'" def pollAndCompleteTask4Try = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4.worker') then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask4Try) and: "verify that the workflow is complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.COMPLETED } } def "Test dynamic fork join failure of dynamic forked task flow"() { setup: "Make sure that the integration_task_2 does not have any retry count" def persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) when: " a dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_workflow', [:], null) then: "verify that the workflow has been successfully started and the first task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: " the first task is 'integration_task_1' output has a list of dynamic tasks" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def dynamicTasksInput = ['xdt1': ['k1': 'v1'], 'xdt2': ['k2': 'v2']] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': dynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "Poll and fail 'integration_task_2'" def pollAndCompleteTask2Try = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.worker', 'it is a failure..') and: "workflow is evaluated by the reconciler" sweep(workflowInstanceId) then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try, ['k1': 'v1']) and: "verify that the workflow is in failed state and 'integration_task_2' has also failed and other tasks are canceled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.CANCELED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.CANCELED tasks[4].referenceTaskName == 'dynamicfanouttask_join' } cleanup: "roll back the change made to integration_task_2 definition" metadataService.updateTaskDef(persistedTask2Definition) } def "Retry a failed dynamic fork join workflow"() { setup: "Make sure that the integration_task_2 does not have any retry count" def persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) when: " a dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_workflow', [:], null) then: "verify that the workflow has been successfully started and the first task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: " the first task is 'integration_task_1' output has a list of dynamic tasks" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def dynamicTasksInput = ['xdt1': ['k1': 'v1'], 'xdt2': ['k2': 'v2']] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': dynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "Poll and fail 'integration_task_2'" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("dynamicfanouttask_join").taskId def pollAndCompleteTask2Try1 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.worker', 'it is a failure..') and: "workflow is evaluated by the reconciler" sweep(workflowInstanceId) then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try1, ['k1': 'v1']) and: "verify that the workflow is in failed state and 'integration_task_2' has also failed and other tasks are canceled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.CANCELED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.CANCELED tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "The workflow is retried" workflowExecutor.retry(workflowInstanceId, false) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.CANCELED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == 'integration_task_3' tasks[6].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_2' and 'integration_task_3'" def pollAndCompleteTask2Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker', ['ok1': 'ov1']) def pollAndCompleteTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.worker', ['ok1': 'ov1']) and: "workflow is evaluated by the reconciler" sweep(workflowInstanceId) then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try2, ['k1': 'v1']) workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask3Try1, ['k2': 'v2']) when: "JOIN task is polled and executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that the workflow has progressed and the 'integration_task_2' and 'integration_task_3' are complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.COMPLETED tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[4].outputData['xdt1']['ok1'] == 'ov1' tasks[4].outputData['xdt2']['ok1'] == 'ov1' tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_3' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'integration_task_4' tasks[7].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_4'" def pollAndCompleteTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4.worker') then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask4Try1) and: "verify that the workflow is complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 8 tasks[7].taskType == 'integration_task_4' tasks[7].status == Task.Status.COMPLETED } cleanup: "roll back the change made to integration_task_2 definition" metadataService.updateTaskDef(persistedTask2Definition) } def "Retry a failed dynamic fork join workflow with forked subworkflow"() { setup: "Make sure that the integration_task_2 does not have any retry count" def persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) when: "the dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_wf_subwf', [:], null) then: "verify that the workflow is started and first task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' } when: "the first task's output has a list of dynamically forked tasks including a subworkflow" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'sub_wf_task' workflowTask2.taskReferenceName = 'xdt1' workflowTask2.workflowTaskType = TaskType.SUB_WORKFLOW SubWorkflowParams subWorkflowParams = new SubWorkflowParams() subWorkflowParams.setName("integration_test_wf3") subWorkflowParams.setVersion(1) workflowTask2.subWorkflowParam = subWorkflowParams WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_10' workflowTask3.taskReferenceName = 'xdt10' def dynamicTasksInput = ['xdt1': ['p1': 'q1', 'p2': 'q2'], 'xdt10': ['k2': 'v2']] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': dynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "the subworkflow is started by issuing a system task call" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("dynamicfanouttask_join").taskId List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) String subworkflowTaskId = polledTaskIds.get(0) asyncSystemTaskExecutor.execute(subWorkflowTask, subworkflowTaskId) then: "verify that the sub workflow task is in a IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "subworkflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.tasks[2].subWorkflowId then: "verify that the sub workflow is RUNNING, and first task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "The 'integration_task_10' is polled and completed" def pollAndCompleteTask10Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_10', 'task10.worker') then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask10Try1) and: "verify that the workflow is updated" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "The task within sub workflow is polled and completed" pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker') then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "the next task in the subworkflow is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "Poll and fail 'integration_task_2'" def pollAndCompleteTask2Try1 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.worker', "failure") and: "the workflow is evaluated" sweep(workflowInstanceId) then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try1) and: "the subworkflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } and: "the workflow is also in FAILED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "The workflow is retried" workflowExecutor.retry(workflowInstanceId, true) then: "verify that the workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.IN_PROGRESS tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].referenceTaskName == 'dynamicfanouttask_join' } and: "the subworkflow is retried and in RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } when: "the workflow is evaluated" sweep(workflowInstanceId) then: "verify that the JOIN is updated" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.IN_PROGRESS !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "Poll and complete 'integration_task_2'" def pollAndCompleteTask2Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try2) and: "the sub workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_3'" def pollAndCompleteTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask3Try1) and: "the sub workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.COMPLETED } when: "the workflow is evaluated" sweep(workflowInstanceId) and: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "the workflow has progressed beyond the join task" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.COMPLETED !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.COMPLETED tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_4'" def pollAndCompleteTask4Try = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4.worker') then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask4Try) and: "verify that the workflow is complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.COMPLETED !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.COMPLETED tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.COMPLETED } cleanup: "roll back the change made to integration_task_2 definition" metadataService.updateTaskDef(persistedTask2Definition) } def "Test dynamic fork join empty output"() { when: " a dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_workflow', [:], null) then: "verify that the workflow has been successfully started and the first task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: " the first task is 'integration_task_1' output has a list of dynamic tasks" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def dynamicTasksInput = ['xdt1': ['k1': 'v1'], 'xdt2': ['k2': 'v2']] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': dynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS tasks[4].referenceTaskName == 'dynamicfanouttask_join' } when: "Poll and complete 'integration_task_2' and 'integration_task_3'" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("dynamicfanouttask_join").taskId def pollAndCompleteTask2Try = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker') def pollAndCompleteTask3Try = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.worker') and: "workflow is evaluated by the reconciler" sweep(workflowInstanceId) then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try, ['k1': 'v1']) workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask3Try, ['k2': 'v2']) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that the workflow has progressed and the 'integration_task_2' and 'integration_task_3' are complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['xdt1', 'xdt2'] tasks[4].status == Task.Status.COMPLETED tasks[4].referenceTaskName == 'dynamicfanouttask_join' tasks[4].outputData.isEmpty() tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.SCHEDULED } when: "Poll and complete 'integration_task_4'" def pollAndCompleteTask4Try = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4.worker') then: "verify that the tasks were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask4Try) and: "verify that the workflow is complete" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[5].taskType == 'integration_task_4' tasks[5].status == Task.Status.COMPLETED } } def "Test dynamic fork join fail when task input is invalid"() { when: "a dynamic fork join workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, 'dynamic_fork_join_workflow', [:], null) then: "verify that the workflow has been successfully started and the first task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: " the first task is 'integration_task_1' output has a list of dynamic tasks" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def invalidDynamicTasksInput = ['xdt1': 'v1', 'xdt2': 'v2'] and: "The 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker', ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': invalidDynamicTasksInput]) then: "verify that the task was completed" workflowTestUtil.verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try) and: "verify that workflow failed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED } } def "Test dynamic fork join return failed workflow when start with invalid input"() { when: "a dynamic fork join workflow is started" WorkflowTask workflowTask2 = new WorkflowTask() workflowTask2.name = 'integration_task_2' workflowTask2.taskReferenceName = 'xdt1' WorkflowTask workflowTask3 = new WorkflowTask() workflowTask3.name = 'integration_task_3' workflowTask3.taskReferenceName = 'xdt2' def invalidDynamicTasksInput = ['xdt1': 'v1', 'xdt2': 'v2'] def workflowInput = ['dynamicTasks': [workflowTask2, workflowTask3], 'dynamicTasksInput': invalidDynamicTasksInput] def dynamicForkJoinTask = new WorkflowTask() dynamicForkJoinTask.name = 'dynamicfanouttask' dynamicForkJoinTask.taskReferenceName = 'dynamicfanouttask' dynamicForkJoinTask.type = 'FORK_JOIN_DYNAMIC' dynamicForkJoinTask.inputParameters = ['dynamicTasks': '${workflow.input.dynamicTasks}', 'dynamicTasksInput': '${workflow.input.dynamicTasksInput}'] dynamicForkJoinTask.dynamicForkTasksParam = 'dynamicTasks' dynamicForkJoinTask.dynamicForkTasksInputParamName = 'dynamicTasksInput' def workflowDef = new WorkflowDef() workflowDef.name = 'DynamicForkJoinStartTest' workflowDef.version = 1 workflowDef.tasks.add(dynamicForkJoinTask) workflowDef.ownerEmail = 'test@harness.com' def startWorkflowInput = new StartWorkflowInput(name: workflowDef.name, version: workflowDef.version, workflowInput: workflowInput, workflowDefinition: workflowDef) def workflowInstanceId = startWorkflowOperation.execute(startWorkflowInput) then: "verify that workflow failed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.isEmpty() } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/EventTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Event import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class EventTaskSpec extends AbstractSpecification { def EVENT_BASED_WORKFLOW = 'test_event_workflow' @Autowired Event eventTask @Autowired QueueDAO queueDAO def setup() { workflowTestUtil.registerWorkflows('event_workflow_integration_test.json') } def "Verify that a event based simple workflow is executed"() { when: "Start a event based workflow" def workflowInstanceId = startWorkflow(EVENT_BASED_WORKFLOW, 1, '', [:], null) then: "Retrieve the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == TaskType.EVENT.name() tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['event_produced'] tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "The integration_task_1 is polled and completed" def polledAndCompletedTry1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task was polled and completed and the workflow is in a complete state" verifyPolledAndAcknowledgedTask(polledAndCompletedTry1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED } } def "Test a workflow with event task that is asyncComplete "() { setup: "Register a workflow definition with event task as asyncComplete" def persistedWorkflowDefinition = metadataService.getWorkflowDef(EVENT_BASED_WORKFLOW, 1) def modifiedWorkflowDefinition = new WorkflowDef() modifiedWorkflowDefinition.name = persistedWorkflowDefinition.name modifiedWorkflowDefinition.version = persistedWorkflowDefinition.version modifiedWorkflowDefinition.tasks = persistedWorkflowDefinition.tasks modifiedWorkflowDefinition.inputParameters = persistedWorkflowDefinition.inputParameters modifiedWorkflowDefinition.outputParameters = persistedWorkflowDefinition.outputParameters modifiedWorkflowDefinition.ownerEmail = persistedWorkflowDefinition.ownerEmail modifiedWorkflowDefinition.tasks[0].asyncComplete = true metadataService.updateWorkflowDef([modifiedWorkflowDefinition]) when: "The event task workflow is started" def workflowInstanceId = startWorkflow(EVENT_BASED_WORKFLOW, 1, '', [:], null) then: "Retrieve the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == TaskType.EVENT.name() tasks[0].status == Task.Status.IN_PROGRESS tasks[0].outputData['event_produced'] } when: "The event task is updated async using the API" Task task = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName('wait0') TaskResult taskResult = new TaskResult(task) taskResult.setStatus(TaskResult.Status.COMPLETED) workflowExecutor.updateTask(taskResult) then: "Ensure that event task is COMPLETED and workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == TaskType.EVENT.name() tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['event_produced'] tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "The integration_task_1 is polled and completed" def polledAndCompletedTry1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task was polled and completed and the workflow is in a complete state" verifyPolledAndAcknowledgedTask(polledAndCompletedTry1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == TaskType.EVENT.name() tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['event_produced'] tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED } cleanup: "Ensure that the changes to the workflow def are reverted" metadataService.updateWorkflowDef([persistedWorkflowDefinition]) } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/ExclusiveJoinSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class ExclusiveJoinSpec extends AbstractSpecification { @Shared def EXCLUSIVE_JOIN_WF = "ExclusiveJoinTestWorkflow" def setup() { workflowTestUtil.registerWorkflows('exclusive_join_integration_test.json') } def setTaskResult(String workflowInstanceId, String taskId, TaskResult.Status status, Map output) { TaskResult taskResult = new TaskResult(); taskResult.setTaskId(taskId) taskResult.setWorkflowInstanceId(workflowInstanceId) taskResult.setStatus(status) taskResult.setOutputData(output) return taskResult } def "Test that the default decision is run"() { given: "The input parameter required to make decision_1 is null to ensure that the default decision is run" def input = ["decision_1": "null"] when: "An exclusive join workflow is started with then workflow input" def workflowInstanceId = startWorkflow(EXCLUSIVE_JOIN_WF, 1, 'exclusive_join_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1' + '.integration.worker', ["taskReferenceName": "task1"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'EXCLUSIVE_JOIN' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['taskReferenceName'] == 'task1' } } def "Test when the one decision is true and the other is decision null"() { given: "The input parameter required to make decision_1 true and decision_2 null" def input = ["decision_1": "true", "decision_2": "null"] when: "An exclusive join workflow is started with then workflow input" def workflowInstanceId = startWorkflow(EXCLUSIVE_JOIN_WF, 1, 'exclusive_join_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1' + '.integration.worker', ["taskReferenceName": "task1"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2' + '.integration.worker', ["taskReferenceName": "task2"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'EXCLUSIVE_JOIN' tasks[4].status == Task.Status.COMPLETED tasks[4].outputData['taskReferenceName'] == 'task2' } } def "Test when both the decisions, decision_1 and decision_2 are true"() { given: "The input parameters to ensure that both the decisions are true" def input = ["decision_1": "true", "decision_2": "true"] when: "An exclusive join workflow is started with then workflow input" def workflowInstanceId = startWorkflow(EXCLUSIVE_JOIN_WF, 1, 'exclusive_join_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1' + '.integration.worker', ["taskReferenceName": "task1"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2' + '.integration.worker', ["taskReferenceName": "task2"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_3' tasks[4].status == Task.Status.SCHEDULED } when: "the task 'integration_task_3' is polled and completed" def polledAndCompletedTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3' + '.integration.worker', ["taskReferenceName": "task3"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask3Try1) and: "verify that the 'integration_task_3' is COMPLETED and the workflow has COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_3' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'EXCLUSIVE_JOIN' tasks[5].status == Task.Status.COMPLETED tasks[5].outputData['taskReferenceName'] == 'task3' } } def "Test when decision_1 is false and decision_3 is default"() { given: "The input parameter required to make decision_1 false and decision_3 default" def input = ["decision_1": "false", "decision_3": "null"] when: "An exclusive join workflow is started with then workflow input" def workflowInstanceId = startWorkflow(EXCLUSIVE_JOIN_WF, 1, 'exclusive_join_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1' + '.integration.worker', ["taskReferenceName": "task1"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_4' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_4' is polled and completed" def polledAndCompletedTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4' + '.integration.worker', ["taskReferenceName": "task4"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask4Try1) and: "verify that the 'integration_task_4' is COMPLETED and the workflow has COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_4' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'EXCLUSIVE_JOIN' tasks[4].status == Task.Status.COMPLETED tasks[4].outputData['taskReferenceName'] == 'task4' } } def "Test when decision_1 is false and decision_3 is true"() { given: "The input parameter required to make decision_1 false and decision_3 true" def input = ["decision_1": "false", "decision_3": "true"] when: "An exclusive join workflow is started with then workflow input" def workflowInstanceId = startWorkflow(EXCLUSIVE_JOIN_WF, 1, 'exclusive_join_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1' + '.integration.worker', ["taskReferenceName": "task1"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_4' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_4' is polled and completed" def polledAndCompletedTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4' + '.integration.worker', ["taskReferenceName": "task4"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask4Try1) and: "verify that the 'integration_task_4' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_4' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_5' tasks[4].status == Task.Status.SCHEDULED } when: "the task 'integration_task_5' is polled and completed" def polledAndCompletedTask5Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_5', 'task5' + '.integration.worker', ["taskReferenceName": "task5"]) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask5Try1) and: "verify that the 'integration_task_4' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_4' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_5' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'EXCLUSIVE_JOIN' tasks[5].status == Task.Status.COMPLETED tasks[5].outputData['taskReferenceName'] == 'task5' } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/ExternalPayloadStorageSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.test.base.AbstractSpecification import com.netflix.conductor.test.utils.MockExternalPayloadStorage import com.netflix.conductor.test.utils.UserTask import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPayload import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedLargePayloadTask import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class ExternalPayloadStorageSpec extends AbstractSpecification { @Shared def LINEAR_WORKFLOW_T1_T2 = 'integration_test_wf' @Shared def CONDITIONAL_SYSTEM_TASK_WORKFLOW = 'ConditionalSystemWorkflow' @Shared def FORK_JOIN_WF = 'FanInOutTest' @Shared def DYNAMIC_FORK_JOIN_WF = "DynamicFanInOutTest" @Shared def WORKFLOW_WITH_INLINE_SUB_WF = 'WorkflowWithInlineSubWorkflow' @Shared def WORKFLOW_WITH_DECISION_AND_TERMINATE = 'ConditionalTerminateWorkflow' @Shared def WORKFLOW_WITH_SYNCHRONOUS_SYSTEM_TASK = 'workflow_with_synchronous_system_task' @Autowired UserTask userTask @Autowired SubWorkflow subWorkflowTask @Autowired Join joinTask @Autowired MockExternalPayloadStorage mockExternalPayloadStorage def setup() { workflowTestUtil.registerWorkflows('simple_workflow_1_integration_test.json', 'conditional_system_task_workflow_integration_test.json', 'fork_join_integration_test.json', 'simple_workflow_with_sub_workflow_inline_def_integration_test.json', 'decision_and_terminate_integration_test.json', 'workflow_with_synchronous_system_task.json', 'dynamic_fork_join_integration_test.json' ) } def "Test simple workflow using external payload storage"() { given: "An existing simple workflow definition" metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) and: "input required to start large payload workflow" def correlationId = 'wf_external_storage' String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_2' with external payload storage" pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask("integration_task_2", "task2.integration.worker", "") then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) then: "verify that the 'integration_task_2' is complete and the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 output.isEmpty() tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } } def "Test workflow with synchronous system task using external payload storage"() { given: "An existing workflow definition with sync system task followed by a simple task" metadataService.getWorkflowDef(WORKFLOW_WITH_SYNCHRONOUS_SYSTEM_TASK, 1) and: "input required to start large payload workflow" def correlationId = 'wf_external_storage' String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_SYNCHRONOUS_SYSTEM_TASK, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'JSON_JQ_TRANSFORM' tasks[1].status == Task.Status.COMPLETED tasks[1].outputData['result'] == 104 // output of .tp2.TEST_SAMPLE | length expression from output.json. On assertion failure, check workflow definition and output.json } } def "Test conditional workflow with system task using external payload storage"() { given: "An existing workflow definition" metadataService.getWorkflowDef(CONDITIONAL_SYSTEM_TASK_WORKFLOW, 1) and: "input required to start large payload workflow" String workflowInputPath = uploadInitialWorkflowInput() def correlationId = "conditional_system_external_storage" when: "the workflow is started" def workflowInstanceId = startWorkflow(CONDITIONAL_SYSTEM_TASK_WORKFLOW, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == "DECISION" tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == "USER_TASK" tasks[2].status == Task.Status.SCHEDULED tasks[2].inputData.isEmpty() } when: "the system task 'USER_TASK' is started by issuing a system task call" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def taskId = workflow.getTaskByRefName('user_task').taskId asyncSystemTaskExecutor.execute(userTask, taskId) then: "verify that the user task is in a COMPLETED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == "DECISION" tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == "USER_TASK" tasks[2].status == Task.Status.COMPLETED tasks[2].inputData.isEmpty() tasks[2].outputData.get("size") == 104 tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.SCHEDULED } when: "poll and complete and 'integration_task_3'" def pollAndCompleteTask3 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.integration.worker', ['op': 'success_task3']) then: "verify that the 'integration_task_3' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask3) then: "verify that the 'integration_task_3' is complete and the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 output.isEmpty() tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == "DECISION" tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == "USER_TASK" tasks[2].status == Task.Status.COMPLETED tasks[2].inputData.isEmpty() tasks[2].outputData.get("size") == 104 tasks[3].taskType == 'integration_task_3' tasks[3].status == Task.Status.COMPLETED } } def "Test fork join workflow using external payload storage"() { given: "An existing fork join workflow definition" metadataService.getWorkflowDef(FORK_JOIN_WF, 1) and: "input required to start large payload workflow" def correlationId = 'fork_join_external_storage' String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(FORK_JOIN_WF, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' } when: "the first task of the left fork is polled and completed" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("fanouttask_join").taskId def polledAndAckTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndAckTask) and: "task is completed and the next task in the fork is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "the first task of the right fork is polled and completed with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def polledAndAckLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_2', 'task2.integration.worker', taskOutputPath) then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(polledAndAckLargePayloadTask) and: "task is completed and the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "the second task of the left fork is polled and completed with external payload storage" polledAndAckLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_3', 'task3.integration.worker', taskOutputPath) and: "the workflow is evaluated" sweep(workflowInstanceId) then: "verify that the 'integration_task_3' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(polledAndAckLargePayloadTask) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "task is completed and the next task after join in scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].outputData.isEmpty() tasks[3].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[3].outputData.isEmpty() tasks[4].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_3' tasks[4].outputData.isEmpty() tasks[5].status == Task.Status.SCHEDULED tasks[5].taskType == 'integration_task_4' } when: "the task 'integration_task_4' is polled and completed" polledAndAckTask = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task4.integration.worker') then: "verify that the 'integration_task_4' was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndAckTask) and: "task is completed and the workflow is in completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].outputData.isEmpty() tasks[3].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[3].outputData.isEmpty() tasks[4].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_3' tasks[4].outputData.isEmpty() tasks[5].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_4' } } def "Test workflow with subworkflow using external payload storage"() { given: "An existing workflow definition" metadataService.getWorkflowDef(WORKFLOW_WITH_INLINE_SUB_WF, 1) and: "input required to start large payload workflow" String workflowInputPath = uploadInitialWorkflowInput() def correlationId = "workflow_with_inline_sub_wf_external_storage" when: "the workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_INLINE_SUB_WF, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.SCHEDULED tasks[1].inputData.isEmpty() } when: "the subworkflow is started by issuing a system task call" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = workflow.getTaskByRefName('swt').taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) then: "verify that the sub workflow task is in a IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.IN_PROGRESS tasks[1].inputData.isEmpty() } when: "sub workflow is retrieved" workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowInstanceId = workflow.getTaskByRefName('swt').subWorkflowId then: "verify that the sub workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 input.isEmpty() tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'integration_task_3' } when: "poll and complete the 'integration_task_3' with external payload storage" pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_3', 'task3.integration.worker', taskOutputPath) then: "verify that the 'integration_task_3' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the sub workflow is completed" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 input.isEmpty() tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_3' tasks[0].outputData.isEmpty() output.isEmpty() } and: "the subworkflow task is completed and the workflow is in running state" sweep(workflowInstanceId) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.COMPLETED tasks[1].inputData.isEmpty() tasks[1].outputData.isEmpty() tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].inputData.isEmpty() } when: "poll and complete the 'integration_task_2' with external payload storage" pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_2', 'task2.integration.worker', taskOutputPath) then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the task is completed and the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 output.isEmpty() tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.COMPLETED tasks[1].inputData.isEmpty() tasks[1].outputData.isEmpty() tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].inputData.isEmpty() tasks[2].outputData.isEmpty() } } def "Test retry workflow using external payload storage"() { setup: "Modify the task definition" def persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 2, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) modifiedTask2Definition.setRetryDelaySeconds(0) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing simple workflow definition" metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) and: "input required to start large payload workflow" def correlationId = 'retry_wf_external_storage' String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and fail the 'integration_task_2'" def pollAndFailTask2Try1 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndFailTask2Try1) and: "verify that task is retried and workflow is still running" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].inputData.isEmpty() tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].inputData.isEmpty() } when: "poll and complete the retried 'integration_task_2'" def pollAndCompleteTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'success_task2']) then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask2) and: "verify that the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 output.isEmpty() tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].inputData.isEmpty() tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].inputData.isEmpty() } cleanup: metadataService.updateTaskDef(persistedTask2Definition) } def "Test workflow with terminate in decision branch using external payload storage"() { given: "An existing workflow definition" metadataService.getWorkflowDef(WORKFLOW_WITH_DECISION_AND_TERMINATE, 1) and: "input required to start large payload workflow" String workflowInputPath = uploadInitialWorkflowInput() def correlationId = "decision_terminate_external_storage" when: "the workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_DECISION_AND_TERMINATE, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].seq == 1 } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = uploadLargeTaskOutput() def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has FAILED due to terminate task" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 3 output.isEmpty() reasonForIncompletion.contains('Workflow is FAILED by TERMINATE task') tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[0].seq == 1 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[1].seq == 2 tasks[2].taskType == 'TERMINATE' tasks[2].status == Task.Status.COMPLETED tasks[2].inputData.isEmpty() tasks[2].seq == 3 tasks[2].outputData.isEmpty() } } def "Test dynamic fork join workflow with subworkflow using external payload storage"() { given: "An existing dynamic fork join workflow definition" metadataService.getWorkflowDef(DYNAMIC_FORK_JOIN_WF, 1) and: "input required to start large payload workflow" def correlationId = "dynamic_fork_join_subworkflow_external_storage" String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(DYNAMIC_FORK_JOIN_WF, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING input.isEmpty() tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_1' with external payload storage" String taskOutputPath = "${UUID.randomUUID()}.json" mockExternalPayloadStorage.upload(taskOutputPath, mockExternalPayloadStorage.curateDynamicForkLargePayload()) def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_1', 'task1.integration.worker', taskOutputPath) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) and: "verify that workflow has progressed further ahead and new dynamic tasks have been scheduled with externalized payloads" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) with(workflow) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED !tasks[0].outputData tasks[1].taskType == 'FORK' !tasks[1].inputData tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SUB_WORKFLOW' !tasks[2].inputData tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].referenceTaskName == 'dynamicfanouttask_join' } } def "Test update task output multiple times using external payload storage"() { given: "An existing simple workflow definition" metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) when: "the workflow is started" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, 'multi_task_update_external_storage', new HashMap(), null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and update 'integration_task_1' with external payload storage output" String taskOutputPath = uploadLargeTaskOutput() workflowTestUtil.pollAndUpdateTask('integration_task_1', 'task1.integration.worker', taskOutputPath, null, 1) then: "verify that 'integration_task1's output is updated correctly" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].outputData.isEmpty() tasks[0].externalOutputPayloadStoragePath == taskOutputPath } when: "poll and update 'integration_task_1' with no additional output" workflowTestUtil.pollAndUpdateTask('integration_task_1', 'task1.integration.worker', null, null, 1) then: "verify that 'integration_task1's output is updated correctly" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].outputData.isEmpty() // no duplicate upload tasks[0].externalOutputPayloadStoragePath == taskOutputPath } when: "poll and complete 'integration_task_1' with additional output" Map output = ['k1': 'v1', 'k2': 'v2'] workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', output, 1) then: "verify that 'integration_task1 is complete and output is updated correctly" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() // upload again with additional output tasks[0].externalOutputPayloadStoragePath != taskOutputPath verifyPayload(output, mockExternalPayloadStorage.downloadPayload(tasks[0].externalOutputPayloadStoragePath)) tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and update 'integration_task_2' with output" Map output1 = ['k1': 'v1', 'k2': 'v2'] workflowTestUtil.pollAndUpdateTask('integration_task_2', 'task1.integration.worker', null, output1, 1) then: "verify that 'integration_task2's output is updated correctly" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED tasks[1].externalOutputPayloadStoragePath == null verifyPayload(output1, tasks[1].outputData) } when: "poll and complete 'integration_task_2' with additional output" Map output2 = ['k3': 'v3', 'k4': 'v4'] workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', output2, 1) then: "verify that 'integration_task2 is complete and output is updated correctly" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.isEmpty() tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED tasks[1].externalOutputPayloadStoragePath == null output1.putAll(output2) verifyPayload(output1, tasks[1].outputData) } } def "Test fork join workflow exceed external storage limit should fail the task and workflow"() { given: "An existing fork join workflow definition" metadataService.getWorkflowDef(FORK_JOIN_WF, 1) and: "input required to start large payload workflow" def correlationId = 'fork_join_external_storage' String workflowInputPath = uploadInitialWorkflowInput() when: "the workflow is started" def workflowInstanceId = startWorkflow(FORK_JOIN_WF, 1, correlationId, null, workflowInputPath) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' } when: "the first task of the left fork is polled and completed" def polledAndAckTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("fanouttask_join").taskId then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndAckTask) and: "task is completed and the next task in the fork is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "the first task of the right fork is polled and completed with external payload storage" String taskOutputPath = "${UUID.randomUUID()}.json" mockExternalPayloadStorage.upload(taskOutputPath, mockExternalPayloadStorage.createLargePayload(500)) def polledAndAckLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_2', 'task2.integration.worker', taskOutputPath) then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(polledAndAckLargePayloadTask) and: "task is completed and the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "the second task of the left fork is polled and completed with external payload storage" taskOutputPath = "${UUID.randomUUID()}.json" mockExternalPayloadStorage.upload(taskOutputPath, mockExternalPayloadStorage.createLargePayload(500)) polledAndAckLargePayloadTask = workflowTestUtil.pollAndCompleteLargePayloadTask('integration_task_3', 'task3.integration.worker', taskOutputPath) then: "verify that the 'integration_task_3' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(polledAndAckLargePayloadTask) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "task is completed and the join task is failed because of exceeding size limit" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].outputData.isEmpty() tasks[3].status == Task.Status.FAILED_WITH_TERMINAL_ERROR tasks[3].taskType == 'JOIN' tasks[3].outputData.isEmpty() !tasks[3].getExternalOutputPayloadStoragePath() } } private String uploadLargeTaskOutput() { String taskOutputPath = "${UUID.randomUUID()}.json" mockExternalPayloadStorage.upload(taskOutputPath, mockExternalPayloadStorage.readOutputDotJson(), 0) return taskOutputPath } private String uploadInitialWorkflowInput() { String workflowInputPath = "${UUID.randomUUID()}.json" mockExternalPayloadStorage.upload(workflowInputPath, ['param1': 'p1 value', 'param2': 'p2 value', 'case': 'two']) return workflowInputPath } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/FailureWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared class FailureWorkflowSpec extends AbstractSpecification { @Shared def WORKFLOW_WITH_TERMINATE_TASK_FAILED = 'test_terminate_task_failed_wf' @Shared def PARENT_WORKFLOW_WITH_FAILURE_TASK = 'test_task_failed_parent_wf' @Autowired SubWorkflow subWorkflowTask def setup() { workflowTestUtil.registerWorkflows( 'failure_workflow_for_terminate_task_workflow.json', 'terminate_task_failed_workflow_integration.json', 'test_task_failed_parent_workflow.json', 'test_task_failed_sub_workflow.json' ) } def "Test workflow with a task that failed"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the failed task" def testId = 'testId' def workflowInstanceId = startWorkflow(WORKFLOW_WITH_TERMINATE_TASK_FAILED, 1, testId, workflowInput, null) then: "Verify that the workflow has failed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 reasonForIncompletion == "Early exit in terminate" tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'TERMINATE' tasks[1].seq == 2 output def failedWorkflowId = output['conductor.failure_workflow'] as String def workflowCorrelationId = correlationId def workflowFailureTaskId = tasks[1].taskId with(workflowExecutionService.getExecutionStatus(failedWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED correlationId == workflowCorrelationId input['workflowId'] == workflowInstanceId input['failureTaskId'] == workflowFailureTaskId tasks.size() == 1 tasks[0].taskType == 'LAMBDA' input['failedWorkflow'] != null } } } def "Test workflow with a task failed in subworkflow"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the subworkflow task" def workflowInstanceId = startWorkflow(PARENT_WORKFLOW_WITH_FAILURE_TASK, 1, '', workflowInput, null) then: "verify that the workflow has started and the tasks are as expected" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].referenceTaskName == 'lambdaTask1' tasks[0].seq == 1 tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].seq == 2 } when: "subworkflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = workflow.getTaskByRefName("test_task_failed_sub_wf").getTaskId() asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.getTaskByRefName("test_task_failed_sub_wf").subWorkflowId then: "verify that the sub workflow has failed" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 reasonForIncompletion.contains('Workflow is FAILED by TERMINATE task') tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'TERMINATE' tasks[1].seq == 2 } then: "Verify that the workflow has failed and correct inputs passed into the failure workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].referenceTaskName == 'lambdaTask1' tasks[0].seq == 1 tasks[1].status == Task.Status.FAILED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].seq == 2 def failedWorkflowId = output['conductor.failure_workflow'] as String def workflowCorrelationId = correlationId def workflowFailureTaskId = tasks[1].taskId with(workflowExecutionService.getExecutionStatus(failedWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED correlationId == workflowCorrelationId input['workflowId'] == workflowInstanceId input['failureTaskId'] == workflowFailureTaskId tasks.size() == 1 tasks[0].taskType == 'LAMBDA' input['failedWorkflow'] != null } } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/ForkJoinSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared class ForkJoinSpec extends AbstractSpecification { @Autowired Join joinTask @Shared def FORK_JOIN_WF = 'FanInOutTest' @Shared def FORK_JOIN_NESTED_WF = 'FanInOutNestedTest' @Shared def FORK_JOIN_NESTED_SUB_WF = 'FanInOutNestedSubWorkflowTest' @Shared def WORKFLOW_FORK_JOIN_OPTIONAL_SW = "integration_test_fork_join_optional_sw" @Shared def FORK_JOIN_SUB_WORKFLOW = 'integration_test_fork_join_sw' @Autowired SubWorkflow subWorkflowTask def setup() { workflowTestUtil.registerWorkflows('fork_join_integration_test.json', 'fork_join_with_no_task_retry_integration_test.json', 'nested_fork_join_integration_test.json', 'simple_workflow_1_integration_test.json', 'nested_fork_join_with_sub_workflow_integration_test.json', 'simple_one_task_sub_workflow_integration_test.json', 'fork_join_with_optional_sub_workflow_forks_integration_test.json', 'fork_join_sub_workflow.json' ) } /** * start * | * fork * / \ * task1 task2 * | / * task3 / * \ / * \ / * join * | * task4 * | * End */ def "Test a simple workflow with fork join success flow"() { when: "A fork join workflow is started" def workflowInstanceId = startWorkflow(FORK_JOIN_WF, 1, 'fanoutTest', [:], null) then: "verify that the workflow has started and the starting nodes of the each fork are in scheduled state" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' } when: "The first task of the fork is polled and completed" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("fanouttask_join").getTaskId() def polledAndAckTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker') then: "verify that the 'integration_task_1' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask1Try1) and: "The workflow has been updated and has all the required tasks in the right status to move forward" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "The 'integration_task_3' is polled and completed" def polledAndAckTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task1.worker') then: "verify that the 'integration_task_3' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask3Try1) and: "The workflow has been updated with the task status and task list" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_3' } when: "The other node of the fork is completed by completing 'integration_task_2'" def polledAndAckTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.worker') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the 'integration_task_2' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask2Try1) when: "JOIN task executed by the async executor" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "The workflow has been updated with the task status and task list" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[5].status == Task.Status.SCHEDULED tasks[5].taskType == 'integration_task_4' } when: "The last task of the workflow is then polled and completed integration_task_4'" def polledAndAckTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task1.worker') then: "verify that the 'integration_task_4' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask4Try1) and: "Then verify that the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 6 tasks[5].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_4' } } def "Test a simple workflow with fork join failure flow"() { setup: "Ensure that 'integration_task_2' has a retry count of 0" def persistedIntegrationTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedIntegrationTask2Definition = new TaskDef(persistedIntegrationTask2Definition.name, persistedIntegrationTask2Definition.description, persistedIntegrationTask2Definition.ownerEmail, 0, 0, persistedIntegrationTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedIntegrationTask2Definition) when: "A fork join workflow is started" def workflowInstanceId = startWorkflow(FORK_JOIN_WF, 1, 'fanoutTest', [:], null) then: "verify that the workflow has started and the starting nodes of the each fork are in scheduled state" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' } when: "The first task of the fork is polled and completed" def polledAndAckTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker') then: "verify that the 'integration_task_1' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask1Try1) and: "The workflow has been updated and has all the required tasks in the right status to move forward" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_3' } when: "The other node of the fork is completed by completing 'integration_task_2'" def polledAndAckTask2Try1 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task1.worker', 'Failed....') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the 'integration_task_2' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask2Try1) and: "the workflow is in the failed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[2].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[3].status == Task.Status.CANCELED tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].taskType == 'integration_task_3' } cleanup: "Restore the task definitions that were modified as part of this feature testing" metadataService.updateTaskDef(persistedIntegrationTask2Definition) } def "Test retrying a failed fork join workflow"() { when: "A fork join workflow is started" def workflowInstanceId = startWorkflow(FORK_JOIN_WF + '_2', 1, 'fanoutTest', [:], null) then: "verify that the workflow has started and the starting nodes of the each fork are in scheduled state" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'integration_task_0_RT_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_0_RT_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' } when: "The first task of the fork is polled and completed" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("fanouttask_join").getTaskId() def polledAndAckTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_0_RT_1', 'task1.worker') then: "verify that the 'integration_task_0_RT_1' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask1Try1) and: "The workflow has been updated and has all the required tasks in the right status to move forward" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0_RT_1' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_0_RT_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_0_RT_3' } when: "The other node of the fork is completed by completing 'integration_task_0_RT_2'" def polledAndAckTask2Try1 = workflowTestUtil.pollAndFailTask('integration_task_0_RT_2', 'task1.worker', 'Failed....') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the 'integration_task_0_RT_1' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask2Try1) and: "the workflow is in the failed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 5 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0_RT_1' tasks[2].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0_RT_2' tasks[3].status == Task.Status.CANCELED tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].taskType == 'integration_task_0_RT_3' } when: "The workflow is retried" workflowExecutor.retry(workflowInstanceId, false) then: "verify that all the workflow is retried and new tasks are added in place of the failed tasks" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0_RT_1' tasks[2].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0_RT_2' tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].taskType == 'integration_task_0_RT_3' tasks[5].status == Task.Status.SCHEDULED tasks[5].taskType == 'integration_task_0_RT_2' tasks[6].status == Task.Status.SCHEDULED tasks[6].taskType == 'integration_task_0_RT_3' } when: "The 'integration_task_0_RT_3' is polled and completed" def polledAndAckTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_0_RT_3', 'task1.worker') then: "verify that the 'integration_task_3' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask3Try1) when: "The other node of the fork is completed by completing 'integration_task_0_RT_2'" def polledAndAckTask2Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_0_RT_2', 'task1.worker') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the 'integration_task_2' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask2Try2) when: "JOIN task is polled and executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) and: "The last task of the workflow is then polled and completed integration_task_0_RT_4'" def polledAndAckTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_0_RT_4', 'task1.worker') then: "verify that the 'integration_task_0_RT_4' was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask4Try1) then: "Then verify that the workflow is completed and the task list of execution is as expected" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 8 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_0_RT_1' tasks[2].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_0_RT_2' tasks[3].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[4].status == Task.Status.CANCELED tasks[4].taskType == 'integration_task_0_RT_3' tasks[5].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_0_RT_2' tasks[6].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_0_RT_3' tasks[7].status == Task.Status.COMPLETED tasks[7].taskType == 'integration_task_0_RT_4' } } def "Test nested fork join workflow success flow"() { given: "Input for the nested fork join workflow" Map input = new HashMap() input["case"] = "a" when: "A nested workflow is started with the input" def workflowInstanceId = startWorkflow(FORK_JOIN_NESTED_WF, 1, 'fork_join_nested_test', input, null) then: "verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks.findAll { it.referenceTaskName in ['t11', 't12', 't13', 'fork1', 'fork2'] }.size() == 5 tasks.findAll { it.referenceTaskName in ['t1', 't2', 't16'] }.size() == 0 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_11' tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_12' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_13' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS tasks[6].inputData['joinOn'] == ['t11', 'join2'] } when: "Poll and Complete tasks: 'integration_task_11', 'integration_task_12' and 'integration_task_13'" def outerJoinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join1").getTaskId() def innerJoinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join2").getTaskId() def polledAndAckTask11Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_11', 'task11.worker') def polledAndAckTask12Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_12', 'task12.worker') def polledAndAckTask13Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_13', 'task13.worker') then: "verify that tasks 'integration_task_11', 'integration_task_12' and 'integration_task_13' were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask11Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask12Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask13Try1) and: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 10 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_11' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_12' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_13' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS tasks[6].inputData['joinOn'] == ['t11', 'join2'] tasks[7].taskType == 'integration_task_14' tasks[7].status == Task.Status.SCHEDULED tasks[8].taskType == 'DECISION' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'integration_task_16' tasks[9].status == Task.Status.SCHEDULED } when: "Poll and Complete tasks: 'integration_task_16' and 'integration_task_14'" def polledAndAckTask16Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_16', 'task16.worker') def polledAndAckTask14Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_14', 'task14.worker') then: "verify that tasks 'integration_task_16' and 'integration_task_14'were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask16Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask14Try1) and: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 11 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS tasks[6].inputData['joinOn'] == ['t11', 'join2'] tasks[7].taskType == 'integration_task_14' tasks[7].status == Task.Status.COMPLETED tasks[8].taskType == 'DECISION' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'integration_task_16' tasks[9].status == Task.Status.COMPLETED tasks[10].taskType == 'integration_task_19' tasks[10].status == Task.Status.SCHEDULED } when: "Poll and Complete tasks: 'integration_task_19'" def polledAndAckTask19Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_19', 'task19.worker') then: "verify that tasks 'integration_task_19' polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask19Try1) and: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 12 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.IN_PROGRESS tasks[6].inputData['joinOn'] == ['t11', 'join2'] tasks[10].taskType == 'integration_task_19' tasks[10].status == Task.Status.COMPLETED tasks[11].taskType == 'integration_task_20' tasks[11].status == Task.Status.SCHEDULED } when: "Poll and Complete tasks: 'integration_task_20'" def polledAndAckTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task20.worker') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that task 'integration_task_20' is polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask20Try1) when: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 13 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'JOIN' tasks[6].status == Task.Status.COMPLETED tasks[6].inputData['joinOn'] == ['t11', 'join2'] tasks[11].taskType == 'integration_task_20' tasks[11].status == Task.Status.COMPLETED tasks[12].taskType == 'integration_task_15' tasks[12].status == Task.Status.SCHEDULED } when: "Poll and Complete tasks: 'integration_task_15'" def polledAndAckTask15Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_15', 'task15.worker') then: "verify that tasks 'integration_task_15' polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask15Try1) and: "verify that the workflow is in a complete state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 13 tasks[12].taskType == 'integration_task_15' tasks[12].status == Task.Status.COMPLETED } } def "Test nested workflow which contains a sub workflow task"() { given: "Input for the nested fork join workflow" Map input = new HashMap() input["case"] = "a" when: "A nested workflow is started with the input" def workflowInstanceId = startWorkflow(FORK_JOIN_NESTED_SUB_WF, 1, 'fork_join_nested_test', input, null) then: "The workflow is in the running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks.findAll { it.referenceTaskName in ['t11', 't12', 't13', 'fork1', 'fork2', 'sw1'] }.size() == 6 tasks.findAll { it.referenceTaskName in ['t1', 't2', 't16'] }.size() == 0 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_11' tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_12' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'integration_task_13' tasks[4].status == Task.Status.SCHEDULED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.SCHEDULED tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.IN_PROGRESS tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] } when: "Poll and Complete tasks: 'integration_task_11', 'integration_task_12' and 'integration_task_13'" def outerJoinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join1").getTaskId() def innerJoinId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("join2").getTaskId() def polledAndAckTask11Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_11', 'task11.worker') def polledAndAckTask12Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_12', 'task12.worker') def polledAndAckTask13Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_13', 'task13.worker') workflowExecutionService.getExecutionStatus(workflowInstanceId, true) then: "verify that tasks 'integration_task_11', 'integration_task_12' and 'integration_task_13' were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask11Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask12Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask13Try1) and: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 11 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_11' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'FORK' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_12' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'integration_task_13' tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.SCHEDULED tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.IN_PROGRESS tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] tasks[8].taskType == 'integration_task_14' tasks[8].status == Task.Status.SCHEDULED tasks[9].taskType == 'DECISION' tasks[9].status == Task.Status.COMPLETED tasks[10].taskType == 'integration_task_16' tasks[10].status == Task.Status.SCHEDULED } when: "Poll and Complete tasks: 'integration_task_16' and 'integration_task_14'" def polledAndAckTask16Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_16', 'task16.worker') def polledAndAckTask14Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_14', 'task14.worker') and: "Get the sub workflow id associated with the SubWorkflow Task sw1 and start the system task" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = workflow.getTaskByRefName("sw1").getTaskId() asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) def updatedWorkflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowInstanceId = updatedWorkflow.getTaskByRefName('sw1').subWorkflowId then: "verify that tasks 'integration_task_16' and 'integration_task_14'were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask16Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask14Try1) with(updatedWorkflow) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 12 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.IN_PROGRESS tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.IN_PROGRESS tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] tasks[8].taskType == 'integration_task_14' tasks[8].status == Task.Status.COMPLETED tasks[9].taskType == 'DECISION' tasks[9].status == Task.Status.COMPLETED tasks[10].taskType == 'integration_task_16' tasks[10].status == Task.Status.COMPLETED tasks[11].taskType == 'integration_task_19' tasks[11].status == Task.Status.SCHEDULED } and: "verify that the simple Sub Workflow is in running state and the first task related to it is scheduled" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Poll and complete all the tasks associated with the sub workflow" def polledAndAckTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.worker') def polledAndAckTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker') then: "verify that tasks 'integration_task_1' and 'integration_task_2'were polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask1Try1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask2Try1) and: "verify that the simple Sub Workflow is in a COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } and: " verify that the sub workflow task is completed and other preceding tasks are added to the workflow task list" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 12 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[6].taskType == 'SUB_WORKFLOW' tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.IN_PROGRESS tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] tasks[11].taskType == 'integration_task_19' tasks[11].status == Task.Status.SCHEDULED } when: "Also the poll and complete the 'integration_task_19'" def polledAndAckTask19Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_19', 'task19.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask19Try1) and: "verify that the integration_task_19 is completed and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 13 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.IN_PROGRESS tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.IN_PROGRESS tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] tasks[11].taskType == 'integration_task_19' tasks[11].status == Task.Status.COMPLETED tasks[12].taskType == 'integration_task_20' tasks[12].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_20'" def polledAndAckTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task20.worker') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask20Try1) when: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify that the integration_task_20 is completed and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 14 tasks[5].taskType == 'JOIN' tasks[5].status == Task.Status.COMPLETED tasks[5].inputData['joinOn'] == ['t14', 't20'] tasks[7].taskType == 'JOIN' tasks[7].status == Task.Status.COMPLETED tasks[7].inputData['joinOn'] == ['t11', 'join2', 'sw1'] tasks[12].taskType == 'integration_task_20' tasks[12].status == Task.Status.COMPLETED tasks[13].taskType == 'integration_task_15' tasks[13].status == Task.Status.SCHEDULED } when: "poll and complete the 'integration_task_15'" def polledAndAckTask15Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_15', 'task15.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckTask15Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 14 tasks[13].taskType == 'integration_task_15' tasks[13].status == Task.Status.COMPLETED } } def "Test fork join with sub workflows containing optional tasks"() { given: "A input to the workflow that has forks of sub workflows with an optional task" Map workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "A workflow that has forks of sub workflows with an optional task is started" def workflowInstanceId = startWorkflow(WORKFLOW_FORK_JOIN_OPTIONAL_SW, 1, '', workflowInput, null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.IN_PROGRESS } when: "both the sub workflows are started by issuing a system task call" def workflowWithScheduledSubWorkflows = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId1 = workflowWithScheduledSubWorkflows.getTaskByRefName('st1').taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId1) def subWorkflowTaskId2 = workflowWithScheduledSubWorkflows.getTaskByRefName('st2').taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId2) def joinTaskId = workflowWithScheduledSubWorkflows.getTaskByRefName("fanouttask_join").taskId then: "verify that the sub workflow tasks are in a IN PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.IN_PROGRESS tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 'st2'] tasks[3].status == Task.Status.IN_PROGRESS } and: "Also verify that the sub workflows are in a RUNNING state" def workflowWithRunningSubWorkflows = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowInstanceId1 = workflowWithRunningSubWorkflows.getTaskByRefName('st1').subWorkflowId with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId1, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'simple_task_in_sub_wf' } def subWorkflowInstanceId2 = workflowWithRunningSubWorkflows.getTaskByRefName('st2').subWorkflowId with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId2, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'simple_task_in_sub_wf' } when: "The 'simple_task_in_sub_wf' belonging to both the sub workflows is polled and failed" def polledAndAckSubWorkflowTask1 = workflowTestUtil.pollAndFailTask('simple_task_in_sub_wf', 'task1.worker', 'Failed....') def polledAndAckSubWorkflowTask2 = workflowTestUtil.pollAndFailTask('simple_task_in_sub_wf', 'task1.worker', 'Failed....') then: "verify that both the tasks were polled and failed" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckSubWorkflowTask1) workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndAckSubWorkflowTask2) and: "verify that both the sub workflows are in failed state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId1, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'simple_task_in_sub_wf' } with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId2, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'simple_task_in_sub_wf' } sweep(workflowInstanceId) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that the workflow is in a COMPLETED state and the join task is also marked as COMPLETED_WITH_ERRORS" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.COMPLETED_WITH_ERRORS tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.COMPLETED_WITH_ERRORS tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.COMPLETED_WITH_ERRORS } when: "do a rerun on the sub workflow" def reRunSubWorkflowRequest = new RerunWorkflowRequest() reRunSubWorkflowRequest.reRunFromWorkflowId = subWorkflowInstanceId1 workflowExecutor.rerun(reRunSubWorkflowRequest) then: "verify that the sub workflows are in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId1, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'simple_task_in_sub_wf' } and: "parent workflow remains the same" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.COMPLETED_WITH_ERRORS tasks[2].taskType == 'SUB_WORKFLOW' tasks[2].status == Task.Status.COMPLETED_WITH_ERRORS tasks[3].taskType == 'JOIN' tasks[3].status == Task.Status.COMPLETED_WITH_ERRORS } } def "Test fork join with sub workflow task using task definition"() { given: "A input to the workflow that has fork with sub workflow task" Map workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "A workflow that has fork with sub workflow task is started" def workflowInstanceId = startWorkflow(FORK_JOIN_SUB_WORKFLOW, 1, '', workflowInput, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.IN_PROGRESS } when: "the subworkflow is started by issuing a system task call" def parentWorkflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = parentWorkflow.getTaskByRefName('st1').taskId def jointaskId = parentWorkflow.getTaskByRefName("fanouttask_join").taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) then: "verify that the sub workflow task is in a IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.IN_PROGRESS tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.IN_PROGRESS } when: "sub workflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowInstanceId = workflow.getTaskByRefName('st1').subWorkflowId then: "verify that the sub workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'simple_task_in_sub_wf' } when: "the 'simple_task_in_sub_wf' belonging to the sub workflow is polled and failed" def polledAndFailSubWorkflowTask = workflowTestUtil.pollAndFailTask('simple_task_in_sub_wf', 'task1.worker', 'Failed....') then: "verify that the task was polled and failed" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndFailSubWorkflowTask) and: "verify that the sub workflow is in failed state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'simple_task_in_sub_wf' } and: "verify that the workflow is in a RUNNING state and sub workflow task is retried" sweep(workflowInstanceId) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].status == Task.Status.SCHEDULED } when: "the sub workflow is started by issuing a system task call" parentWorkflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) subWorkflowTaskId = parentWorkflow.getTaskByRefName('st1').taskId asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) then: "verify that the sub workflow task is in a IN PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].status == Task.Status.IN_PROGRESS } when: "sub workflow is retrieved" workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) subWorkflowInstanceId = workflow.getTaskByRefName('st1').subWorkflowId then: "verify that the sub workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'simple_task_in_sub_wf' } when: "the 'simple_task_in_sub_wf' belonging to the sub workflow is polled and completed" def polledAndCompletedSubWorkflowTask = workflowTestUtil.pollAndCompleteTask('simple_task_in_sub_wf', 'subworkflow.task.worker') then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndCompletedSubWorkflowTask) and: "verify that the sub workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'simple_task_in_sub_wf' } and: "verify that the workflow is in a RUNNING state and sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].status == Task.Status.COMPLETED } when: "the simple task is polled and completed" def polledAndCompletedSimpleTask = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.worker') and: "workflow is evaluated" sweep(workflowInstanceId) then: "verify that the task was polled and acknowledged" workflowTestUtil.verifyPolledAndAcknowledgedTask(polledAndCompletedSimpleTask) when: "JOIN task is executed" asyncSystemTaskExecutor.execute(joinTask, jointaskId) then: "verify that the workflow is in a COMPLETED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'JOIN' tasks[3].inputData['joinOn'] == ['st1', 't2'] tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/HierarchicalForkJoinSubworkflowRerunSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class HierarchicalForkJoinSubworkflowRerunSpec extends AbstractSpecification { @Shared def FORK_JOIN_HIERARCHICAL_SUB_WF = 'hierarchical_fork_join_swf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Autowired Join joinTask String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('hierarchical_fork_join_swf.json', 'simple_workflow_1_integration_test.json' ) //region Test setup: 3 workflows reach FAILED state because task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(FORK_JOIN_HIERARCHICAL_SUB_WF, 1) and: "input required to start the workflow execution" String correlationId = 'rerun_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : FORK_JOIN_HIERARCHICAL_SUB_WF, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(FORK_JOIN_HIERARCHICAL_SUB_WF, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "poll and complete the integration_task_2 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the leaf workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } //endregion } def cleanup() { metadataService.updateTaskDef(persistedTask2Definition) } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the root workflow. * * Expectation: The root workflow gets a new execution with the same id and spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the NEW leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test rerun on the root-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the root workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = rootWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the root workflow created a new execution" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task in the root workflow" def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task in the mid-level workflow" def midJoinId = workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(newMidLevelWorkflowId) then: "the root workflow is in COMPLETED state" assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the mid-level workflow. * * Expectation: The mid-level workflow gets a new execution with the same id and spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the mid level workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = midLevelWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the mid workflow created a new execution" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "poll and complete the integration_task_2 task in the mid level workflow" def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the leaf workflow. * * Expectation: The leaf workflow gets a new execution with the same id and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the leaf-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the leaf workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = leafWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the leaf workflow created a new execution" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the mid level and root workflows are sweeped" sweep(midLevelWorkflowId) sweep(rootWorkflowId) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify that the mid level workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify that the root workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete both tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } void assertWorkflowIsCompleted(String workflowId) { assert with(workflowExecutionService.getExecutionStatus(workflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/HierarchicalForkJoinSubworkflowRestartSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class HierarchicalForkJoinSubworkflowRestartSpec extends AbstractSpecification { @Shared def FORK_JOIN_HIERARCHICAL_SUB_WF = 'hierarchical_fork_join_swf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Autowired Join joinTask String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('hierarchical_fork_join_swf.json', 'simple_workflow_1_integration_test.json' ) //region Test setup: 3 workflows reach FAILED state. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(FORK_JOIN_HIERARCHICAL_SUB_WF, 1) and: "input required to start the workflow execution" String correlationId = 'retry_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : FORK_JOIN_HIERARCHICAL_SUB_WF, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(FORK_JOIN_HIERARCHICAL_SUB_WF, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the leaf workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } //endregion } def cleanup() { metadataService.updateTaskDef(persistedTask2Definition) } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the root workflow. * * Expectation: The root workflow gets a new execution with the same id and spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the NEW leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test restart on the root in a 3-level subworkflow"() { //region Test case when: "do a restart on the root workflow" workflowExecutor.restart(rootWorkflowId, false) then: "verify that the root workflow created a new execution" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task in the root workflow" def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task in the mid-level workflow" def midJoinId = workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(newMidLevelWorkflowId) then: "the root workflow is in COMPLETED state" assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the mid-level workflow. * * Expectation: The mid-level workflow gets a new execution with the same id and spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test restart on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a restart on the mid level workflow" workflowExecutor.restart(midLevelWorkflowId, false) then: "verify that the mid workflow created a new execution" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "poll and complete the integration_task_2 task in the mid level workflow" def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) and: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the leaf workflow. * * Expectation: The leaf workflow gets a new execution with the same id and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test restart on the leaf in a 3-level subworkflow"() { //region Test case when: "do a restart on the leaf workflow" workflowExecutor.restart(leafWorkflowId, false) then: "verify that the leaf workflow created a new execution" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the mid level and root workflows are sweeped" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify that the root workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete both tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } void assertWorkflowIsCompleted(String workflowId) { assert with(workflowExecutionService.getExecutionStatus(workflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/HierarchicalForkJoinSubworkflowRetrySpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class HierarchicalForkJoinSubworkflowRetrySpec extends AbstractSpecification { @Shared def FORK_JOIN_HIERARCHICAL_SUB_WF = 'hierarchical_fork_join_swf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Autowired Join joinTask String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('hierarchical_fork_join_swf.json', 'simple_workflow_1_integration_test.json' ) //region Test setup: 3 workflows reach FAILED state. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(FORK_JOIN_HIERARCHICAL_SUB_WF, 1) and: "input required to start the workflow execution" String correlationId = 'retry_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : FORK_JOIN_HIERARCHICAL_SUB_WF, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(FORK_JOIN_HIERARCHICAL_SUB_WF, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the leaf workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } //endregion } def cleanup() { metadataService.updateTaskDef(persistedTask2Definition) } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the root workflow. * * Expectation: The root workflow spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test retry on the root in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(rootWorkflowId, false) then: "verify that the root workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == TASK_TYPE_SUB_WORKFLOW tasks[4].status == Task.Status.SCHEDULED tasks[4].retriedTaskId == tasks[1].taskId } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) def midJoinId = workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(newMidLevelWorkflowId) then: "the root workflow is in COMPLETED state" assertSubWorkflowTaskIsRetriedAndWorkflowCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the root workflow. * * Expectation: The leaf workflow is retried and both its parent (mid-level) and grand parent (root) workflows are also retried. * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a retry on the mid level workflow" workflowExecutor.retry(midLevelWorkflowId, false) then: "verify that the mid workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS tasks[4].taskType == TASK_TYPE_SUB_WORKFLOW tasks[4].status == Task.Status.SCHEDULED tasks[4].retriedTaskId == tasks[1].taskId } and: "verify the SUB_WORKFLOW task in root workflow is IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 2 tasks in the leaf workflow" def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertSubWorkflowTaskIsRetriedAndWorkflowCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the mid-level workflow. * * Expectation: The mid-level workflow spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the leaf in a 3-level subworkflow"() { //region Test case when: "do a retry on the leaf workflow" workflowExecutor.retry(leafWorkflowId, false) then: "verify that the leaf workflow is in RUNNING state and failed task is retried" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } then: "verify that the mid-level workflow's SUB_WORKFLOW task is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the root workflow's SUB_WORKFLOW task is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify that the mid-level workflow's JOIN task is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify that the root workflow's JOIN task is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "verify that the mid level and root workflows reach COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the root workflow. * * Expectation: The leaf workflow is retried and both its parent (mid-level) and grand parent (root) workflows are also retried. * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the root with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(rootWorkflowId, true) then: "verify that the sub workflow task in root workflow is IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the sub workflow task in mid level workflow is IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the previously failed task in leaf workflow is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify the mid level workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify the root workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) and: "the root workflow is in COMPLETED state" assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the mid-level workflow. * * Expectation: The leaf workflow resumes its FAILED task and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the mid-level with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(midLevelWorkflowId, true) then: "verify that the sub workflow task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the sub workflow task in mid level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the previously failed task in leaf workflow is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify the mid level workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify the root workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the previously failed integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) and: "the root workflow is in COMPLETED state" assertWorkflowIsCompleted(rootWorkflowId) //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the leaf workflow. * * Expectation: The leaf workflow resumes its FAILED task and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the leaf with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the leaf workflow" workflowExecutor.retry(leafWorkflowId, true) then: "verify that the leaf workflow is in RUNNING state and failed task is retried" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } and: "verify that the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.CANCELED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) def midJoinId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).getTaskByRefName("fanouttask_join").taskId def rootJoinId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).getTaskByRefName("fanouttask_join").taskId then: "verify the mid level workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } and: "verify the root workflow's JOIN is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.IN_PROGRESS } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, midJoinId) asyncSystemTaskExecutor.execute(joinTask, rootJoinId) then: "the new mid level workflow is in COMPLETED state" assertWorkflowIsCompleted(midLevelWorkflowId) and: "the root workflow is in COMPLETED state" assertWorkflowIsCompleted(rootWorkflowId) //endregion } void assertWorkflowIsCompleted(String workflowId) { assert with(workflowExecutionService.getExecutionStatus(workflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.COMPLETED } } void assertSubWorkflowTaskIsRetriedAndWorkflowCompleted(String workflowId) { assert with(workflowExecutionService.getExecutionStatus(workflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 5 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == TASK_TYPE_JOIN tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_SUB_WORKFLOW tasks[4].status == Task.Status.COMPLETED tasks[4].retriedTaskId == tasks[1].taskId } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/JsonJQTransformSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared class JsonJQTransformSpec extends AbstractSpecification { @Shared def JSON_JQ_TRANSFORM_WF = 'test_json_jq_transform_wf' @Shared def SEQUENTIAL_JSON_JQ_TRANSFORM_WF = 'sequential_json_jq_transform_wf' @Shared def JSON_JQ_TRANSFORM_RESULT_WF = 'json_jq_transform_result_wf' def setup() { workflowTestUtil.registerWorkflows( 'simple_json_jq_transform_integration_test.json', 'sequential_json_jq_transform_integration_test.json', 'json_jq_transform_result_integration_test.json' ) } /** * Given the following input JSON *{* "in1": {* "array": [ "a", "b" ] *}, * "in2": {* "array": [ "c", "d" ] *}*}* expect the workflow task to transform to following result: *{* out: [ "a", "b", "c", "d" ] *}*/ def "Test workflow with json jq transform task succeeds"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['in1'] = new HashMap() workflowInput['in1']['array'] = ["a", "b"] workflowInput['in2'] = new HashMap() workflowInput['in2']['array'] = ["c", "d"] when: "workflow which has the json jq transform task has started" def workflowInstanceId = startWorkflow(JSON_JQ_TRANSFORM_WF, 1, '', workflowInput, null) then: "verify that the workflow and task are completed with expected output" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") } } /** * Given the following input JSON *{* "in1": "a", * "in2": "b" *}* using the same query from the success test, jq will try to get in1['array'] * and fail since 'in1' is a string */ def "Test workflow with json jq transform task fails"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['in1'] = "a" workflowInput['in2'] = "b" when: "workflow which has the json jq transform task has started" def workflowInstanceId = startWorkflow(JSON_JQ_TRANSFORM_WF, 1, '', workflowInput, null) then: "verify that the workflow and task failed with expected error" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].reasonForIncompletion == 'Cannot index string with string \"array\"' } } /** * Given the following invalid input JSON *{* "in1": "a", * "in2": "b" *}* using the same query from the success test, jq will try to get in1['array'] * and fail since 'in1' is a string. * * Re-run failed system task with the following valid input JSON will fix the workflow *{* "in1": {* "array": [ "a", "b" ] *}, * "in2": {* "array": [ "c", "d" ] *}*}* expect the workflow task to transform to following result: *{* out: [ "a", "b", "c", "d" ] *} */ def "Test rerun workflow with failed json jq transform task"() { given: "workflow input" def invalidInput = new HashMap() invalidInput['in1'] = "a" invalidInput['in2'] = "b" def validInput = new HashMap() def input = new HashMap() input['in1'] = new HashMap() input['in1']['array'] = ["a", "b"] input['in2'] = new HashMap() input['in2']['array'] = ["c", "d"] validInput['input'] = input validInput['queryExpression'] = '.input as $_ | { out: ($_.in1.array + $_.in2.array) }' when: "workflow which has the json jq transform task started" def workflowInstanceId = startWorkflow(JSON_JQ_TRANSFORM_WF, 1, '', invalidInput, null) then: "verify that the workflow and task failed with expected error" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].reasonForIncompletion == 'Cannot index string with string \"array\"' } when: "workflow which has the json jq transform task reran" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = workflowInstanceId def reRunTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[0].taskId reRunWorkflowRequest.reRunFromTaskId = reRunTaskId reRunWorkflowRequest.taskInput = validInput workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the workflow and task are completed with expected output" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") } } def "Test json jq transform task with nested json object succeeds"() { given: "workflow input" def workflowInput = new HashMap() workflowInput["method"] = "POST" workflowInput['body'] = new HashMap() workflowInput['body']['name'] = "Beary Beariston" workflowInput['body']['title'] = "the Brown Bear" workflowInput["requestTransform"] = "{name: (.body.name + \" you are \" + .body.title) }" workflowInput["responseTransform"] = "{result: \"reply: \" + .response.body.message}" when: "workflow which has the json jq transform task has started" def workflowInstanceId = startWorkflow(SEQUENTIAL_JSON_JQ_TRANSFORM_WF, 1, '', workflowInput, null) then: "verify that the workflow and task are completed with expected output" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") HashMap result1 = (HashMap) tasks[0].outputData.get("result") result1.get("method") == workflowInput["method"] result1.get("requestTransform") == workflowInput["requestTransform"] result1.get("responseTransform") == workflowInput["responseTransform"] tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'JSON_JQ_TRANSFORM' tasks[1].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") HashMap result2 = (HashMap) tasks[1].outputData.get("result") result2.get("name") == "Beary Beariston you are the Brown Bear" } } def "Test json jq transform task with different json object results succeeds"() { given: "workflow input" def workflowInput = new HashMap() workflowInput["requestedAction"] = "redeliver" when: "workflow which has the json jq transform task has started" def workflowInstanceId = startWorkflow(JSON_JQ_TRANSFORM_RESULT_WF, 1, '', workflowInput, null) then: "verify that the workflow and task are completed with expected output" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'JSON_JQ_TRANSFORM' tasks[0].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") assert tasks[0].outputData.get("result") == "CREATE" tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'DECISION' assert tasks[1].inputData.get("case") == "CREATE" tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'JSON_JQ_TRANSFORM' tasks[2].outputData.containsKey("result") && tasks[0].outputData.containsKey("resultList") List result = (List) tasks[2].outputData.get("result") assert result.size() == 3 assert result.indexOf("redeliver") >= 0 tasks[3].status == Task.Status.COMPLETED tasks[3].taskType == 'DECISION' assert tasks[3].inputData.get("case") == "true" } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/LambdaAndTerminateTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class LambdaAndTerminateTaskSpec extends AbstractSpecification { @Shared def WORKFLOW_WITH_TERMINATE_TASK = 'test_terminate_task_wf' @Shared def WORKFLOW_WITH_TERMINATE_TASK_FAILED = 'test_terminate_task_failed_wf' @Shared def WORKFLOW_WITH_LAMBDA_TASK = 'test_lambda_wf' @Shared def PARENT_WORKFLOW_WITH_TERMINATE_TASK = 'test_terminate_task_parent_wf' @Shared def WORKFLOW_WITH_DECISION_AND_TERMINATE = "ConditionalTerminateWorkflow" @Autowired SubWorkflow subWorkflowTask def setup() { workflowTestUtil.registerWorkflows( 'failure_workflow_for_terminate_task_workflow.json', 'terminate_task_completed_workflow_integration_test.json', 'terminate_task_failed_workflow_integration.json', 'simple_lambda_workflow_integration_test.json', 'terminate_task_parent_workflow.json', 'terminate_task_sub_workflow.json', 'decision_and_terminate_integration_test.json' ) } def "Test workflow with a terminate task when the status is completed"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the terminate task" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_TERMINATE_TASK, 1, '', workflowInput, null) then: "Ensure that the workflow has started and the first task is in scheduled state and workflow output should be terminate task's output" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 reasonForIncompletion.contains('Workflow is COMPLETED by TERMINATE task') tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'TERMINATE' tasks[1].seq == 2 output.size() == 1 output as String == "[result:[testvalue:true]]" } } def "Test workflow with a terminate task when the status is failed"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the terminate task" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_TERMINATE_TASK_FAILED, 1, '', workflowInput, null) then: "Verify that the workflow has failed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 reasonForIncompletion == "Early exit in terminate" tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'TERMINATE' tasks[1].seq == 2 output def failedWorkflowId = output['conductor.failure_workflow'] as String with(workflowExecutionService.getExecutionStatus(failedWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED input['workflowId'] == workflowInstanceId tasks.size() == 1 tasks[0].taskType == 'LAMBDA' } } } def "Test workflow with a terminate task when the workflow has a subworkflow"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the terminate task" def workflowInstanceId = startWorkflow(PARENT_WORKFLOW_WITH_TERMINATE_TASK, 1, '', workflowInput, null) then: "verify that the workflow has started and the tasks are as expected" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'LAMBDA' tasks[1].referenceTaskName == 'lambdaTask1' tasks[1].seq == 2 tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'LAMBDA' tasks[2].referenceTaskName == 'lambdaTask2' tasks[2].seq == 3 tasks[3].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'JOIN' tasks[3].seq == 4 tasks[4].status == Task.Status.SCHEDULED tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].seq == 5 tasks[5].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'WAIT' tasks[5].seq == 6 } when: "subworkflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowTaskId = workflow.getTaskByRefName("test_terminate_subworkflow").getTaskId() asyncSystemTaskExecutor.execute(subWorkflowTask, subWorkflowTaskId) workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.getTaskByRefName("test_terminate_subworkflow").subWorkflowId then: "verify that the sub workflow is RUNNING, and the task within is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_3' tasks[0].status == Task.Status.SCHEDULED } when: "Complete the WAIT task that should cause the TERMINATE task to execute" def waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[5] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) then: "Verify that the workflow has completed and the SUB_WORKFLOW is not still IN_PROGRESS (should be SKIPPED)" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 reasonForIncompletion.contains('Workflow is COMPLETED by TERMINATE task') tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'FORK' tasks[0].seq == 1 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'LAMBDA' tasks[1].referenceTaskName == 'lambdaTask1' tasks[1].seq == 2 tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'LAMBDA' tasks[2].referenceTaskName == 'lambdaTask2' tasks[2].seq == 3 tasks[3].status == Task.Status.CANCELED tasks[3].taskType == 'JOIN' tasks[3].seq == 4 tasks[4].status == Task.Status.CANCELED tasks[4].taskType == 'SUB_WORKFLOW' tasks[4].seq == 5 tasks[5].status == Task.Status.COMPLETED tasks[5].taskType == 'WAIT' tasks[5].seq == 6 tasks[6].status == Task.Status.COMPLETED tasks[6].taskType == 'TERMINATE' tasks[6].seq == 7 } and: "ensure that the subworkflow is terminated" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 reasonForIncompletion.contains('Parent workflow has been terminated with reason: Workflow is COMPLETED by' + ' TERMINATE task') tasks[0].taskType == 'integration_task_3' tasks[0].status == Task.Status.CANCELED } } def "Test workflow with a terminate task within a decision branch"() { given: "workflow input" Map workflowInput = new HashMap() workflowInput['param1'] = 'p1' workflowInput['param2'] = 'p2' workflowInput['case'] = 'two' when: "The workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_DECISION_AND_TERMINATE, 1, '', workflowInput, null) then: "verify that the workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].seq == 1 } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op':'task1 completed']) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has FAILED due to terminate task" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 3 output.size() == 1 output as String == "[output:task1 completed]" reasonForIncompletion.contains('Workflow is FAILED by TERMINATE task') tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['op'] == 'task1 completed' tasks[0].seq == 1 tasks[1].taskType == 'DECISION' tasks[1].status == Task.Status.COMPLETED tasks[1].seq == 2 tasks[2].taskType == 'TERMINATE' tasks[2].status == Task.Status.COMPLETED tasks[2].seq == 3 } } def "Test workflow with lambda task"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['a'] = 1 when: "Start the workflow which has the terminate task" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_LAMBDA_TASK, 1, '', workflowInput, null) then: "verify that the task is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'LAMBDA' tasks[0].outputData as String == "[result:[testvalue:true]]" tasks[0].seq == 1 } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/NestedForkJoinSubWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW class NestedForkJoinSubWorkflowSpec extends AbstractSpecification { @Shared def FORK_JOIN_NESTED_SUB_WF = 'nested_fork_join_swf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" @Autowired QueueDAO queueDAO @Autowired Join joinTask @Autowired SubWorkflow subWorkflowTask String parentWorkflowId, subworkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('nested_fork_join_swf.json', 'simple_workflow_1_integration_test.json' ) //region Test setup: 3 workflows reach FAILED state. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(FORK_JOIN_NESTED_SUB_WF, 1) and: "input required to start the workflow execution" String correlationId = 'retry_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : SIMPLE_WORKFLOW] when: "the workflow is started" parentWorkflowId = startWorkflow(FORK_JOIN_NESTED_SUB_WF, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "poll and complete the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) and: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds.get(0)) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def parentWorkflowInstance = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) with(parentWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" subworkflowId = parentWorkflowInstance.tasks[2].subWorkflowId with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "poll and fail the integration_task_2 task in the sub workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'task2 failed') then: "the sub workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the parent workflow is swept" sweep(parentWorkflowId) with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.CANCELED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.CANCELED } //endregion } def cleanup() { metadataService.updateTaskDef(persistedTask2Definition) } /** * On a nested fork join workflow where all workflows reach FAILED state because of a FAILED task * in the sub workflow. * * A restart is executed on the sub workflow. * * Expectation: The sub workflow spawns a execution with the same id. * When the sub workflow completes successfully, the parent workflow also completes successfully. */ def "test restart on the sub workflow in a nested fork join workflow"() { when: workflowExecutor.restart(subworkflowId, false) then: "verify that the subworkflow is RUNNING state" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "verify that the parent workflow is updated" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.CANCELED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.CANCELED } when: "the parent workflow is swept" def workflow = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) def outerJoinId = workflow.getTaskByRefName("outer_join").taskId def innerJoinId = workflow.getTaskByRefName("inner_join").taskId sweep(parentWorkflowId) then: "verify that the flag is reset and JOIN is updated" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "poll and complete both tasks in the sub workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the subworkflow completed" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } and: "verify that the parent workflow's sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "the parent workflow is swept" sweep(parentWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify that the parent workflow reaches COMPLETED with all tasks completed" assertParentWorkflowIsComplete() } /** * On a nested fork join workflow where all workflows reach FAILED state because of a FAILED task * in the sub workflow. * * A restart is executed on the parent workflow. * * Expectation: The parent workflow spawns a execution with the same id, which in turn creates a new instance of the sub workflow. * When the sub workflow completes successfully, the parent workflow also completes successfully. */ def "test restart on the parent workflow in a nested fork join workflow"() { when: workflowExecutor.restart(parentWorkflowId, false) then: "verify that the parent workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds.get(0)) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) def workflow = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) def outerJoinId = workflow.getTaskByRefName("outer_join").taskId def innerJoinId = workflow.getTaskByRefName("inner_join").taskId then: "verify that SUB_WORKFLOW task in in progress" def parentWorkflowInstance = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } and: "verify that a new instance of the sub workflow is created" def newSubWorkflowId = parentWorkflowInstance.tasks[2].subWorkflowId newSubWorkflowId != subworkflowId with(workflowExecutionService.getExecutionStatus(newSubWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete both tasks in the sub workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the subworkflow completed" with(workflowExecutionService.getExecutionStatus(newSubWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } and: "verify that the parent workflow's sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "the parent workflow is swept" sweep(parentWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify that the parent workflow reaches COMPLETED with all tasks completed" assertParentWorkflowIsComplete() } /** * On a nested fork join workflow where all workflows reach FAILED state because of a FAILED task * in the sub workflow. * * A retry is executed on the parent workflow. * * Expectation: The parent workflow spawns a execution with the same id, which in turn creates a new instance of the sub workflow. * When the sub workflow completes successfully, the parent workflow also completes successfully. */ def "test retry on the parent workflow in a nested fork join workflow"() { when: workflowExecutor.retry(parentWorkflowId, false) then: "verify that the parent workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.FAILED tasks[2].retried tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS tasks[7].taskType == TASK_TYPE_SUB_WORKFLOW tasks[7].status == Task.Status.SCHEDULED tasks[7].retriedTaskId == tasks[2].taskId } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds.get(0)) then: "verify that SUB_WORKFLOW task in in progress" def parentWorkflowInstance = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.FAILED tasks[2].retried tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS tasks[7].taskType == TASK_TYPE_SUB_WORKFLOW tasks[7].status == Task.Status.IN_PROGRESS tasks[7].retriedTaskId == tasks[2].taskId } and: "verify that a new instance of the sub workflow is created" def newSubWorkflowId = parentWorkflowInstance.tasks[7].subWorkflowId newSubWorkflowId != subworkflowId with(workflowExecutionService.getExecutionStatus(newSubWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete both tasks in the sub workflow" def workflow = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) def outerJoinId = workflow.getTaskByRefName("outer_join").taskId def innerJoinId = workflow.getTaskByRefName("inner_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the subworkflow completed" with(workflowExecutionService.getExecutionStatus(newSubWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } and: "verify that the parent workflow's sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 8 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.FAILED tasks[2].retried tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS tasks[7].taskType == TASK_TYPE_SUB_WORKFLOW tasks[7].status == Task.Status.COMPLETED tasks[7].retriedTaskId == tasks[2].taskId } when: "the parent workflow is swept" sweep(parentWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify that the parent workflow reaches COMPLETED with all tasks completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 8 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.FAILED tasks[2].retried tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.COMPLETED tasks[7].taskType == TASK_TYPE_SUB_WORKFLOW tasks[7].status == Task.Status.COMPLETED tasks[7].retriedTaskId == tasks[2].taskId } } /** * On a nested fork join workflow where all workflows reach FAILED state because of a FAILED task * in the sub workflow. * * A retry with resume flag is executed on the parent workflow. * * Expectation: The parent workflow spawns a execution with the same id, which in turn creates a new instance of the sub workflow. * When the sub workflow completes successfully, the parent workflow also completes successfully. */ def "test retry with resume on the parent workflow in a nested fork join workflow"() { when: workflowExecutor.retry(parentWorkflowId, true) then: "verify that the sub workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } and: "verify that the parent's SUB_WORKFLOW task is updated" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.CANCELED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.CANCELED } when: "the parent is swept" sweep(parentWorkflowId) then: "verify that parent's JOIN task in in progress" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "poll and complete the failed task in the sub workflow" def workflow = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) def outerJoinId = workflow.getTaskByRefName("outer_join").taskId def innerJoinId = workflow.getTaskByRefName("inner_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the subworkflow completed" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } and: "verify that the parent workflow's sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "the parent workflow is swept" sweep(parentWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify the parent workflow reaches COMPLETED state" assertParentWorkflowIsComplete() } /** * On a nested fork join workflow where all workflows reach FAILED state because of a FAILED task * in the sub workflow. * * A retry is executed on the parent workflow. * * Expectation: The parent workflow spawns a execution with the same id, which in turn creates a new instance of the sub workflow. * When the sub workflow completes successfully, the parent workflow also completes successfully. */ def "test retry on the sub workflow in a nested fork join workflow"() { when: workflowExecutor.retry(subworkflowId, false) then: "verify that the sub workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } and: "verify that the parent's SUB_WORKFLOW task is updated" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.CANCELED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.CANCELED } when: "the parent is swept" sweep(parentWorkflowId) then: "verify that parent's JOIN task in in progress" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.IN_PROGRESS !tasks[2].subworkflowChanged tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "poll and complete the failed task in the sub workflow" def workflow = workflowExecutionService.getExecutionStatus(parentWorkflowId, true) def outerJoinId = workflow.getTaskByRefName("outer_join").taskId def innerJoinId = workflow.getTaskByRefName("inner_join").taskId workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the subworkflow completed" with(workflowExecutionService.getExecutionStatus(subworkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } and: "verify that the parent workflow's sub workflow task is completed" with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.IN_PROGRESS } when: "the parent workflow is swept" sweep(parentWorkflowId) and: "JOIN tasks are executed" asyncSystemTaskExecutor.execute(joinTask, innerJoinId) asyncSystemTaskExecutor.execute(joinTask, outerJoinId) then: "verify the parent workflow reaches COMPLETED state" assertParentWorkflowIsComplete() } private void assertParentWorkflowIsComplete() { assert with(workflowExecutionService.getExecutionStatus(parentWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[0].taskType == TASK_TYPE_FORK tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_FORK tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == TASK_TYPE_JOIN tasks[4].status == Task.Status.COMPLETED tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == TASK_TYPE_JOIN tasks[6].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SetVariableTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared class SetVariableTaskSpec extends AbstractSpecification { @Shared def SET_VARIABLE_WF = 'test_set_variable_wf' def setup() { workflowTestUtil.registerWorkflows( 'simple_set_variable_workflow_integration_test.json' ) } def "Test workflow with set variable task"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['var'] = "var_test_value" when: "Start the workflow which has the set variable task" def workflowInstanceId = startWorkflow(SET_VARIABLE_WF, 1, '', workflowInput, null) then: "verify that the task is completed and variables were set" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'SET_VARIABLE' tasks[0].status == Task.Status.COMPLETED variables as String == '[var:var_test_value]' output as String == '[variables:[var:var_test_value]]' } } def "Test workflow with set variable task passing variables payload size threshold"() { given: "workflow input" def workflowInput = new HashMap() long maxThreshold = 2 workflowInput['var'] = String.join("", Collections.nCopies(1 + ((int) (maxThreshold * 1024 / 8)), "01234567")) when: "Start the workflow which has the set variable task" def workflowInstanceId = startWorkflow(SET_VARIABLE_WF, 1, '', workflowInput, null) def EXTRA_HASHMAP_SIZE = 17 def expectedErrorMessage = String.format( "The variables payload size: %d of workflow: %s is greater than the permissible limit: %d bytes", EXTRA_HASHMAP_SIZE + maxThreshold * 1024 + 1, workflowInstanceId, maxThreshold) then: "verify that the task is completed and variables were set" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].taskType == 'SET_VARIABLE' tasks[0].status == Task.Status.FAILED_WITH_TERMINAL_ERROR tasks[0].reasonForIncompletion == expectedErrorMessage variables as String == '[:]' output as String == '[variables:[:]]' } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SimpleWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.exception.ConflictException import com.netflix.conductor.core.exception.NotFoundException import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SimpleWorkflowSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Shared def LINEAR_WORKFLOW_T1_T2 = 'integration_test_wf' @Shared def INTEGRATION_TEST_WF_NON_RESTARTABLE = "integration_test_wf_non_restartable" def setup() { //Register LINEAR_WORKFLOW_T1_T2, RTOWF, WORKFLOW_WITH_OPTIONAL_TASK workflowTestUtil.registerWorkflows('simple_workflow_1_integration_test.json', 'simple_workflow_with_resp_time_out_integration_test.json') } def "Test simple workflow completion"() { given: "An existing simple workflow definition" metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) and: "input required to start the workflow execution" String correlationId = 'unit_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' when: "Start a workflow based on the registered simple workflow" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Poll and complete the 'integration_task_1' " def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete 'integration_task_2'" def pollAndCompleteTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the 'integration_task_2' has been polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try1, ['tp1': inputParam1, 'tp2': 'task1.done']) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED output.containsKey('o3') } } def "Test simple workflow with null inputs"() { when: "An existing simple workflow definition" def workflowDef = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) then: workflowDef.getTasks().get(0).getInputParameters().containsKey('someNullKey') when: "Start a workflow based on the registered simple workflow with one input param null" String correlationId = "unit_test_1" def input = new HashMap() input.put("param1", "p1 value") input.put("param2", null) def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "verify the workflow has started and the input params have propagated" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 input['param2'] == null tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED !tasks[0].inputData['someNullKey'] } when: "'integration_task_1' is polled and completed with output data" def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['someOtherKey': ['a': 1, 'A': null], 'someKey': null]) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the task is completed and the output data has propagated as input data to 'integration_task_2'" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData.containsKey('someKey') !tasks[0].outputData['someKey'] def someOtherKey = tasks[0].outputData['someOtherKey'] as Map someOtherKey.containsKey('A') !someOtherKey['A'] } } def "Test simple workflow terminal error condition"() { setup: "Modify the task definition and the workflow output definition" def persistedTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTask1Definition = new TaskDef(persistedTask1Definition.name, persistedTask1Definition.description, persistedTask1Definition.ownerEmail, 1, persistedTask1Definition.timeoutSeconds, persistedTask1Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask1Definition) def workflowDef = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) def outputParameters = workflowDef.outputParameters outputParameters['validationErrors'] = '${t1.output.ErrorMessage}' metadataService.updateWorkflowDef(workflowDef) when: "A simple workflow which is started" String correlationId = "unit_test_1" def input = new HashMap() input.put("param1", "p1 value") input.put("param2", "p2 value") def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "verify that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 } when: "Rewind the running workflow that was just started" workflowExecutor.restart(workflowInstanceId, false) then: "Ensure that a exception is thrown when a running workflow is being rewind" def exceptionThrown = thrown(ConflictException.class) exceptionThrown != null when: "'integration_task_1' is polled and failed with terminal error" def polledIntegrationTask1 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') TaskResult taskResult = new TaskResult(polledIntegrationTask1) taskResult.reasonForIncompletion = 'NON TRANSIENT ERROR OCCURRED: An integration point required to complete the task is down' taskResult.status = TaskResult.Status.FAILED_WITH_TERMINAL_ERROR taskResult.addOutputData('TERMINAL_ERROR', 'Integration endpoint down: FOOBAR') taskResult.addOutputData('ErrorMessage', 'There was a terminal error') workflowExecutionService.updateTask(taskResult) sweep(workflowInstanceId) then: "The first polled task is integration_task_1 and the workflowInstanceId of the task is same as running workflowInstanceId" polledIntegrationTask1 polledIntegrationTask1.taskType == 'integration_task_1' polledIntegrationTask1.workflowInstanceId == workflowInstanceId and: "verify that the workflow is in a failed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED def t1 = getTaskByRefName('t1') reasonForIncompletion == "Task ${t1.taskId} failed with status: FAILED and reason: " + "'NON TRANSIENT ERROR OCCURRED: An integration point required to complete the task is down'" output['o1'] == 'p1 value' output['validationErrors'] == 'There was a terminal error' t1.retryCount == 0 failedReferenceTaskNames == ['t1'] as HashSet failedTaskNames == ['integration_task_1'] as HashSet } cleanup: metadataService.updateTaskDef(modifiedTask1Definition) outputParameters.remove('validationErrors') metadataService.updateWorkflowDef(workflowDef) } def "Test Simple Workflow with response timeout "() { given: 'Workflow input and correlationId' def correlationId = 'unit_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "Start a workflow that has a response time out" def workflowInstanceId = startWorkflow('RTOWF', 1, correlationId, workflowInput, null) then: "Workflow is in running state and the task 'task_rt' is ready to be polled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'task_rt' tasks[0].status == Task.Status.SCHEDULED } queueDAO.getSize('task_rt') == 1 when: "Poll for a 'task_rt' task and then ack the task" def polledTaskRtTry1 = workflowExecutionService.poll('task_rt', 'task1.integration.worker.testTimeout') then: "Verify that the 'task_rt' was polled" polledTaskRtTry1 polledTaskRtTry1.taskType == 'task_rt' polledTaskRtTry1.workflowInstanceId == workflowInstanceId polledTaskRtTry1.status == Task.Status.IN_PROGRESS when: "An additional poll is done wto retrieved another 'task_rt'" def noTaskAvailable = workflowExecutionService.poll('task_rt', 'task1.integration.worker.testTimeout') then: "Ensure that there is no additional 'task_rt' available to poll" !noTaskAvailable when: "The processing of the polled task takes more time than the response time out" Thread.sleep(10000) workflowExecutor.decide(workflowInstanceId) then: "Expect a new task to be added to the queue in place of the timed out task" queueDAO.getSize('task_rt') == 1 with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].status == Task.Status.TIMED_OUT tasks[1].status == Task.Status.SCHEDULED } when: "The task_rt is polled again and the task is set to be called back after 2 seconds" def polledTaskRtTry2 = workflowExecutionService.poll('task_rt', 'task1.integration.worker.testTimeout') polledTaskRtTry2.callbackAfterSeconds = 2 polledTaskRtTry2.status = Task.Status.IN_PROGRESS workflowExecutionService.updateTask(new TaskResult(polledTaskRtTry2)) then: "verify that the polled task is not null" polledTaskRtTry2 and: "verify the state of the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].status == Task.Status.SCHEDULED } when: "induce the time for the call back for the task to expire and run the un ack process" Thread.sleep(2010) queueDAO.processUnacks(polledTaskRtTry2.taskDefName) and: "run the decide process on the workflow" workflowExecutor.decide(workflowInstanceId) and: "poll for the task and then complete the task 'task_rt' " def pollAndCompleteTaskTry3 = workflowTestUtil.pollAndCompleteTask('task_rt', 'task1.integration.worker.testTimeout', ['op': 'task1.done']) then: 'Verify that the task was polled ' verifyPolledAndAcknowledgedTask(pollAndCompleteTaskTry3) when: "The next task of the workflow is polled and then completed" def polledIntegrationTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker.testTimeout') then: "Verify that 'integration_task_2' is polled and acked" verifyPolledAndAcknowledgedTask(polledIntegrationTask2Try1) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED } } def "Test if the workflow definitions with and without schema version can be registered"() { given: "A workflow definition with no schema version" def workflowDef1 = new WorkflowDef() workflowDef1.name = 'Test_schema_version1' workflowDef1.version = 1 workflowDef1.ownerEmail = "test@harness.com" and: "A new workflow task is created" def workflowTask = new WorkflowTask() workflowTask.name = 'integration_task_1' workflowTask.taskReferenceName = 't1' workflowDef1.tasks.add(workflowTask) and: "The workflow definition with no schema version is saved" metadataService.updateWorkflowDef(workflowDef1) and: "A workflow definition with a schema version is created" def workflowDef2 = new WorkflowDef() workflowDef2.name = 'Test_schema_version2' workflowDef2.version = 1 workflowDef2.schemaVersion = 2 workflowDef2.ownerEmail = "test@harness.com" workflowDef2.tasks.add(workflowTask) and: "The workflow definition with schema version is persisted" metadataService.updateWorkflowDef(workflowDef2) when: "The persisted workflow definitions are retrieved by their name" def foundWorkflowDef1 = metadataService.getWorkflowDef(workflowDef1.getName(), 1) def foundWorkflowDef2 = metadataService.getWorkflowDef(workflowDef2.getName(), 1) then: "Ensure that the schema version is by default 2" foundWorkflowDef1 foundWorkflowDef1.schemaVersion == 2 foundWorkflowDef2 foundWorkflowDef2.schemaVersion == 2 } def "Test Simple workflow restart without using the latest definition"() { setup: "Register a task definition with no retries" def persistedTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTaskDefinition = new TaskDef(persistedTask1Definition.name, persistedTask1Definition.description, persistedTask1Definition.ownerEmail, 0, persistedTask1Definition.timeoutSeconds, persistedTask1Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTaskDefinition) when: "Get the workflow definition associated with the simple workflow" WorkflowDef workflowDefinition = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) then: "Ensure that there is a workflow definition" workflowDefinition workflowDefinition.failureWorkflow StringUtils.isNotBlank(workflowDefinition.failureWorkflow) when: "Start a simple workflow with non null params" def correlationId = 'integration_test_1' + UUID.randomUUID().toString() def workflowInput = new HashMap() String inputParam1 = 'p1 value' workflowInput['param1'] = inputParam1 workflowInput['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "A workflow instance has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } when: "poll the task that is queued and fail the task" def polledIntegrationTask1Try1 = workflowTestUtil.pollAndFailTask('integration_task_1', 'task1.integration.worker', 'failed..') then: "The workflow ends up in a failed state" verifyPolledAndAcknowledgedTask(polledIntegrationTask1Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'integration_task_1' } when: "Rewind the workflow which is in the failed state without the latest definition" workflowExecutor.restart(workflowInstanceId, false) then: "verify that the rewound workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } when: "Poll for the 'integration_task_1' " def polledIntegrationTask1Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task is polled and the workflow is in a running state" verifyPolledAndAcknowledgedTask(polledIntegrationTask1Try2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' } when: def polledIntegrationTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: verifyPolledAndAcknowledgedTask(polledIntegrationTask2Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' } cleanup: metadataService.updateTaskDef(persistedTask1Definition) } def "Test Simple workflow restart with the latest definition"() { setup: "Register a task definition with no retries" def persistedTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTaskDefinition = new TaskDef(persistedTask1Definition.name, persistedTask1Definition.description, persistedTask1Definition.ownerEmail, 0, persistedTask1Definition.timeoutSeconds, persistedTask1Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTaskDefinition) when: "Get the workflow definition associated with the simple workflow" WorkflowDef workflowDefinition = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) then: "Ensure that there is a workflow definition" workflowDefinition workflowDefinition.failureWorkflow StringUtils.isNotBlank(workflowDefinition.failureWorkflow) when: "Start a simple workflow with non null params" def correlationId = 'integration_test_1' + UUID.randomUUID().toString() def workflowInput = new HashMap() String inputParam1 = 'p1 value' workflowInput['param1'] = inputParam1 workflowInput['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "A workflow instance has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } when: "poll the task that is queued and fail the task" def polledIntegrationTask1Try1 = workflowTestUtil.pollAndFailTask('integration_task_1', 'task1.integration.worker', 'failed..') then: "the workflow ends up in a failed state" verifyPolledAndAcknowledgedTask(polledIntegrationTask1Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'integration_task_1' } when: "A new version of the workflow definition is registered" WorkflowTask workflowTask = new WorkflowTask() workflowTask.name = 'integration_task_20' workflowTask.taskReferenceName = 'task_added' workflowTask.workflowTaskType = TaskType.SIMPLE workflowDefinition.tasks.add(workflowTask) workflowDefinition.version = 2 metadataService.updateWorkflowDef(workflowDefinition) and: "rewind/restart the workflow with the latest workflow definition" workflowExecutor.restart(workflowInstanceId, true) then: "verify that the rewound workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } when: "Poll and complete the 'integration_task_1' " def polledIntegrationTask1Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task is polled and the workflow is in a running state" verifyPolledAndAcknowledgedTask(polledIntegrationTask1Try2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' } when: "Poll and complete the 'integration_task_2' " def polledIntegrationTask2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(polledIntegrationTask2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } when: "Poll and complete the 'integration_task_20' " def polledIntegrationTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task1.integration.worker') then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(polledIntegrationTask20Try1) def polledIntegrationTask20 = polledIntegrationTask20Try1[0] as Task polledIntegrationTask20.workflowInstanceId == workflowInstanceId polledIntegrationTask20.referenceTaskName == 'task_added' with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 } cleanup: metadataService.updateTaskDef(persistedTask1Definition) metadataService.unregisterWorkflowDef(workflowDefinition.getName(), 2) } def "Test simple workflow with task retries"() { setup: "Change the task definition to ensure that it has retries and delay between retries" def integrationTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTaskDefinition = new TaskDef(integrationTask2Definition.name, integrationTask2Definition.description, integrationTask2Definition.ownerEmail, 3, integrationTask2Definition.timeoutSeconds, integrationTask2Definition.responseTimeoutSeconds) modifiedTaskDefinition.retryDelaySeconds = 2 metadataService.updateTaskDef(modifiedTaskDefinition) when: "A new simple workflow is started" def correlationId = 'integration_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "verify that the workflow has started" workflowInstanceId def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) workflow.status == Workflow.WorkflowStatus.RUNNING when: "Poll for the first task and complete the task" def polledIntegrationTask1 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') polledIntegrationTask1.status = Task.Status.COMPLETED def polledIntegrationTask1Output = "task1.output -> " + polledIntegrationTask1.inputData['p1'] + "." + polledIntegrationTask1.inputData['p2'] polledIntegrationTask1.outputData['op'] = polledIntegrationTask1Output workflowExecutionService.updateTask(new TaskResult(polledIntegrationTask1)) then: "verify that the 'integration_task_1' is polled and completed" with(polledIntegrationTask1) { inputData.containsKey('p1') inputData.containsKey('p2') inputData['p1'] == 'p1 value' inputData['p2'] == 'p2 value' } //Need to figure out how to use expect and where here when: " 'integration_task_2' is polled and marked as failed for the first time" Tuple polledAndFailedTaskTry1 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failure...0', null, 2) then: "verify that the task was polled and the input params of the tasks are as expected" verifyPolledAndAcknowledgedTask(polledAndFailedTaskTry1, ['tp2': polledIntegrationTask1Output, 'tp1': 'p1 value']) when: " 'integration_task_2' is polled and marked as failed for the second time" Tuple polledAndFailedTaskTry2 = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failure...0', null, 2) then: "verify that the task was polled and the input params of the tasks are as expected" verifyPolledAndAcknowledgedTask(polledAndFailedTaskTry2, ['tp2': polledIntegrationTask1Output, 'tp1': 'p1 value']) when: "'integration_task_2' is polled and marked as completed for the third time" def polledAndCompletedTry3 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the task was polled and the input params of the tasks are as expected" verifyPolledAndAcknowledgedTask(polledAndCompletedTry3, ['tp2': polledIntegrationTask1Output, 'tp1': 'p1 value']) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.FAILED tasks[3].taskType == 'integration_task_2' tasks[3].status == Task.Status.COMPLETED tasks[1].taskId == tasks[2].retriedTaskId tasks[2].taskId == tasks[3].retriedTaskId failedReferenceTaskNames == ['t2'] as HashSet failedTaskNames == ['integration_task_2'] as HashSet } cleanup: metadataService.updateTaskDef(integrationTask2Definition) } def "Test simple workflow with retry at workflow level"() { setup: "Change the task definition to ensure that it has retries and no delay between retries" def integrationTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTaskDefinition = new TaskDef(integrationTask1Definition.name, integrationTask1Definition.description, integrationTask1Definition.ownerEmail, 1, integrationTask1Definition.timeoutSeconds, integrationTask1Definition.responseTimeoutSeconds) modifiedTaskDefinition.retryDelaySeconds = 0 metadataService.updateTaskDef(modifiedTaskDefinition) when: "Start a simple workflow with non null params" def correlationId = 'retry_test' + UUID.randomUUID().toString() def workflowInput = new HashMap() String inputParam1 = 'p1 value' workflowInput['param1'] = inputParam1 workflowInput['param2'] = 'p2 value' and: "start a simple workflow with input params" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "verify that the workflow has started and the next task is scheduled" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].getInputData().get("p3") == tasks[0].getTaskId() } with(metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1)) { failureWorkflow StringUtils.isNotBlank(failureWorkflow) } when: "The first task 'integration_task_1' is polled and failed" Tuple polledAndFailedTask1Try1 = workflowTestUtil.pollAndFailTask('integration_task_1', 'task1.integration.worker', 'failure...0') then: "verify that the task was polled and acknowledged and the workflow is still in a running state" verifyPolledAndAcknowledgedTask(polledAndFailedTask1Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].status == Task.Status.FAILED tasks[1].status == Task.Status.SCHEDULED tasks[1].getInputData().get("p3") == tasks[1].getTaskId() } when: "The first task 'integration_task_1' is polled and failed for the second time" Tuple polledAndFailedTask1Try2 = workflowTestUtil.pollAndFailTask('integration_task_1', 'task1.integration.worker', 'failure...0') then: "verify that the task was polled and acknowledged and the workflow is still in a running state" verifyPolledAndAcknowledgedTask(polledAndFailedTask1Try2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].status == Task.Status.FAILED tasks[1].status == Task.Status.FAILED } when: "The workflow is retried" workflowExecutor.retry(workflowInstanceId, false) then: with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].status == Task.Status.FAILED tasks[1].status == Task.Status.FAILED tasks[2].status == Task.Status.SCHEDULED tasks[2].getInputData().get("p3") == tasks[2].getTaskId() } when: "The 'integration_task_1' task is polled and is completed" def polledAndCompletedTry3 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task2.integration.worker') then: "verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTry3) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[2].status == Task.Status.COMPLETED tasks[3].status == Task.Status.SCHEDULED } when: "The 'integration_task_2' task is polled and is completed" def polledAndCompletedTaskTry1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTaskTry1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[2].status == Task.Status.COMPLETED tasks[3].status == Task.Status.COMPLETED failedReferenceTaskNames == ['t1'] as HashSet failedTaskNames == ['integration_task_1'] as HashSet } cleanup: metadataService.updateTaskDef(integrationTask1Definition) } def "Test Long running simple workflow"() { given: "A new simple workflow is started" def correlationId = 'integration_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "start a new workflow with the input" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "verify that the workflow is in running state and the task queue has an entry for the first task of the workflow" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING } workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 when: "the first task 'integration_task_1' is polled and then sent back with a callBack seconds" def pollTaskTry1 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') pollTaskTry1.outputData['op'] = 'task1.in.progress' pollTaskTry1.callbackAfterSeconds = 5 pollTaskTry1.status = Task.Status.IN_PROGRESS workflowExecutionService.updateTask(new TaskResult(pollTaskTry1)) then: "verify that the task is polled and acknowledged" pollTaskTry1 and: "the input data of the data is as expected" pollTaskTry1.inputData.containsKey('p1') pollTaskTry1.inputData['p1'] == 'p1 value' pollTaskTry1.inputData.containsKey('p2') pollTaskTry1.inputData['p1'] == 'p1 value' and: "the task queue reflects the presence of 'integration_task_1' " workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 when: "the 'integration_task_1' task is polled again" def pollTaskTry2 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') then: "verify that there was no task polled" !pollTaskTry2 when: "the 'integration_task_1' is polled again after a delay of 5 seconds and completed" Thread.sleep(5000) def task1Try3Tuple = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(task1Try3Tuple, [:]) and: "verify that the workflow is updated with the latest task" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' tasks[0].outputData['op'] == 'task1.done' } when: "the 'integration_task_1' is polled and completed" def task2Try1Tuple = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the task was polled and completed with the expected inputData for the task that was polled" verifyPolledAndAcknowledgedTask(task2Try1Tuple, ['tp2': 'task1.done', 'tp1': 'p1 value']) and: "The workflow is in a completed state and reflects the tasks that are completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' } } def "Test simple workflow when the task's call back after seconds are reset"() { given: "A new simple workflow is started" def correlationId = 'integration_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "start a new workflow with the input" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "verify that the workflow is in running state and the task queue has an entry for the first task of the workflow" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED } workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 when: "the first task 'integration_task_1' is polled and then sent back with a callBack seconds" def pollTaskTry1 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') pollTaskTry1.outputData['op'] = 'task1.in.progress' pollTaskTry1.callbackAfterSeconds = 3600 pollTaskTry1.status = Task.Status.IN_PROGRESS workflowExecutionService.updateTask(new TaskResult(pollTaskTry1)) then: "verify that the task is polled and acknowledged" pollTaskTry1 and: "the input data of the data is as expected" pollTaskTry1.inputData.containsKey('p1') pollTaskTry1.inputData['p1'] == 'p1 value' pollTaskTry1.inputData.containsKey('p2') pollTaskTry1.inputData['p1'] == 'p1 value' and: "the task queue reflects the presence of 'integration_task_1' " workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 when: "the 'integration_task_1' task is polled again" def pollTaskTry2 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') then: "verify that there was no task polled" !pollTaskTry2 when: "the 'integration_task_1' task is polled again" def pollTaskTry3 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') then: "verify that there was no task polled" !pollTaskTry3 when: "The callbackSeconds of the tasks in progress for the workflow are reset" workflowExecutor.resetCallbacksForWorkflow(workflowInstanceId) and: "the 'integration_task_1' is polled again after all the in progress tasks are reset" def task1Try4Tuple = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(task1Try4Tuple) and: "verify that the workflow is updated with the latest task" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' tasks[0].outputData['op'] == 'task1.done' } when: "the 'integration_task_1' is polled and completed" def task2Try1Tuple = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the task was polled and completed with the expected inputData for the task that was polled" verifyPolledAndAcknowledgedTask(task2Try1Tuple, ['tp2': 'task1.done', 'tp1': 'p1 value']) and: "The workflow is in a completed state and reflects the tasks that are completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' } } def "Test non restartable simple workflow"() { setup: "Change the task definition to ensure that it has no retries and register a non restartable workflow" def integrationTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTaskDefinition = new TaskDef(integrationTask1Definition.name, integrationTask1Definition.description, integrationTask1Definition.ownerEmail, 0, integrationTask1Definition.timeoutSeconds, integrationTask1Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTaskDefinition) def simpleWorkflowDefinition = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2, 1) simpleWorkflowDefinition.name = INTEGRATION_TEST_WF_NON_RESTARTABLE simpleWorkflowDefinition.restartable = false metadataService.updateWorkflowDef(simpleWorkflowDefinition) when: "A non restartable workflow is started" def correlationId = 'integration_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(INTEGRATION_TEST_WF_NON_RESTARTABLE, 1, correlationId, workflowInput, null) and: "the 'integration_task_1' is polled and failed" Tuple polledAndFailedTaskTry1 = workflowTestUtil.pollAndFailTask('integration_task_1', 'task1.integration.worker', 'failure...0') then: "verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndFailedTaskTry1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'integration_task_1' } when: "The failed workflow is rewound" workflowExecutor.restart(workflowInstanceId, false) and: "The first task 'integration_task_1' is polled and completed" def task1Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "Verify that the task is polled and acknowledged" verifyPolledAndAcknowledgedTask(task1Try2) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' } when: "The second task 'integration_task_2' is polled and completed" def task2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "Verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(task2Try1, ['tp2': 'task1.done', 'tp1': 'p1 value']) and: "The workflow is in a completed state and reflects the tasks that are completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[0].status == Task.Status.COMPLETED tasks[0].taskType == 'integration_task_1' output['o3'] == 'task1.done' } when: "The successfully completed non restartable workflow is rewound" workflowExecutor.restart(workflowInstanceId, false) then: "Ensure that an exception is thrown" thrown(NotFoundException.class) cleanup: "clean up the changes made to the task and workflow definition during start up" metadataService.updateTaskDef(integrationTask1Definition) simpleWorkflowDefinition.name = LINEAR_WORKFLOW_T1_T2 simpleWorkflowDefinition.restartable = true metadataService.updateWorkflowDef(simpleWorkflowDefinition) } def "Test simple workflow when update task's result with call back after seconds"() { given: "A new simple workflow is started" def correlationId = 'integration_test_1' def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "start a new workflow with the input" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, workflowInput, null) then: "verify that the workflow is in running state and the task queue has an entry for the first task of the workflow" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED } workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 when: "the first task 'integration_task_1' is polled and then sent back with no callBack seconds" def pollTaskTry1 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') pollTaskTry1.outputData['op'] = 'task1.in.progress' pollTaskTry1.status = Task.Status.IN_PROGRESS workflowExecutionService.updateTask(new TaskResult(pollTaskTry1)) then: "verify that the task is polled and acknowledged" pollTaskTry1 and: "the input data of the data is as expected" pollTaskTry1.inputData.containsKey('p1') pollTaskTry1.inputData['p1'] == 'p1 value' pollTaskTry1.inputData.containsKey('p2') pollTaskTry1.inputData['p1'] == 'p1 value' and: "the task gets put back into the queue of 'integration_task_1' immediately for future poll" workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 and: "The task in in SCHEDULED status with workerId reset" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].callbackAfterSeconds == 0 } when: "the 'integration_task_1' task is polled again" def pollTaskTry2 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') pollTaskTry2.outputData['op'] = 'task1.in.progress' pollTaskTry2.status = Task.Status.IN_PROGRESS pollTaskTry2.callbackAfterSeconds = 3600 workflowExecutionService.updateTask(new TaskResult(pollTaskTry2)) then: "verify that the task is polled and acknowledged" pollTaskTry2 and: "the task gets put back into the queue of 'integration_task_1' with callbackAfterSeconds delay for future poll" workflowExecutionService.getTaskQueueSizes(['integration_task_1']).get('integration_task_1') == 1 and: "The task in in SCHEDULED status with workerId reset" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED tasks[0].callbackAfterSeconds == pollTaskTry2.callbackAfterSeconds } when: "the 'integration_task_1' task is polled again" def pollTaskTry3 = workflowExecutionService.poll('integration_task_1', 'task1.integration.worker') then: "verify that there was no task polled" !pollTaskTry3 } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/StartWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.StartWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import com.netflix.conductor.test.utils.MockExternalPayloadStorage import spock.lang.Shared import spock.lang.Unroll class StartWorkflowSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired StartWorkflow startWorkflowTask @Autowired MockExternalPayloadStorage mockExternalPayloadStorage @Shared def WORKFLOW_THAT_STARTS_ANOTHER_WORKFLOW = 'workflow_that_starts_another_workflow' static String workflowInputPath = "${UUID.randomUUID()}.json" def setup() { workflowTestUtil.registerWorkflows('workflow_that_starts_another_workflow.json', 'simple_workflow_1_integration_test.json') mockExternalPayloadStorage.upload(workflowInputPath, StartWorkflowSpec.class.getResourceAsStream("/start_workflow_input.json"), 0) } @Unroll def "start another workflow using #testCase.name"() { setup: 'create the correlationId for the starter workflow' def correlationId = UUID.randomUUID().toString() when: "starter workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_THAT_STARTS_ANOTHER_WORKFLOW, 1, correlationId, testCase.workflowInput, testCase.workflowInputPath) then: "verify that the starter workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'START_WORKFLOW' tasks[0].status == Task.Status.SCHEDULED } when: "the START_WORKFLOW task is started" List polledTaskIds = queueDAO.pop("START_WORKFLOW", 1, 200) String startWorkflowTaskId = polledTaskIds.get(0) asyncSystemTaskExecutor.execute(startWorkflowTask, startWorkflowTaskId) then: "verify the START_WORKFLOW task and workflow are COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'START_WORKFLOW' tasks[0].status == Task.Status.COMPLETED } when: "the started workflow is retrieved" def startWorkflowTask = workflowExecutionService.getTask(startWorkflowTaskId) String startedWorkflowId = startWorkflowTask.outputData['workflowId'] then: "verify that the started workflow is RUNNING" with(workflowExecutionService.getExecutionStatus(startedWorkflowId, false)) { status == Workflow.WorkflowStatus.RUNNING it.correlationId == correlationId // when the "starter" workflow is started with input from external payload storage, // it sends a large input to the "started" workflow // see start_workflow_input.json if(testCase.workflowInputPath) { externalInputPayloadStoragePath != null } else { input != null } } where: testCase << [workflowName(), workflowDef(), workflowRequestWithExternalPayloadStorage()] } def "start_workflow does not conform to StartWorkflowRequest"() { given: "start_workflow that does not conform to StartWorkflowRequest" def startWorkflowParam = ['param1': 'value1', 'param2': 'value2'] def workflowInput = ['start_workflow': startWorkflowParam] when: "starter workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_THAT_STARTS_ANOTHER_WORKFLOW, 1, null, workflowInput, null) then: "verify that the starter workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'START_WORKFLOW' tasks[0].status == Task.Status.SCHEDULED } when: "the START_WORKFLOW task is started" List polledTaskIds = queueDAO.pop("START_WORKFLOW", 1, 200) String startWorkflowTaskId = polledTaskIds.get(0) asyncSystemTaskExecutor.execute(startWorkflowTask, startWorkflowTaskId) then: "verify the START_WORKFLOW task and workflow FAILED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 1 tasks[0].taskType == 'START_WORKFLOW' tasks[0].status == Task.Status.FAILED tasks[0].reasonForIncompletion != null } } /** * Builds a TestCase for a StartWorkflowRequest with a WorkflowDef that contains two tasks. */ static workflowDef() { def task1 = ['name': 'integration_task_1', 'taskReferenceName': 't1', 'type': 'SIMPLE', 'inputParameters': ['tp1': '${workflow.input.param1}', 'tp2': '${workflow.input.param2}', 'tp3': '${CPEWF_TASK_ID}']] def task2 = ['name': 'integration_task_2', 'taskReferenceName': 't2', 'type': 'SIMPLE', 'inputParameters': ['tp1': '${workflow.input.param1}', 'tp2': '${t1.output.op}', 'tp3': '${CPEWF_TASK_ID}']] def workflowDef = ['name': 'dynamic_wf', 'version': 1, 'tasks': [task1, task2], 'ownerEmail': 'abc@abc.com'] def startWorkflow = ['name': 'dynamic_wf', 'workflowDef': workflowDef] new TestCase(name: 'workflow definition', workflowInput: ['startWorkflow': startWorkflow]) } /** * Builds a TestCase for a StartWorkflowRequest with a workflow name. */ static workflowName() { def startWorkflow = ['name': 'integration_test_wf', 'input': ['param1': 'value1', 'param2': 'value2']] new TestCase(name: 'name and version', workflowInput: ['startWorkflow': startWorkflow]) } /** * Builds a TestCase for a StartWorkflowRequest with a workflow name and input in external payload storage. */ static workflowRequestWithExternalPayloadStorage() { new TestCase(name: 'name and version with external input', workflowInputPath: workflowInputPath) } static class TestCase { String name Map workflowInput String workflowInputPath } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SubWorkflowRerunSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SubWorkflowRerunSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Shared def WORKFLOW_WITH_SUBWORKFLOW = 'integration_test_wf_with_sub_wf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('simple_workflow_1_integration_test.json', 'workflow_with_sub_workflow_1_integration_test.json') //region Test setup: 3 workflows. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'rerun_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : WORKFLOW_WITH_SUBWORKFLOW, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the leaf-level workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level subworkflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } //endregion } def cleanup() { metadataService.updateTaskDef(persistedTask2Definition) } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the root workflow. * * Expectation: The root workflow spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test rerun on the root-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the root workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = rootWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "poll and complete the 'integration_task_1' task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op1': 'task1.done']) and: "verify that the root workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) then: "the new mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } then: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed with taskId on the root workflow. * * Expectation: The root workflow gets a new execution with the same id and both the mid-level workflow and leaf workflows are also reran. * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the root-level with taskId in a 3-level subworkflow"() { //region Test case when: "do a rerun on the root workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = rootWorkflowId def reRunTaskId = workflowExecutionService.getExecutionStatus(rootWorkflowId, true).tasks[0].taskId reRunWorkflowRequest.reRunFromTaskId = reRunTaskId workflowExecutor.rerun(reRunWorkflowRequest) then: "poll and complete the 'integration_task_1' task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op1': 'task1.done']) and: "verify that the root workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) then: "the new mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } then: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the mid-level workflow. * * Expectation: The mid-level workflow gets a new execution with the same id and spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the mid level workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = midLevelWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the mid workflow created a new execution" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "verify the SUB_WORKFLOW task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the task in the mid level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'integration_task_1' } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the mid-level workflow with taskId. * * Expectation: The mid-level workflow gets a new execution with the same id and spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the mid-level with taskId in a 3-level subworkflow"() { //region Test case when: "do a rerun on the mid level workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = midLevelWorkflowId def reRunTaskId = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true).tasks[0].taskId reRunWorkflowRequest.reRunFromTaskId = reRunTaskId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the mid workflow created a new execution" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "verify the SUB_WORKFLOW task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the task in the mid level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'integration_task_1' } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the leaf workflow. * * Expectation: The leaf workflow gets a new execution with the same id and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the leaf-level in a 3-level subworkflow"() { //region Test case when: "do a rerun on the leaf workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = leafWorkflowId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the leaf workflow creates a new execution" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the root workflow's SUB_WORKFLOW is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A rerun is executed on the leaf workflow. * * Expectation: The leaf workflow gets a new execution with the same id and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test rerun on the leaf-level with taskId in a 3-level subworkflow"() { //region Test case when: "do a rerun on the leaf workflow" def reRunWorkflowRequest = new RerunWorkflowRequest() reRunWorkflowRequest.reRunFromWorkflowId = leafWorkflowId def reRunTaskId = workflowExecutionService.getExecutionStatus(leafWorkflowId, true).tasks[0].taskId reRunWorkflowRequest.reRunFromTaskId = reRunTaskId workflowExecutor.rerun(reRunWorkflowRequest) then: "verify that the leaf workflow creates a new execution" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the root workflow's SUB_WORKFLOW is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SubWorkflowRestartSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SubWorkflowRestartSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Shared def WORKFLOW_WITH_SUBWORKFLOW = 'integration_test_wf_with_sub_wf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('simple_one_task_sub_workflow_integration_test.json', 'simple_workflow_1_integration_test.json', 'workflow_with_sub_workflow_1_integration_test.json') //region Test setup: 3 workflows reach FAILED state. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'retry_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : WORKFLOW_WITH_SUBWORKFLOW, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level subworkflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } //endregion } def cleanup() { // Ensure that changes to the task def are reverted metadataService.updateTaskDef(persistedTask2Definition) } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the root workflow. * * Expectation: The root workflow gets a new execution with the same id and spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the NEW leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test restart on the root in a 3-level subworkflow"() { //region Test case when: "do a restart on the root workflow" workflowExecutor.restart(rootWorkflowId, false) then: "poll and complete the 'integration_task_1' task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op1': 'task1.done']) and: "verify that the root workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.SCHEDULED } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) then: "the new mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } then: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the mid-level workflow. * * Expectation: The mid-level workflow gets a new execution with the same id and spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test restart on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a restart on the mid level workflow" workflowExecutor.restart(midLevelWorkflowId, false) then: "verify that the mid workflow created a new execution" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "verify the SUB_WORKFLOW task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the task in the mid level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'integration_task_1' } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A restart is executed on the leaf workflow. * * Expectation: The leaf workflow gets a new execution with the same id and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test restart on the leaf in a 3-level subworkflow"() { //region Test case when: "do a restart on the leaf workflow" workflowExecutor.restart(leafWorkflowId, false) then: "verify that the leaf workflow creates a new execution" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the root workflow's SUB_WORKFLOW is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SubWorkflowRetrySpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SubWorkflowRetrySpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Shared def WORKFLOW_WITH_SUBWORKFLOW = 'integration_test_wf_with_sub_wf' @Shared def SIMPLE_WORKFLOW = "integration_test_wf" String rootWorkflowId, midLevelWorkflowId, leafWorkflowId TaskDef persistedTask2Definition def setup() { workflowTestUtil.registerWorkflows('simple_one_task_sub_workflow_integration_test.json', 'simple_workflow_1_integration_test.json', 'workflow_with_sub_workflow_1_integration_test.json') //region Test setup: 3 workflows reach FAILED state. Task 'integration_task_2' in leaf workflow is FAILED. setup: "Modify task definition to 0 retries" persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'retry_on_root_in_3level_wf' def input = [ 'param1' : 'p1 value', 'param2' : 'p2 value', 'subwf' : WORKFLOW_WITH_SUBWORKFLOW, 'nextSubwf': SIMPLE_WORKFLOW] when: "the workflow is started" rootWorkflowId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" def rootWorkflowInstance = workflowExecutionService.getExecutionStatus(rootWorkflowId, true) with(rootWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS } and: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" midLevelWorkflowId = rootWorkflowInstance.tasks[1].subWorkflowId with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def midLevelWorkflowInstance = workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true) then: "verify that the mid-level workflow is RUNNING, and first task is in SCHEDULED state" leafWorkflowId = midLevelWorkflowInstance.tasks[1].subWorkflowId def leafWorkflowInstance = workflowExecutionService.getExecutionStatus(leafWorkflowId, true) with(leafWorkflowInstance) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "the leaf workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } when: "the mid level workflow is 'decided'" sweep(midLevelWorkflowId) then: "the mid level subworkflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } when: "the root level workflow is 'decided'" sweep(rootWorkflowId) then: "the root level workflow is in FAILED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the root workflow. * * Expectation: The root workflow spawns a NEW mid-level workflow, which in turn spawns a NEW leaf workflow. * When the leaf workflow completes successfully, both the NEW mid-level and root workflows also complete successfully. */ def "Test retry on the root in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(rootWorkflowId, false) then: "poll and complete the 'integration_task_1' task" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op1': 'task1.done']) and: "verify that the root workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } when: "the subworkflow task should be in SCHEDULED state and is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newMidLevelWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new mid level workflow is created and is in RUNNING state" newMidLevelWorkflowId != midLevelWorkflowId with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task in the mid-level workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) and: "poll and execute the sub workflow task" polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the two tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "the new leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the new mid level and root workflows are 'decided'" sweep(newMidLevelWorkflowId) sweep(rootWorkflowId) then: "the new mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(newMidLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } then: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the root workflow. * * Expectation: The leaf workflow is retried and both its parent (mid-level) and grand parent (root) workflows are also retried. * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the root with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(rootWorkflowId, true) then: "verify that the sub workflow task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the sub workflow task in mid level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the previously failed task in leaf workflow is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "the mid level workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after "decide" } and: "the root workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after "decide" } when: "poll and complete the integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "the new mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } and: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the mid-level workflow. * * Expectation: The mid-level workflow spawns a NEW leaf workflow and also updates its parent (root workflow). * When the NEW leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the mid-level in a 3-level subworkflow"() { //region Test case when: "do a retry on the mid level workflow" workflowExecutor.retry(midLevelWorkflowId, false) then: "verify that the mid workflow created a new SUB_WORKFLOW task" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } and: "verify the SUB_WORKFLOW task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "the SUB_WORKFLOW task in mid level workflow is started by issuing a system task call" def polledTaskIds = queueDAO.pop(TASK_TYPE_SUB_WORKFLOW, 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def newLeafWorkflowId = workflowExecutionService.getTask(polledTaskIds[0]).subWorkflowId then: "verify that a new leaf workflow is created and is in RUNNING state" newLeafWorkflowId != leafWorkflowId with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 2 tasks in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the new leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(newLeafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == TASK_TYPE_SUB_WORKFLOW tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the mid-level workflow. * * Expectation: The leaf workflow is retried and both its parent (mid-level) and grand parent (root) workflows are also retried. * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the mid-level with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the root workflow" workflowExecutor.retry(midLevelWorkflowId, true) then: "verify that the sub workflow task in root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the sub workflow task in mid level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the previously failed task in leaf workflow is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "the mid level workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after "decide" } and: "the root workflow is in RUNNING state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS !tasks[1].subworkflowChanged // flag is reset after "decide" } when: "poll and complete the previously failed integration_task_2 task" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "the mid level workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide } and: "the root workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed on the leaf workflow. * * Expectation: The leaf workflow resumes its FAILED task and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the leaf in a 3-level subworkflow"() { //region Test case when: "do a retry on the leaf workflow" workflowExecutor.retry(leafWorkflowId, false) then: "verify that the leaf workflow is in RUNNING state and failed task is retried" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the root workflow' is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } /** * On a 3-level workflow where all workflows reach FAILED state because of a FAILED task * in the leaf workflow. * * A retry is executed with resume flag on the leaf workflow. * * Expectation: The leaf workflow resumes its FAILED task and updates both its parent (mid-level) and grandparent (root). * When the leaf workflow completes successfully, both the mid-level and root workflows also complete successfully. */ def "Test retry on the leaf with resume flag in a 3-level subworkflow"() { //region Test case when: "do a retry on the leaf workflow" workflowExecutor.retry(leafWorkflowId, true) then: "verify that the leaf workflow is in RUNNING state and failed task is retried" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED tasks[2].retriedTaskId == tasks[1].taskId } then: "verify that the mid-level workflow is updated" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } and: "verify that the root workflow is updated" with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the scheduled task in the leaf workflow" workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the leaf workflow reached COMPLETED state" with(workflowExecutionService.getExecutionStatus(leafWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[1].retried tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[2].retriedTaskId == tasks[1].taskId } when: "the mid level and root workflows are 'decided'" sweep(midLevelWorkflowId) sweep(rootWorkflowId) then: "verify that the mid level and root workflows reach COMPLETED state" with(workflowExecutionService.getExecutionStatus(midLevelWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } with(workflowExecutionService.getExecutionStatus(rootWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED (!tasks[1].subworkflowChanged) // flag is reset after decide } //endregion } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SubWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.SubWorkflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SubWorkflowSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired SubWorkflow subWorkflowTask @Shared def WORKFLOW_WITH_SUBWORKFLOW = 'integration_test_wf_with_sub_wf' @Shared def SUB_WORKFLOW = "sub_workflow" @Shared def SIMPLE_WORKFLOW = "integration_test_wf" def setup() { workflowTestUtil.registerWorkflows('simple_one_task_sub_workflow_integration_test.json', 'simple_workflow_1_integration_test.json', 'workflow_with_sub_workflow_1_integration_test.json') } def "Test retrying a subworkflow where parent workflow timed out due to workflowTimeout"() { setup: "Register a workflow definition with a timeout policy set to timeout workflow" def persistedWorkflowDefinition = metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) def modifiedWorkflowDefinition = new WorkflowDef() modifiedWorkflowDefinition.name = persistedWorkflowDefinition.name modifiedWorkflowDefinition.version = persistedWorkflowDefinition.version modifiedWorkflowDefinition.tasks = persistedWorkflowDefinition.tasks modifiedWorkflowDefinition.inputParameters = persistedWorkflowDefinition.inputParameters modifiedWorkflowDefinition.outputParameters = persistedWorkflowDefinition.outputParameters modifiedWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.TIME_OUT_WF modifiedWorkflowDefinition.timeoutSeconds = 10 modifiedWorkflowDefinition.ownerEmail = persistedWorkflowDefinition.ownerEmail metadataService.updateWorkflowDef([modifiedWorkflowDefinition]) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SUB_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'wf_with_subwf_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' input['subwf'] = 'sub_workflow' when: "the workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) and: "verify that the 'integration_task1' is complete and the next task (subworkflow) is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.SCHEDULED } when: "the subworkflow is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) String subworkflowTaskId = polledTaskIds.get(0) asyncSystemTaskExecutor.execute(subWorkflowTask, subworkflowTaskId) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.IN_PROGRESS } when: "subworkflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.tasks[1].subWorkflowId then: "verify that the sub workflow is RUNNING, and first task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.SCHEDULED } when: "a delay of 10 seconds is introduced and the workflow is sweeped to run the evaluation" Thread.sleep(10000) sweep(workflowInstanceId) then: "ensure that the workflow has been TIMED OUT and subworkflow task is CANCELED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TIMED_OUT tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.CANCELED } and: "ensure that the subworkflow is TERMINATED and task is CANCELED" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.CANCELED } when: "the subworkflow is retried" workflowExecutor.retry(subWorkflowId, false) then: "ensure that the subworkflow is RUNNING and task is retried" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.CANCELED tasks[1].taskType == 'simple_task_in_sub_wf' tasks[1].status == Task.Status.SCHEDULED } and: "verify that change flag is set on the sub workflow task in parent" workflowExecutionService.getTask(subworkflowTaskId).subworkflowChanged when: "Polled for simple_task_in_sub_wf task in subworkflow" pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('simple_task_in_sub_wf', 'task1.integration.worker', ['op': 'simple_task_in_sub_wf.done']) then: "verify that the 'simple_task_in_sub_wf' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) and: "verify that the subworkflow is in a completed state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.CANCELED tasks[1].taskType == 'simple_task_in_sub_wf' tasks[1].status == Task.Status.COMPLETED } and: "subworkflow task is in a completed state" with(workflowExecutionService.getTask(subworkflowTaskId)) { status == Task.Status.COMPLETED subworkflowChanged } and: "the parent workflow is swept" sweep(workflowInstanceId) and: "the parent workflow has been resumed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged output['op'] == 'simple_task_in_sub_wf.done' } cleanup: "Ensure that the changes to the workflow def are reverted" metadataService.updateWorkflowDef([persistedWorkflowDefinition]) } def "Test terminating a subworkflow terminates parent workflow"() { given: "Existing workflow and subworkflow definitions" metadataService.getWorkflowDef(SUB_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'wf_with_subwf_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' input['subwf'] = 'sub_workflow' when: "Start a workflow with subworkflow based on the registered definition" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Polled for integration_task_1 task" def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the 'integration_task1' is complete and the next task (subworkflow) is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.SCHEDULED } when: "Polled for and executed subworkflow task" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.tasks[1].subWorkflowId then: "verify that the 'sub_workflow_task' is polled and IN_PROGRESS" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.IN_PROGRESS } and: "verify that the sub workflow is RUNNING, and first task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.SCHEDULED } when: "subworkflow is terminated" def terminateReason = "terminating from a test case" workflowExecutor.terminateWorkflow(subWorkflowId, terminateReason) then: "verify that sub workflow is in terminated state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'simple_task_in_sub_wf' tasks[0].status == Task.Status.CANCELED reasonForIncompletion == terminateReason } and: sweep(workflowInstanceId) and: "verify that parent workflow is in terminated state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.CANCELED reasonForIncompletion && reasonForIncompletion.contains(terminateReason) } } def "Test retrying a workflow with subworkflow resume"() { setup: "Modify task definition to 0 retries" def persistedTask2Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_2').get() def modifiedTask2Definition = new TaskDef(persistedTask2Definition.name, persistedTask2Definition.description, persistedTask2Definition.ownerEmail, 0, persistedTask2Definition.timeoutSeconds, persistedTask2Definition.responseTimeoutSeconds) metadataService.updateTaskDef(modifiedTask2Definition) and: "an existing workflow with subworkflow and registered definitions" metadataService.getWorkflowDef(SIMPLE_WORKFLOW, 1) metadataService.getWorkflowDef(WORKFLOW_WITH_SUBWORKFLOW, 1) and: "input required to start the workflow execution" String correlationId = 'wf_retry_with_subwf_resume_test' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' input['subwf'] = 'integration_test_wf' when: "the workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_SUBWORKFLOW, 1, correlationId, input, null) then: "verify that the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) and: "verify that the 'integration_task_1' is complete and the next task (subworkflow) is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.SCHEDULED } when: "the subworkflow is started by issuing a system task call" List polledTaskIds = queueDAO.pop("SUB_WORKFLOW", 1, 200) asyncSystemTaskExecutor.execute(subWorkflowTask, polledTaskIds[0]) then: "verify that the 'sub_workflow_task' is in a IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TaskType.SUB_WORKFLOW.name() tasks[1].status == Task.Status.IN_PROGRESS } when: "subworkflow is retrieved" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) def subWorkflowId = workflow.tasks[1].subWorkflowId then: "verify that the sub workflow is RUNNING, and first task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) and: "verify that the 'integration_task_1' is complete and the next task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and fail the integration_task_2 task" def pollAndFailTask = workflowTestUtil.pollAndFailTask('integration_task_2', 'task2.integration.worker', 'failed') then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndFailTask) then: "the sub workflow ends up in a FAILED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED } and: sweep(workflowInstanceId) and: "the workflow is in a FAILED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.FAILED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SUB_WORKFLOW' tasks[1].status == Task.Status.FAILED } when: "the workflow is retried by resuming subworkflow task" workflowExecutor.retry(workflowInstanceId, true) then: "the subworkflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } and: "the workflow is in a RUNNING state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.IN_PROGRESS tasks[1].subworkflowChanged } when: "poll and complete the integration_task_2 task" pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker', ['op': 'task2.done']) then: "verify that the 'integration_task_2' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) then: "the integration_task_2 is complete sub workflow ends up in a COMPLETED state" with(workflowExecutionService.getExecutionStatus(subWorkflowId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.FAILED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED } and: sweep(workflowInstanceId) then: "the workflow is in a COMPLETED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == TASK_TYPE_SUB_WORKFLOW tasks[1].status == Task.Status.COMPLETED !tasks[1].subworkflowChanged } cleanup: "Ensure that changes to the task def are reverted" metadataService.updateTaskDef(persistedTask2Definition) } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SwitchTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.tasks.Join import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import spock.lang.Unroll import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SwitchTaskSpec extends AbstractSpecification { @Autowired Join joinTask @Shared def SWITCH_WF = "SwitchWorkflow" @Shared def FORK_JOIN_SWITCH_WF = "ForkConditionalTest" @Shared def COND_TASK_WF = "ConditionalTaskWF" @Shared def SWITCH_NODEFAULT_WF = "SwitchWithNoDefaultCaseWF" def setup() { //initialization code for each feature workflowTestUtil.registerWorkflows('simple_switch_task_integration_test.json', 'switch_and_fork_join_integration_test.json', 'conditional_switch_task_workflow_integration_test.json', 'switch_with_no_default_case_integration_test.json') } def "Test simple switch workflow"() { given: "Workflow an input of a workflow with switch task" Map input = new HashMap() input['param1'] = 'p1' input['param2'] = 'p2' input['case'] = 'c' when: "A switch workflow is started with the workflow input" def workflowInstanceId = startWorkflow(SWITCH_WF, 1, 'switch_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_1' is polled and completed" def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) and: "verify that the 'integration_task_1' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[2].taskType == 'integration_task_2' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_20' tasks[3].status == Task.Status.SCHEDULED } when: "the task 'integration_task_20' is polled and completed" def polledAndCompletedTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask20Try1) and: "verify that the 'integration_task_20' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[3].taskType == 'integration_task_20' tasks[3].status == Task.Status.COMPLETED } } def "Test a workflow that has a switch task that leads to a fork join"() { given: "Workflow an input of a workflow with switch task" Map input = new HashMap() input['param1'] = 'p1' input['param2'] = 'p2' input['case'] = 'c' when: "A switch workflow is started with the workflow input" def workflowInstanceId = startWorkflow(FORK_JOIN_SWITCH_WF, 1, 'switch_forkjoin', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 5 tasks[0].taskType == 'FORK' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SWITCH' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.SCHEDULED tasks[4].taskType == 'JOIN' tasks[4].status == Task.Status.IN_PROGRESS } when: "the tasks 'integration_task_1' and 'integration_task_10' are polled and completed" def joinTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName("joinTask").taskId def polledAndCompletedTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') def polledAndCompletedTask10Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_10', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask1Try1) verifyPolledAndAcknowledgedTask(polledAndCompletedTask10Try1) and: "verify that the 'integration_task_1' and 'integration_task_10' are COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 6 tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED tasks[3].taskType == 'integration_task_10' tasks[3].status == Task.Status.COMPLETED tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask2Try1) and: "verify that the 'integration_task_2' is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 7 tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.IN_PROGRESS tasks[5].taskType == 'integration_task_2' tasks[5].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_20' tasks[6].status == Task.Status.SCHEDULED } when: "the task 'integration_task_20' is polled and completed" def polledAndCompletedTask20Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_20', 'task1.integration.worker') and: "the workflow is evaluated" sweep(workflowInstanceId) then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask20Try1) when: "JOIN task is polled and executed" asyncSystemTaskExecutor.execute(joinTask, joinTaskId) then: "verify that the JOIN is COMPLETED and the workflow has progressed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 7 tasks[4].taskType == 'JOIN' tasks[4].inputData['joinOn'] == ['t20', 't10'] tasks[4].status == Task.Status.COMPLETED tasks[6].taskType == 'integration_task_20' tasks[6].status == Task.Status.COMPLETED } } def "Test default case condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the default case is executed" Map input = new HashMap() input['param1'] = 'xxx' input['param2'] = 'two' when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, 'conditional_default', input, null) then: "verify that the workflow is running and the default condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['evaluationResult'] == ['xxx'] tasks[1].taskType == 'integration_task_10' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_10' is polled and completed" def polledAndCompletedTask10Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_10', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask10Try1) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[1].taskType == 'integration_task_10' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['evaluationResult'] == ['null'] } } @Unroll def "Test case 'nested' and '#caseValue' condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the 'nested' and '#caseValue' switch tree is executed" Map input = new HashMap() input['param1'] = 'nested' input['param2'] = caseValue when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, workflowCorrelationId, input, null) then: "verify that the workflow is running and the 'nested' and '#caseValue' condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['evaluationResult'] == ['nested'] tasks[1].taskType == 'SWITCH' tasks[1].status == Task.Status.COMPLETED tasks[1].outputData['evaluationResult'] == [caseValue] tasks[2].taskType == expectedTaskName tasks[2].status == Task.Status.SCHEDULED } when: "the task '#expectedTaskName' is polled and completed" def polledAndCompletedTaskTry1 = workflowTestUtil.pollAndCompleteTask(expectedTaskName, 'task.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTaskTry1) and: with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[2].taskType == expectedTaskName tasks[2].status == endTaskStatus tasks[3].taskType == 'SWITCH' tasks[3].status == Task.Status.COMPLETED tasks[3].outputData['evaluationResult'] == ['null'] } where: caseValue | expectedTaskName | workflowCorrelationId || endTaskStatus 'two' | 'integration_task_2' | 'conditional_nested_two' || Task.Status.COMPLETED 'one' | 'integration_task_1' | 'conditional_nested_one' || Task.Status.COMPLETED } def "Test 'three' case condition execution of a conditional workflow"() { given: "input for a workflow to ensure that the default case is executed" Map input = new HashMap() input['param1'] = 'three' input['param2'] = 'two' input['finalCase'] = 'notify' when: "A conditional workflow is started with the workflow input" def workflowInstanceId = startWorkflow(COND_TASK_WF, 1, 'conditional_three', input, null) then: "verify that the workflow is running and the 'three' condition case was executed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData['evaluationResult'] == ['three'] tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_3' is polled and completed" def polledAndCompletedTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask3Try1) and: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 4 tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['evaluationResult'] == ['notify'] tasks[3].taskType == 'integration_task_4' tasks[3].status == Task.Status.SCHEDULED } when: "the task 'integration_task_4' is polled and completed" def polledAndCompletedTask4Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_4', 'task1.integration.worker') then: "verify that the tasks are completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask4Try1) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 4 tasks[1].taskType == 'integration_task_3' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'SWITCH' tasks[2].status == Task.Status.COMPLETED tasks[2].outputData['evaluationResult'] == ['notify'] tasks[3].taskType == 'integration_task_4' tasks[3].status == Task.Status.COMPLETED } } def "Test switch with no default case workflow"() { given: "Workflow input" Map input = new HashMap() input['param1'] = 'p1' input['param2'] = 'p2' when: "A switch workflow is started with the workflow input" def workflowInstanceId = startWorkflow(SWITCH_NODEFAULT_WF, 1, 'switch_no_default_workflow', input, null) then: "verify that the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "the task 'integration_task_2' is polled and completed" def polledAndCompletedTaskTry = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the task is completed and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTaskTry) and: "verify that the 'integration_task_2' is COMPLETED and the workflow is in COMPLETED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'SWITCH' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/SystemTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import com.netflix.conductor.test.utils.UserTask import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class SystemTaskSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired UserTask userTask @Shared def ASYNC_COMPLETE_SYSTEM_TASK_WORKFLOW = 'async_complete_integration_test_wf' def setup() { workflowTestUtil.registerWorkflows('simple_workflow_with_async_complete_system_task_integration_test.json') } def "Test system task with asyncComplete set to true"() { given: "An existing workflow definition with async complete system task" metadataService.getWorkflowDef(ASYNC_COMPLETE_SYSTEM_TASK_WORKFLOW, 1) and: "input required to start the workflow" String correlationId = 'async_complete_test' + UUID.randomUUID() def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' when: "the workflow is started" def workflowInstanceId = startWorkflow(ASYNC_COMPLETE_SYSTEM_TASK_WORKFLOW, 1, correlationId, input, null) then: "ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the integration_task_1 task" def pollAndCompleteTask = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask) and: "verify that the 'integration_task1' is complete and the next task is in SCHEDULED state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'USER_TASK' tasks[1].status == Task.Status.SCHEDULED } when: "the system task is started by issuing a system task call" List polledTaskIds = queueDAO.pop("USER_TASK", 1, 200) asyncSystemTaskExecutor.execute(userTask, polledTaskIds[0]) then: "verify that the system task is in IN_PROGRESS state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == "USER_TASK" tasks[1].status == Task.Status.IN_PROGRESS } when: "sweeper evaluates the workflow" sweep(workflowInstanceId) then: "workflow state is unchanged" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == "USER_TASK" tasks[1].status == Task.Status.IN_PROGRESS } when: "result of the user task is curated" Task task = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).getTaskByRefName('user_task') def taskResult = new TaskResult(task) taskResult.status = TaskResult.Status.COMPLETED taskResult.outputData['op'] = 'user.task.done' and: "external signal is simulated with this output to complete the system task" workflowExecutor.updateTask(taskResult) then: "ensure that the system task is COMPLETED and workflow is COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'USER_TASK' tasks[1].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/TaskLimitsWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import com.netflix.conductor.test.utils.UserTask import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class TaskLimitsWorkflowSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Autowired UserTask userTask def RATE_LIMITED_SYSTEM_TASK_WORKFLOW = 'test_rate_limit_system_task_workflow' def RATE_LIMITED_SIMPLE_TASK_WORKFLOW = 'test_rate_limit_simple_task_workflow' def CONCURRENCY_EXECUTION_LIMITED_WORKFLOW = 'test_concurrency_limits_workflow' def setup() { workflowTestUtil.registerWorkflows( 'rate_limited_system_task_workflow_integration_test.json', 'rate_limited_simple_task_workflow_integration_test.json', 'concurrency_limited_task_workflow_integration_test.json' ) } def "Verify that the rate limiting for system tasks is honored"() { when: "Start a workflow that has a rate limited system task in it" def workflowInstanceId = startWorkflow(RATE_LIMITED_SYSTEM_TASK_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'USER_TASK' tasks[0].status == Task.Status.SCHEDULED } when: "Execute the user task" def scheduledTask1 = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[0] asyncSystemTaskExecutor.execute(userTask, scheduledTask1.taskId) then: "Verify the state of the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'USER_TASK' tasks[0].status == Task.Status.COMPLETED } when: "A new instance of the workflow is started" def workflowTwoInstanceId = startWorkflow(RATE_LIMITED_SYSTEM_TASK_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'USER_TASK' tasks[0].status == Task.Status.SCHEDULED } when: "Execute the user task on the second workflow" def scheduledTask2 = workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true).tasks[0] asyncSystemTaskExecutor.execute(userTask, scheduledTask2.taskId) then: "Verify the state of the workflow is still in running state" with(workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'USER_TASK' tasks[0].status == Task.Status.SCHEDULED } } def "Verify that the rate limiting for simple tasks is honored"() { when: "Start a workflow that has a rate limited simple task in it" def workflowInstanceId = startWorkflow(RATE_LIMITED_SIMPLE_TASK_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'test_simple_task_with_rateLimits' tasks[0].status == Task.Status.SCHEDULED } when: "polling and completing the task" Tuple polledAndCompletedTask = workflowTestUtil.pollAndCompleteTask('test_simple_task_with_rateLimits', 'rate.limit.test.worker') then: "verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask) and: "the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'test_simple_task_with_rateLimits' tasks[0].status == Task.Status.COMPLETED } when: "A new instance of the workflow is started" def workflowTwoInstanceId = startWorkflow(RATE_LIMITED_SIMPLE_TASK_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'test_simple_task_with_rateLimits' tasks[0].status == Task.Status.SCHEDULED } when: "polling for the task" def polledTask = workflowExecutionService.poll('test_simple_task_with_rateLimits', 'rate.limit.test.worker') then: "verify that no task is returned" !polledTask when: "sleep for 10 seconds to ensure rate limit duration is past" Thread.sleep(10000L) and: "the task offset time is reset to ensure that a task is returned on the next poll" queueDAO.resetOffsetTime('test_simple_task_with_rateLimits', workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true).tasks[0].taskId) and: "polling and completing the task" polledAndCompletedTask = workflowTestUtil.pollAndCompleteTask('test_simple_task_with_rateLimits', 'rate.limit.test.worker') then: "verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndCompletedTask) and: "the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'test_simple_task_with_rateLimits' tasks[0].status == Task.Status.COMPLETED } } def "Verify that concurrency limited tasks are honored during workflow execution"() { when: "Start a workflow that has a concurrency execution limited task in it" def workflowInstanceId = startWorkflow(CONCURRENCY_EXECUTION_LIMITED_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'test_task_with_concurrency_limit' tasks[0].status == Task.Status.SCHEDULED } when: "The task is polled and acknowledged" def polledTask1 = workflowExecutionService.poll('test_task_with_concurrency_limit', 'test_task_worker') then: "Verify that the task was polled and acknowledged" polledTask1.taskType == 'test_task_with_concurrency_limit' polledTask1.workflowInstanceId == workflowInstanceId when: "A additional workflow that has a concurrency execution limited task in it" def workflowTwoInstanceId = startWorkflow(CONCURRENCY_EXECUTION_LIMITED_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'test_task_with_concurrency_limit' tasks[0].status == Task.Status.SCHEDULED } when: "The task is polled" def polledTaskTry1 = workflowExecutionService.poll('test_task_with_concurrency_limit', 'test_task_worker') then: "Verify that there is no task returned" !polledTaskTry1 when: "The task that was polled and acknowledged is completed" polledTask1.status = Task.Status.COMPLETED workflowExecutionService.updateTask(new TaskResult(polledTask1)) and: "The task offset time is reset to ensure that a task is returned on the next poll" queueDAO.resetOffsetTime('test_task_with_concurrency_limit', workflowExecutionService.getExecutionStatus(workflowTwoInstanceId, true).tasks[0].taskId) then: "Verify that the first workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 1 tasks[0].taskType == 'test_task_with_concurrency_limit' tasks[0].status == Task.Status.COMPLETED } and: "The task is polled again and acknowledged" def polledTaskTry2 = workflowExecutionService.poll('test_task_with_concurrency_limit', 'test_task_worker') then: "Verify that the task is returned since there are no tasks in progress" polledTaskTry2.taskType == 'test_task_with_concurrency_limit' polledTaskTry2.workflowInstanceId == workflowTwoInstanceId } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/TestWorkflowSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import java.util.concurrent.ArrayBlockingQueue import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.common.run.WorkflowTestRequest import com.netflix.conductor.core.operation.StartWorkflowOperation import com.netflix.conductor.service.WorkflowTestService import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedLargePayloadTask import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class TestWorkflowSpec extends AbstractSpecification { @Autowired WorkflowTestService workflowTestService def "Run Workflow Test with simple tasks"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['var'] = "var_test_value" WorkflowTestRequest request = new WorkflowTestRequest(); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test_workflow"); workflowDef.setVersion(1); workflowDef.setOwnerEmail("owner@example.com"); WorkflowTask task1 = new WorkflowTask(); task1.setType(TaskType.TASK_TYPE_SIMPLE); task1.setName("task1"); task1.setTaskReferenceName("task1"); WorkflowTask task2 = new WorkflowTask(); task2.setType(TaskType.TASK_TYPE_SIMPLE); task2.setName("task2"); task2.setTaskReferenceName("task2"); workflowDef.getTasks().add(task1); workflowDef.getTasks().add(task2); request.setName(workflowDef.getName()); request.setVersion(workflowDef.getVersion()); Queue task1Executions = new LinkedList<>(); task1Executions.add(new WorkflowTestRequest.TaskMock(TaskResult.Status.COMPLETED, Map.of("key", "value"))); request.getTaskRefToMockOutput().put("task1", task1Executions); request.setWorkflowDef(workflowDef); when: "Start the workflow which has the set variable task" def workflow = workflowTestService.testWorkflow(request) then: "verify that the simple task is scheduled" with(workflowExecutionService.getExecutionStatus(workflow.getWorkflowId(), true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'task1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData["key"] == "value" tasks[1].taskType == 'task2' tasks[1].status == Task.Status.SCHEDULED } } def "Run Workflow Test with decision task"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['var'] = "var_test_value" WorkflowTestRequest request = new WorkflowTestRequest(); WorkflowDef workflowDef = new WorkflowDef(); workflowDef.setName("test_workflow"); workflowDef.setVersion(1); workflowDef.setOwnerEmail("owner@example.com"); WorkflowTask task1 = new WorkflowTask(); task1.setType(TaskType.TASK_TYPE_SIMPLE); task1.setName("task1"); task1.setTaskReferenceName("task1"); WorkflowTask decision = new WorkflowTask(); decision.setType(TaskType.TASK_TYPE_SWITCH); decision.setName("switch"); decision.setTaskReferenceName("switch"); decision.setEvaluatorType("value-param") decision.setExpression("switchCaseValue") decision.getInputParameters().put("switchCaseValue", "\${workflow.input.case}") WorkflowTask d1 = new WorkflowTask(); d1.setType(TaskType.TASK_TYPE_SIMPLE); d1.setName("task1"); d1.setTaskReferenceName("d1"); WorkflowTask d2 = new WorkflowTask(); d2.setType(TaskType.TASK_TYPE_SIMPLE); d2.setName("task2"); d2.setTaskReferenceName("d2"); decision.getDecisionCases().put("a", Arrays.asList(d1)); decision.getDecisionCases().put("b", Arrays.asList(d2)); workflowDef.getTasks().add(task1); workflowDef.getTasks().add(decision); request.setName(workflowDef.getName()); request.setVersion(workflowDef.getVersion()); Queue task1Executions = new LinkedList<>(); task1Executions.add(new WorkflowTestRequest.TaskMock(TaskResult.Status.COMPLETED, Map.of("key", "value"))); request.getTaskRefToMockOutput().put("task1", task1Executions); request.setWorkflowDef(workflowDef); request.setInput(Map.of("case", "b")); when: "Start the workflow which has the set variable task" def workflow = workflowTestService.testWorkflow(request) then: "verify that the simple task is scheduled" with(workflowExecutionService.getExecutionStatus(workflow.getWorkflowId(), true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'task1' tasks[0].status == Task.Status.COMPLETED tasks[0].outputData["key"] == "value" tasks[1].taskType == 'SWITCH' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'task2' tasks[2].referenceTaskName == 'd2' tasks[2].status == Task.Status.SCHEDULED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/WaitTaskSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedLargePayloadTask import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class WaitTaskSpec extends AbstractSpecification { @Shared def WAIT_BASED_WORKFLOW = 'test_wait_workflow' def SET_VARIABLE_WORKFLOW = 'set_variable_workflow_integration_test' def setup() { workflowTestUtil.registerWorkflows('wait_workflow_integration_test.json', 'set_variable_workflow_integration_test.json') } def "Test workflow with set variable task"() { given: "workflow input" def workflowInput = new HashMap() workflowInput['var'] = "var_test_value" when: "Start the workflow which has the set variable task" def workflowInstanceId = startWorkflow(SET_VARIABLE_WORKFLOW, 1, '', workflowInput, null) then: "verify that the simple task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'simple' tasks[0].status == Task.Status.SCHEDULED } when: "poll and complete the 'simple' with external payload storage" def pollAndCompleteLargePayloadTask = workflowTestUtil.pollAndCompleteTask('simple', 'simple.worker', ['ok1': 'ov1']) then: "verify that the 'simple' was polled and acknowledged" verifyPolledAndAcknowledgedLargePayloadTask(pollAndCompleteLargePayloadTask) then: "ensure that the wait task is completed and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[0].taskType == 'simple' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SET_VARIABLE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'WAIT' tasks[2].status == Task.Status.IN_PROGRESS variables as String == '[var:var_test_value]' } when: "The wait task is completed" def waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[2] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) then: "ensure that the wait task is completed and the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[0].taskType == 'simple' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'SET_VARIABLE' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'WAIT' tasks[2].status == Task.Status.COMPLETED variables as String == '[var:var_test_value]' output as String == '[variables:[var:var_test_value]]' } } def "Verify that a wait based simple workflow is executed"() { when: "Start a wait task based workflow" def workflowInstanceId = startWorkflow(WAIT_BASED_WORKFLOW, 1, '', [:], null) then: "Retrieve the workflow" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == TaskType.WAIT.name() tasks[0].status == Task.Status.IN_PROGRESS } when: "The wait task is completed" def waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[0] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) then: "ensure that the wait task is completed and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == TaskType.WAIT.name() tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "The integration_task_1 is polled and completed" def polledAndCompletedTry1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "verify that the task was polled and completed and the workflow is in a complete state" verifyPolledAndAcknowledgedTask(polledAndCompletedTry1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.tasks.TaskType import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.common.metadata.workflow.WorkflowTask import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.utils.Utils import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.test.base.AbstractSpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class WorkflowAndTaskConfigurationSpec extends AbstractSpecification { @Autowired QueueDAO queueDAO @Shared def LINEAR_WORKFLOW_T1_T2 = 'integration_test_wf' @Shared def TEMPLATED_LINEAR_WORKFLOW = 'integration_test_template_wf' @Shared def WORKFLOW_WITH_OPTIONAL_TASK = 'optional_task_wf' @Shared def TEST_WORKFLOW = 'integration_test_wf3' @Shared def WAIT_TIME_OUT_WORKFLOW = 'test_wait_timeout' def setup() { //Register LINEAR_WORKFLOW_T1_T2, TEST_WORKFLOW, RTOWF, WORKFLOW_WITH_OPTIONAL_TASK workflowTestUtil.registerWorkflows( 'simple_workflow_1_integration_test.json', 'simple_workflow_1_input_template_integration_test.json', 'simple_workflow_3_integration_test.json', 'simple_workflow_with_optional_task_integration_test.json', 'simple_wait_task_workflow_integration_test.json') } def "Test simple workflow which has an optional task"() { given: "A input parameters for a workflow with an optional task" def correlationId = 'integration_test' + UUID.randomUUID().toString() def workflowInput = new HashMap() workflowInput['param1'] = 'p1 value' workflowInput['param2'] = 'p2 value' when: "An optional task workflow is started" def workflowInstanceId = startWorkflow(WORKFLOW_WITH_OPTIONAL_TASK, 1, correlationId, workflowInput, null) then: "verify that the workflow has started and the optional task is in a scheduled state" workflowInstanceId with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].status == Task.Status.SCHEDULED tasks[0].taskType == 'task_optional' } when: "The first optional task is polled and failed" Tuple polledAndFailedTaskTry1 = workflowTestUtil.pollAndFailTask('task_optional', 'task1.integration.worker', 'NETWORK ERROR') then: "Verify that the task_optional was polled and acknowledged" verifyPolledAndAcknowledgedTask(polledAndFailedTaskTry1) when: "A decide is executed on the workflow" workflowExecutor.decide(workflowInstanceId) then: "verify that the workflow is still running and the first optional task has failed and the retry has kicked in" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].status == Task.Status.FAILED tasks[0].taskType == 'task_optional' tasks[1].status == Task.Status.SCHEDULED tasks[1].taskType == 'task_optional' } when: "Poll the optional task again and do not complete it and run decide" workflowExecutionService.poll('task_optional', 'task1.integration.worker') Thread.sleep(5000) workflowExecutor.decide(workflowInstanceId) then: "Ensure that the workflow is updated" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].status == Task.Status.COMPLETED_WITH_ERRORS tasks[1].taskType == 'task_optional' tasks[2].status == Task.Status.SCHEDULED tasks[2].taskType == 'integration_task_2' } when: "The second task 'integration_task_2' is polled and completed" def task2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "Verify that the task was polled and acknowledged" verifyPolledAndAcknowledgedTask(task2Try1) and: "Ensure that the workflow is in completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[2].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_2' } } def "test workflow with input template parsing"() { given: "Input parameters for a workflow with input template" def correlationId = 'integration_test' + UUID.randomUUID().toString() def workflowInput = new HashMap() // leave other params blank on purpose to test input templates workflowInput['param3'] = 'external string' when: "Is executed and completes" def workflowInstanceId = startWorkflow(TEMPLATED_LINEAR_WORKFLOW, 1, correlationId, workflowInput, null) workflowExecutor.decide(workflowInstanceId) def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "Verify that input template is processed" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED output == [ output: "task1.done", param3: 'external string', param2: ['list', 'of', 'strings'], param1: [nested_object: [nested_key: "nested_value"]] ] } } def "Test simple workflow with task time out configuration"() { setup: "Register a task definition with retry policy on time out" def persistedTask1Definition = workflowTestUtil.getPersistedTaskDefinition('integration_task_1').get() def modifiedTaskDefinition = new TaskDef(persistedTask1Definition.name, persistedTask1Definition.description, persistedTask1Definition.ownerEmail, 1, 1, 1) modifiedTaskDefinition.retryDelaySeconds = 0 modifiedTaskDefinition.timeoutPolicy = TaskDef.TimeoutPolicy.RETRY metadataService.updateTaskDef(modifiedTaskDefinition) when: "A simple workflow is started that has a task with time out and retry configured" String correlationId = 'unit_test_1' + UUID.randomUUID() def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' input['failureWfName'] = 'FanInOutTest' def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } and: "The decider queue has one task that is ready to be polled" queueDAO.getSize(Utils.DECIDER_QUEUE) == 1 when: "The the first task 'integration_task_1' is polled and acknowledged" def task1Try1 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that a task was polled" task1Try1 task1Try1.workflowInstanceId == workflowInstanceId and: "Ensure that the decider size queue is 1 to to enable the evaluation" queueDAO.getSize(Utils.DECIDER_QUEUE) == 1 when: "There is a delay of 3 seconds introduced and the workflow is sweeped to run the evaluation" Thread.sleep(3000) sweep(workflowInstanceId) then: "Ensure that the first task has been TIMED OUT and the next task is SCHEDULED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.TIMED_OUT tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "Poll for the task again and acknowledge" def task1Try2 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that a task was polled" task1Try2 task1Try2.workflowInstanceId == workflowInstanceId when: "There is a delay of 3 seconds introduced and the workflow is swept to run the evaluation" Thread.sleep(3000) sweep(workflowInstanceId) then: "Ensure that the first task has been TIMED OUT and the next task is SCHEDULED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TIMED_OUT tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.TIMED_OUT tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.TIMED_OUT } cleanup: "Ensure that the changes of the 'integration_task_1' are reverted" metadataService.updateTaskDef(persistedTask1Definition) } def "Test workflow timeout configurations"() { setup: "Get the workflow definition and change the workflow configuration" def testWorkflowDefinition = metadataService.getWorkflowDef(TEST_WORKFLOW, 1) testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.TIME_OUT_WF testWorkflowDefinition.timeoutSeconds = 5 metadataService.updateWorkflowDef(testWorkflowDefinition) when: "A simple workflow is started that has a workflow timeout configured" String correlationId = 'unit_test_3' + UUID.randomUUID() def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' input['failureWfName'] = 'FanInOutTest' def workflowInstanceId = startWorkflow(TEST_WORKFLOW, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The the first task 'integration_task_1' is polled and acknowledged" def task1Try1 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that a task was polled" task1Try1 task1Try1.workflowInstanceId == workflowInstanceId when: "There is a delay of 6 seconds introduced and the workflow is swept to run the evaluation" Thread.sleep(6000) sweep(workflowInstanceId) then: "Ensure that the workflow has timed out" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TIMED_OUT tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } cleanup: "Ensure that the workflow configuration changes are reverted" testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.ALERT_ONLY testWorkflowDefinition.timeoutSeconds = 0 metadataService.updateWorkflowDef(testWorkflowDefinition) } def "Test retrying a timed out workflow due to workflow timeout"() { setup: "Get the workflow definition and change the workflow configuration" def testWorkflowDefinition = metadataService.getWorkflowDef(TEST_WORKFLOW, 1) testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.TIME_OUT_WF testWorkflowDefinition.timeoutSeconds = 5 metadataService.updateWorkflowDef(testWorkflowDefinition) when: "A simple workflow is started that has a workflow timeout configured" String correlationId = 'retry_timeout_wf' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(TEST_WORKFLOW, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The the first task 'integration_task_1' is polled and acknowledged" def task1Try1 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that a task was polled" task1Try1 task1Try1.workflowInstanceId == workflowInstanceId when: "There is a delay of 6 seconds introduced and the workflow is swept to run the evaluation" Thread.sleep(6000) sweep(workflowInstanceId) then: "Ensure that the workflow has timed out" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TIMED_OUT lastRetriedTime == 0 tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } when: "Retrying the workflow" workflowExecutor.retry(workflowInstanceId, false) then: "Ensure that the workflow is RUNNING and task is retried" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING lastRetriedTime != 0 tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } cleanup: "Ensure that the workflow configuration changes are reverted" testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.ALERT_ONLY testWorkflowDefinition.timeoutSeconds = 0 metadataService.updateWorkflowDef(testWorkflowDefinition) } def "Test retrying a timed out workflow due to workflow timeout without unsuccessful tasks"() { setup: "Get the workflow definition and change the workflow configuration" def testWorkflowDefinition = metadataService.getWorkflowDef(TEST_WORKFLOW, 1) testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.TIME_OUT_WF testWorkflowDefinition.timeoutSeconds = 5 metadataService.updateWorkflowDef(testWorkflowDefinition) when: "A simple workflow is started that has a workflow timeout configured" String correlationId = 'retry_timeout_wf' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(TEST_WORKFLOW, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The the first task 'integration_task_1' is polled and acknowledged" def task1 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that a task was polled" task1 task1.workflowInstanceId == workflowInstanceId when: "There is a delay of 6 seconds introduced and the task is completed" Thread.sleep(6000) task1.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(task1)) then: "verify that the workflow is TIMED_OUT and the task is COMPLETED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TIMED_OUT tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED } when: "Retrying the workflow" workflowExecutor.retry(workflowInstanceId, false) sweep(workflowInstanceId) then: "Ensure that the workflow is RUNNING and next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING lastRetriedTime != 0 tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } cleanup: "Ensure that the workflow configuration changes are reverted" testWorkflowDefinition.timeoutPolicy = WorkflowDef.TimeoutPolicy.ALERT_ONLY testWorkflowDefinition.timeoutSeconds = 0 metadataService.updateWorkflowDef(testWorkflowDefinition) } def "Test re-running the simple workflow multiple times after completion"() { given: "input required to start the workflow execution" String correlationId = 'unit_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' when: "Start a workflow based on the registered simple workflow" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Poll and complete the 'integration_task_1' " def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete 'integration_task_2'" def pollAndCompleteTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the 'integration_task_2' has been polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try1, ['tp1': inputParam1, 'tp2': 'task1.done']) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED output.containsKey('o3') } when: "The completed workflow is re run after integration_task_1" def reRunWorkflowRequest1 = new RerunWorkflowRequest() reRunWorkflowRequest1.reRunFromWorkflowId = workflowInstanceId def reRunTaskId = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[1].taskId reRunWorkflowRequest1.reRunFromTaskId = reRunTaskId def reRun1WorkflowInstanceId = workflowExecutor.rerun(reRunWorkflowRequest1) then: "Verify that the workflow is in running state and has started the re run after task 1" with(workflowExecutionService.getExecutionStatus(reRun1WorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete 'integration_task_2'" def pollAndCompleteReRunTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the 'integration_task_2' has been polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteReRunTask2Try1, ['tp1': inputParam1, 'tp2': 'task1.done']) and: "verify that the re run workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(reRun1WorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED output.containsKey('o3') } when: "The completed workflow is re run" def reRunWorkflowRequest2 = new RerunWorkflowRequest() reRunWorkflowRequest2.reRunFromWorkflowId = workflowInstanceId def reRun2WorkflowInstanceId = workflowExecutor.rerun(reRunWorkflowRequest2) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(reRun2WorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Poll and complete the 'integration_task_1' " def pollAndCompleteReRun2Task1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteReRun2Task1Try1) and: "verify that the 'integration_task1' is complete and the next task is scheduled" with(workflowExecutionService.getExecutionStatus(reRun2WorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete 'integration_task_2'" def pollAndCompleteReRun2Task2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the 'integration_task_2' has been polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteReRun2Task2Try1, ['tp1': inputParam1, 'tp2': 'task1.done']) and: "verify that the workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(reRun2WorkflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED output.containsKey('o3') } } def "Test task skipping in simple workflows"() { when: "A simple workflow is started" String correlationId = 'unit_test_3' + UUID.randomUUID() def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' def workflowInstanceId = startWorkflow(TEST_WORKFLOW, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The second task in the workflow is skipped" workflowExecutor.skipTaskFromWorkflow(workflowInstanceId, 't2', null) then: "Ensure that the second task in the workflow is skipped and the first one is still in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_2' tasks[0].status == Task.Status.SKIPPED tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.SCHEDULED } when: "Poll and complete the 'integration_task_1' " def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "Ensure that the third task is scheduled and the first one is in complete state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'integration_task_1' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_3' tasks[2].status == Task.Status.SCHEDULED } when: "Poll and complete the 'integration_task_3' " def pollAndCompleteTask3Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_3', 'task3.integration.worker') then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask3Try1) and: "verify that the workflow is in a complete state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[2].taskType == 'integration_task_3' tasks[2].status == Task.Status.COMPLETED } } def "Test pause and resume simple workflow"() { given: "input required to start the workflow execution" String correlationId = 'unit_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' when: "Start a workflow based on the registered simple workflow" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "verify that the workflow is in a running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The running workflow is paused" workflowExecutor.pauseWorkflow(workflowInstanceId) and: "Poll and complete the 'integration_task_1' " def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the workflow is in PAUSED state and the next task is not scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.PAUSED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED } when: "The next task in the workflow is polled for" def task2Try1 = workflowExecutionService.poll('integration_task_2', 'task2.integration.worker') then: "verify that there was no task polled" !task2Try1 when: "A decide is run explicitly" workflowExecutor.decide(workflowInstanceId) and: "The next task is polled again" def task2Try2 = workflowExecutionService.poll('integration_task_2', 'task2.integration.worker') then: "verify that there was no task polled" !task2Try2 when: "The workflow is resumed" workflowExecutor.resumeWorkflow(workflowInstanceId) then: "verify that the workflow was resumed and the next task is in a scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll and complete 'integration_task_2'" def pollAndCompleteTask2Try3 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the 'integration_task_2' has been polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try3, ['tp1': inputParam1, 'tp2': 'task1.done']) and: "verify that the re run workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED output.containsKey('o3') } } def "Test wait time out task based simple workflow"() { when: "Start a workflow based on a task that has a registered wait time out" def workflowInstanceId = startWorkflow(WAIT_TIME_OUT_WORKFLOW, 1, '', [:], null) then: "verify that the workflow is running and the first task scheduled" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'WAIT' tasks[0].status == Task.Status.IN_PROGRESS } when: "A delay is introduced" Thread.sleep(3000) and: "A decide is executed on the workflow" workflowExecutor.decide(workflowInstanceId) then: "verify that the workflow is in running state and a replacement task has been scheduled due to time out" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'WAIT' tasks[0].status == Task.Status.TIMED_OUT tasks[1].taskType == 'WAIT' tasks[1].status == Task.Status.SCHEDULED } when: "The wait task is completed" def waitTask = workflowExecutionService.getExecutionStatus(workflowInstanceId, true).tasks[1] waitTask.status = Task.Status.COMPLETED workflowExecutor.updateTask(new TaskResult(waitTask)) and: "verify that the workflow is in running state and the next task is scheduled and 'waitTimeout' task is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 3 tasks[1].taskType == 'WAIT' tasks[1].status == Task.Status.COMPLETED tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.SCHEDULED } and: "Poll and complete the 'integration_task_1' " def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "The workflow is in a completed state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 3 tasks[2].taskType == 'integration_task_1' tasks[2].status == Task.Status.COMPLETED } } def "Test simple workflow with callbackAfterSeconds for tasks"() { given: "input required to start the workflow execution" String correlationId = 'unit_test_1' def input = new HashMap() String inputParam1 = 'p1 value' input['param1'] = inputParam1 input['param2'] = 'p2 value' when: "Start a workflow based on the registered simple workflow" def workflowInstanceId = startWorkflow(LINEAR_WORKFLOW_T1_T2, 1, correlationId, input, null) then: "Ensure that the workflow has started" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The first task is polled and then a callbackAfterSeconds is added to the task" def task1Try1 = workflowExecutionService.poll('integration_task_1', 'task1.worker') task1Try1.status = Task.Status.IN_PROGRESS task1Try1.callbackAfterSeconds = 2L workflowExecutionService.updateTask(new TaskResult(task1Try1)) then: "verify that the workflow is in running state and the task is in SCHEDULED" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "the 'integration_task_1' is polled again" def task1Try2 = workflowExecutionService.poll('integration_task_1', 'task1.worker') then: "Ensure that there was no task polled due to the callBackAfterSeconds" !task1Try2 then: "verify that the workflow is in running state and the task is in progress" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "There is a delay introduced to go over the callbackAfterSeconds interval" Thread.sleep(2050) and: "the 'integration_task_1' is polled and completed" def pollAndCompleteTask1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker', ['op': 'task1.done']) then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask1Try1) and: "verify that the workflow has moved forward and 'integration_task_1 is completed'" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "The second task is polled and then a callbackAfterSeconds is added to the task" def task2Try1 = workflowExecutionService.poll('integration_task_2', 'task2.worker') task2Try1.status = Task.Status.IN_PROGRESS task2Try1.callbackAfterSeconds = 5L workflowExecutionService.updateTask(new TaskResult(task2Try1)) then: "Verify that the workflow is in running state and the task is in scheduled state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "poll for 'integration_task_2'" def task2Try2 = workflowExecutionService.poll('integration_task_2', 'task2.worker') then: "Ensure that there was no task polled due to the callBackAfterSeconds, even though the task is in scheduled state" !task2Try2 when: "A delay is introduced to get over the callBackAfterSeconds interval" Thread.sleep(5100) and: "the 'integration_task_2' is polled and completed" def pollAndCompleteTask2Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task1.integration.worker') then: "verify that the 'integration_task_1' was polled and acknowledged" verifyPolledAndAcknowledgedTask(pollAndCompleteTask2Try1) and: "verify that the workflow has moved forward and 'integration_task_1 is completed'" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } } def "Test workflow with no tasks"() { setup: "Create a workflow definition with no tasks" WorkflowDef emptyWorkflowDef = new WorkflowDef() emptyWorkflowDef.setName("empty_workflow") emptyWorkflowDef.setSchemaVersion(2) when: "a workflow is started with this definition" def input = new HashMap() def correlationId = 'empty_workflow' def workflowInstanceId = startWorkflowOperation.execute(new StartWorkflowInput(workflowDefinition: emptyWorkflowDef, workflowInput: input, correlationId: correlationId)) then: "the workflow is completed" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 0 } } def "Test task def template"() { setup: "Register a task definition with input template" TaskDef templatedTask = new TaskDef() templatedTask.setName('templated_task') def httpRequest = new HashMap<>() httpRequest['method'] = 'GET' httpRequest['vipStack'] = '${STACK2}' httpRequest['uri'] = '/get/something' def body = new HashMap<>() body['inputPaths'] = Arrays.asList('${workflow.input.path1}', '${workflow.input.path2}') body['requestDetails'] = '${workflow.input.requestDetails}' body['outputPath'] = '${workflow.input.outputPath}' httpRequest['body'] = body templatedTask.inputTemplate['http_request'] = httpRequest templatedTask.ownerEmail = "test@harness.com" metadataService.registerTaskDef(Arrays.asList(templatedTask)) and: "set a system property for STACK2" System.setProperty('STACK2', 'test_stack') and: "a workflow definition using this task is created" WorkflowTask workflowTask = new WorkflowTask() workflowTask.setName(templatedTask.getName()) workflowTask.setWorkflowTaskType(TaskType.SIMPLE) workflowTask.setTaskReferenceName("t0") WorkflowDef templateWorkflowDef = new WorkflowDef() templateWorkflowDef.setName("template_workflow") templateWorkflowDef.getTasks().add(workflowTask) templateWorkflowDef.setSchemaVersion(2) templateWorkflowDef.setOwnerEmail("test@harness.com") metadataService.registerWorkflowDef(templateWorkflowDef) and: "the input to the workflow is curated" def requestDetails = new HashMap<>() requestDetails['key1'] = 'value1' requestDetails['key2'] = 42 Map input = new HashMap<>() input['path1'] = 'file://path1' input['path2'] = 'file://path2' input['outputPath'] = 's3://bucket/outputPath' input['requestDetails'] = requestDetails when: "the workflow is started" def correlationId = 'workflow_taskdef_template' def workflowInstanceId = startWorkflowOperation.execute(new StartWorkflowInput(workflowDefinition: templateWorkflowDef, workflowInput: input, correlationId: correlationId)) then: "the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].inputData.get('http_request') instanceof Map tasks[0].inputData.get('http_request')['method'] == 'GET' tasks[0].inputData.get('http_request')['vipStack'] == 'test_stack' tasks[0].inputData.get('http_request')['body'] instanceof Map tasks[0].inputData.get('http_request')['body']['requestDetails'] instanceof Map tasks[0].inputData.get('http_request')['body']['requestDetails']['key1'] == 'value1' tasks[0].inputData.get('http_request')['body']['requestDetails']['key2'] == 42 tasks[0].inputData.get('http_request')['body']['outputPath'] == 's3://bucket/outputPath' tasks[0].inputData.get('http_request')['body']['inputPaths'] instanceof List tasks[0].inputData.get('http_request')['body']['inputPaths'][0] == 'file://path1' tasks[0].inputData.get('http_request')['body']['inputPaths'][1] == 'file://path2' tasks[0].inputData.get('http_request')['uri'] == '/get/something' } } def "Test task def created if not exist"() { setup: "Register a workflow definition with task def not registered" def taskDefName = "task_not_registered" WorkflowTask workflowTask = new WorkflowTask() workflowTask.setName(taskDefName) workflowTask.setWorkflowTaskType(TaskType.SIMPLE) workflowTask.setTaskReferenceName("t0") WorkflowDef testWorkflowDef = new WorkflowDef() testWorkflowDef.setName("test_workflow") testWorkflowDef.getTasks().add(workflowTask) testWorkflowDef.setSchemaVersion(2) testWorkflowDef.setOwnerEmail("test@harness.com") metadataService.registerWorkflowDef(testWorkflowDef) when: "the workflow is started" def correlationId = 'workflow_taskdef_not_registered' def workflowInstanceId = startWorkflowOperation.execute(new StartWorkflowInput(workflowDefinition: testWorkflowDef, workflowInput: [:], correlationId: correlationId)) then: "the workflow is in running state" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskDefName == taskDefName } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/QueueResiliencySpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.resiliency import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpStatus import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.common.utils.ExternalPayloadStorage import com.netflix.conductor.core.exception.NotFoundException import com.netflix.conductor.core.exception.TransientException import com.netflix.conductor.core.utils.QueueUtils import com.netflix.conductor.core.utils.Utils import com.netflix.conductor.rest.controllers.TaskResource import com.netflix.conductor.rest.controllers.WorkflowResource import com.netflix.conductor.test.base.AbstractResiliencySpecification /** * When QueueDAO is unavailable, * Ensure All Worklow and Task resource endpoints either: * 1. Fails and/or throws an Exception * 2. Succeeds * 3. Doesn't involve QueueDAO */ class QueueResiliencySpec extends AbstractResiliencySpecification { @Autowired WorkflowResource workflowResource @Autowired TaskResource taskResource def SIMPLE_TWO_TASK_WORKFLOW = 'integration_test_wf' def setup() { workflowTestUtil.taskDefinitions() workflowTestUtil.registerWorkflows( 'simple_workflow_1_integration_test.json' ) } /// Workflow Resource endpoints def "Verify Start workflow fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def response = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow starts when there are no Queue failures" response when: "We try same request Queue failure" response = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify that workflow start fails with BACKEND_ERROR" 1 * queueDAO.push(*_) >> { throw new TransientException("Queue push failed from Spy") } thrown(TransientException.class) } def "Verify terminate succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "We terminate it when QueueDAO is unavailable" workflowResource.terminate(workflowInstanceId, "Terminated from a test") then: "Verify that terminate is successful without any exceptions" 2 * queueDAO.remove(*_) >> { throw new TransientException("Queue remove failed from Spy") } 0 * queueDAO._ with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } } def "Verify Restart workflow fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) and: "We terminate it when QueueDAO is unavailable" workflowResource.terminate(workflowInstanceId, "Terminated from a test") then: "Verify that workflow is in terminated state" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } when: "We restart workflow when QueueDAO is unavailable" workflowResource.restart(workflowInstanceId, false) then: "" 1 * queueDAO.push(*_) >> { throw new TransientException("Queue push failed from Spy") } 1 * queueDAO.remove(*_) >> { throw new TransientException("Queue remove failed from Spy") } 0 * queueDAO._ thrown(TransientException.class) with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 0 } } def "Verify rerun fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) and: "terminate it" workflowResource.terminate(workflowInstanceId, "Terminated from a test") then: "Verify that workflow is in terminated state" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } when: "Workflow is rerun when QueueDAO is unavailable" def rerunWorkflowRequest = new RerunWorkflowRequest() rerunWorkflowRequest.setReRunFromWorkflowId(workflowInstanceId) workflowResource.rerun(workflowInstanceId, rerunWorkflowRequest) then: "" 1 * queueDAO.push(*_) >> { throw new TransientException("Queue push failed from Spy") } 0 * queueDAO._ thrown(TransientException.class) with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 0 } } def "Verify retry fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) and: "terminate it" workflowResource.terminate(workflowInstanceId, "Terminated from a test") then: "Verify that workflow is in terminated state" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } when: "workflow is restarted when QueueDAO is unavailable" workflowResource.retry(workflowInstanceId, false) then: "Verify retry fails" 1 * queueDAO.push(*_) >> { throw new TransientException("Queue push failed from Spy") } 0 * queueDAO._ thrown(TransientException.class) with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED } } def "Verify getWorkflow succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "We get a workflow when QueueDAO is unavailable" def workflow = workflowResource.getExecutionStatus(workflowInstanceId, true) then: "Verify workflow is returned" 0 * queueDAO._ workflow.getStatus() == Workflow.WorkflowStatus.RUNNING workflow.getTasks().size() == 1 workflow.getTasks()[0].status == Task.Status.SCHEDULED } def "Verify getWorkflows succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "We get a workflow when QueueDAO is unavailable" def workflows = workflowResource.getWorkflows(SIMPLE_TWO_TASK_WORKFLOW, "", true, true) then: "Verify queueDAO is not involved and an exception is not thrown" 0 * queueDAO._ notThrown(Exception) } def "Verify remove workflow succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "We get a workflow when QueueDAO is unavailable" workflowResource.delete(workflowInstanceId, false) then: "Verify queueDAO is called to remove from _deciderQueue" 1 * queueDAO.remove(Utils.DECIDER_QUEUE, _) when: "We try to get deleted workflow, verify the status and check if tasks are not removed from queue" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.TERMINATED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.CANCELED 0 * queueDAO.remove(QueueUtils.getQueueName(tasks[0]), _) } then: thrown(NotFoundException.class) } def "Verify decide succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "We decide a workflow" workflowResource.decide(workflowInstanceId) then: "Verify queueDAO is not involved" 0 * queueDAO._ } def "Verify pause succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The workflow is paused when QueueDAO is unavailable" workflowResource.pauseWorkflow(workflowInstanceId) then: "Verify workflow is paused without any exceptions" 1 * queueDAO.remove(*_) >> { throw new IllegalStateException("Queue remove failed from Spy") } 0 * queueDAO._ with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.PAUSED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } } def "Verify resume fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The workflow is paused" workflowResource.pauseWorkflow(workflowInstanceId) then: "Verify workflow is paused" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.PAUSED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Workflow is resumed when QueueDAO is unavailable" workflowResource.resumeWorkflow(workflowInstanceId) then: "exception is thrown" 1 * queueDAO.push(*_) >> { throw new TransientException("Queue push failed from Spy") } thrown(TransientException.class) with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.PAUSED tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } } def "Verify reset callbacks fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "Task is updated with callBackAfterSeconds" def workflow = workflowResource.getExecutionStatus(workflowInstanceId, true) def task = workflow.getTasks().get(0) def taskResult = new TaskResult(task) taskResult.setCallbackAfterSeconds(120) taskResource.updateTask(taskResult) and: "and then reset callbacks when QueueDAO is unavailable" workflowResource.resetWorkflow(workflowInstanceId) then: "Verify an exception is thrown" 1 * queueDAO.resetOffsetTime(*_) >> { throw new TransientException("Queue resetOffsetTime failed from Spy") } thrown(TransientException.class) } def "Verify search is not impacted by QueueDAO"() { when: "We perform a search" workflowResource.search(0, 1, "", "", "") then: "Verify it doesn't involve QueueDAO" 0 * queueDAO._ } def "Verify search workflows by tasks is not impacted by QueueDAO"() { when: "We perform a search" workflowResource.searchWorkflowsByTasks(0, 1, "", "", "") then: "Verify it doesn't involve QueueDAO" 0 * queueDAO._ } def "Verify get external storage location is not impacted by QueueDAO"() { when: workflowResource.getExternalStorageLocation("", ExternalPayloadStorage.Operation.READ as String, ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT as String) then: "Verify it doesn't involve QueueDAO" 0 * queueDAO._ } /// Task Resource endpoints def "Verify polls return with no result when QueueDAO is unavailable"() { when: "Some task 'integration_task_1' is polled" def responseEntity = taskResource.poll("integration_task_1", "test", "") then: 1 * queueDAO.pop(*_) >> { throw new IllegalStateException("Queue pop failed from Spy") } 0 * queueDAO._ notThrown(Exception) responseEntity && responseEntity.statusCode == HttpStatus.NO_CONTENT && !responseEntity.body } def "Verify updateTask with COMPLETE status succeeds when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The first task 'integration_task_1' is polled" def responseEntity = taskResource.poll("integration_task_1", "test", null) then: "Verify task is returned successfully" responseEntity && responseEntity.statusCode == HttpStatus.OK && responseEntity.body responseEntity.body.status == Task.Status.IN_PROGRESS responseEntity.body.taskType == 'integration_task_1' when: "the above task is updated, while QueueDAO is unavailable" def taskResult = new TaskResult(responseEntity.body) taskResult.setStatus(TaskResult.Status.COMPLETED) def result = taskResource.updateTask(taskResult) then: "updateTask returns successfully without any exceptions" 1 * queueDAO.remove(*_) >> { throw new IllegalStateException("Queue remove failed from Spy") } result == responseEntity.body.taskId notThrown(Exception) } def "Verify updateTask with IN_PROGRESS state fails when QueueDAO is unavailable"() { when: "Start a simple workflow" def workflowInstanceId = workflowResource.startWorkflow(new StartWorkflowRequest() .withName(SIMPLE_TWO_TASK_WORKFLOW) .withVersion(1)) then: "Verify workflow is started" with(workflowResource.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 1 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.SCHEDULED } when: "The first task 'integration_task_1' is polled" def responseEntity = taskResource.poll("integration_task_1", "test", null) then: "Verify task is returned successfully" responseEntity && responseEntity.statusCode == HttpStatus.OK responseEntity.body.status == Task.Status.IN_PROGRESS responseEntity.body.taskType == 'integration_task_1' when: "the above task is updated, while QueueDAO is unavailable" def taskResult = new TaskResult(responseEntity.body) taskResult.setStatus(TaskResult.Status.IN_PROGRESS) taskResult.setCallbackAfterSeconds(120) def result = taskResource.updateTask(taskResult) then: "updateTask fails with an exception" 1 * queueDAO.postpone(*_) >> { throw new IllegalStateException("Queue postpone failed from Spy") } thrown(Exception) } def "verify getTaskQueueSizes fails when QueueDAO is unavailable"() { when: taskResource.size(Arrays.asList("testTaskType", "testTaskType2")) then: 1 * queueDAO.getSize(*_) >> { throw new IllegalStateException("Queue getSize failed from Spy") } thrown(Exception) } def "Verify log doesn't involve QueueDAO"() { when: taskResource.log("testTaskId", "test log") then: 0 * queueDAO._ } def "Verify getTaskLogs doesn't involve QueueDAO"() { when: taskResource.getTaskLogs("testTaskId") then: 0 * queueDAO._ } def "Verify getTask doesn't involve QueueDAO"() { when: taskResource.getTask("testTaskId") then: 0 * queueDAO._ } def "Verify getAllQueueDetails fails when QueueDAO is unavailable"() { when: taskResource.all() then: 1 * queueDAO.queuesDetail() >> { throw new IllegalStateException("Queue queuesDetail failed from Spy") } thrown(Exception) } def "Verify getPollData doesn't involve QueueDAO"() { when: taskResource.getPollData("integration_test_1") then: 0 * queueDAO.queuesDetail() } def "Verify getAllPollData fails when QueueDAO is unavailable"() { when: taskResource.getAllPollData() then: 1 * queueDAO.queuesDetail() >> { throw new IllegalStateException("Queue queuesDetail failed from Spy") } thrown(Exception) } def "Verify task search is not impacted by QueueDAO"() { when: "We perform a search" taskResource.search(0, 1, "", "", "") then: "Verify it doesn't involve QueueDAO" 0 * queueDAO._ } def "Verify task get external storage location is not impacted by QueueDAO"() { when: taskResource.getExternalStorageLocation("", ExternalPayloadStorage.Operation.READ as String, ExternalPayloadStorage.PayloadType.TASK_INPUT as String) then: "Verify it doesn't involve QueueDAO" 0 * queueDAO._ } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/TaskResiliencySpec.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.resiliency import org.springframework.beans.factory.annotation.Autowired import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.run.Workflow import com.netflix.conductor.core.reconciliation.WorkflowRepairService import com.netflix.conductor.test.base.AbstractResiliencySpecification import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask class TaskResiliencySpec extends AbstractResiliencySpecification { @Autowired WorkflowRepairService workflowRepairService @Shared def SIMPLE_TWO_TASK_WORKFLOW = 'integration_test_wf' def setup() { workflowTestUtil.taskDefinitions() workflowTestUtil.registerWorkflows( 'simple_workflow_1_integration_test.json' ) } def "Verify that a workflow recovers and completes on schedule task failure from queue push failure"() { when: "Start a simple workflow" def workflowInstanceId = startWorkflow(SIMPLE_TWO_TASK_WORKFLOW, 1, '', [:], null) then: "Retrieve the workflow" def workflow = workflowExecutionService.getExecutionStatus(workflowInstanceId, true) workflow.status == Workflow.WorkflowStatus.RUNNING workflow.tasks.size() == 1 workflow.tasks[0].taskType == 'integration_task_1' workflow.tasks[0].status == Task.Status.SCHEDULED def taskId = workflow.tasks[0].taskId // Simulate queue push failure when creating a new task, after completing first task when: "The first task 'integration_task_1' is polled and completed" def task1Try1 = workflowTestUtil.pollAndCompleteTask('integration_task_1', 'task1.integration.worker') then: "Verify that the task was polled and acknowledged" 1 * queueDAO.pop(_, 1, _) >> Collections.singletonList(taskId) 1 * queueDAO.ack(*_) >> true 1 * queueDAO.push(*_) >> { throw new IllegalStateException("Queue push failed from Spy") } verifyPolledAndAcknowledgedTask(task1Try1) and: "Ensure that the next task is SCHEDULED even after failing to push taskId message to queue" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED } when: "The second task 'integration_task_2' is polled for" def task1Try2 = workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "Verify that the task was not polled, and the taskId doesn't exist in the queue" task1Try2[0] == null with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.RUNNING tasks.size() == 2 tasks[0].taskType == 'integration_task_1' tasks[0].status == Task.Status.COMPLETED tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.SCHEDULED def currentTaskId = tasks[1].getTaskId() !queueDAO.containsMessage("integration_task_2", currentTaskId) } when: "Running a repair and decide on the workflow" workflowRepairService.verifyAndRepairWorkflow(workflowInstanceId, true) workflowExecutor.decide(workflowInstanceId) workflowTestUtil.pollAndCompleteTask('integration_task_2', 'task2.integration.worker') then: "verify that the next scheduled task can be polled and executed successfully" with(workflowExecutionService.getExecutionStatus(workflowInstanceId, true)) { status == Workflow.WorkflowStatus.COMPLETED tasks.size() == 2 tasks[1].taskType == 'integration_task_2' tasks[1].status == Task.Status.COMPLETED } } } ================================================ FILE: test-harness/src/test/groovy/com/netflix/conductor/test/util/WorkflowTestUtil.groovy ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.util import javax.annotation.PostConstruct import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.core.WorkflowContext import com.netflix.conductor.core.exception.NotFoundException import com.netflix.conductor.core.execution.WorkflowExecutor import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.model.WorkflowModel import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.fasterxml.jackson.databind.ObjectMapper /** * This is a helper class used to initialize task definitions required by the tests when loaded up. * The task definitions that are loaded up in {@link WorkflowTestUtil#taskDefinitions()} method as part of the post construct of the bean. * This class is intended to be used in the Spock integration tests and provides helper methods to: *

      *
    • Terminate all the running Workflows
      • *
    • Get the persisted task definition based on the taskName
      • *
    • pollAndFailTask
      • *
    • pollAndCompleteTask
      • *
    • verifyPolledAndAcknowledgedTask
      • *
    * * Usage: Autowire this class in any Spock based specification: * * {@literal @}Autowired * WorkflowTestUtil workflowTestUtil * */ @Component class WorkflowTestUtil { private final MetadataService metadataService private final ExecutionService workflowExecutionService private final WorkflowExecutor workflowExecutor private final QueueDAO queueDAO private final ObjectMapper objectMapper private static final int RETRY_COUNT = 1 private static final String TEMP_FILE_PATH = "/input.json" private static final String DEFAULT_EMAIL_ADDRESS = "test@harness.com" @Autowired WorkflowTestUtil(MetadataService metadataService, ExecutionService workflowExecutionService, WorkflowExecutor workflowExecutor, QueueDAO queueDAO, ObjectMapper objectMapper) { this.metadataService = metadataService this.workflowExecutionService = workflowExecutionService this.workflowExecutor = workflowExecutor this.queueDAO = queueDAO this.objectMapper = objectMapper } /** * This function registers all the taskDefinitions required to enable spock based integration testing */ @PostConstruct void taskDefinitions() { WorkflowContext.set(new WorkflowContext("integration_app")) (0..20).collect { "integration_task_$it" } .findAll { !getPersistedTaskDefinition(it).isPresent() } .collect { new TaskDef(it, it, DEFAULT_EMAIL_ADDRESS, 1, 120, 120) } .forEach { metadataService.registerTaskDef([it]) } (0..4).collect { "integration_task_0_RT_$it" } .findAll { !getPersistedTaskDefinition(it).isPresent() } .collect { new TaskDef(it, it, DEFAULT_EMAIL_ADDRESS, 0, 120, 120) } .forEach { metadataService.registerTaskDef([it]) } metadataService.registerTaskDef([new TaskDef('short_time_out', 'short_time_out', DEFAULT_EMAIL_ADDRESS, 1, 5, 5)]) //This taskWithResponseTimeOut is required by the integration test which exercises the response time out scenarios TaskDef taskWithResponseTimeOut = new TaskDef() taskWithResponseTimeOut.name = "task_rt" taskWithResponseTimeOut.timeoutSeconds = 120 taskWithResponseTimeOut.retryCount = RETRY_COUNT taskWithResponseTimeOut.retryDelaySeconds = 0 taskWithResponseTimeOut.responseTimeoutSeconds = 10 taskWithResponseTimeOut.ownerEmail = DEFAULT_EMAIL_ADDRESS TaskDef optionalTask = new TaskDef() optionalTask.setName("task_optional") optionalTask.setTimeoutSeconds(5) optionalTask.setRetryCount(1) optionalTask.setTimeoutPolicy(TaskDef.TimeoutPolicy.RETRY) optionalTask.setRetryDelaySeconds(0) optionalTask.setResponseTimeoutSeconds(5) optionalTask.setOwnerEmail(DEFAULT_EMAIL_ADDRESS) TaskDef simpleSubWorkflowTask = new TaskDef() simpleSubWorkflowTask.setName('simple_task_in_sub_wf') simpleSubWorkflowTask.setRetryCount(0) simpleSubWorkflowTask.setOwnerEmail(DEFAULT_EMAIL_ADDRESS) TaskDef subWorkflowTask = new TaskDef() subWorkflowTask.setName('sub_workflow_task') subWorkflowTask.setRetryCount(1) subWorkflowTask.setResponseTimeoutSeconds(5) subWorkflowTask.setRetryDelaySeconds(0) subWorkflowTask.setOwnerEmail(DEFAULT_EMAIL_ADDRESS) TaskDef waitTimeOutTask = new TaskDef() waitTimeOutTask.name = 'waitTimeout' waitTimeOutTask.timeoutSeconds = 2 waitTimeOutTask.responseTimeoutSeconds = 2 waitTimeOutTask.retryCount = 1 waitTimeOutTask.timeoutPolicy = TaskDef.TimeoutPolicy.RETRY waitTimeOutTask.retryDelaySeconds = 10 waitTimeOutTask.ownerEmail = DEFAULT_EMAIL_ADDRESS TaskDef userTask = new TaskDef() userTask.setName("user_task") userTask.setTimeoutSeconds(20) userTask.setResponseTimeoutSeconds(20) userTask.setRetryCount(1) userTask.setTimeoutPolicy(TaskDef.TimeoutPolicy.RETRY) userTask.setRetryDelaySeconds(10) userTask.setOwnerEmail(DEFAULT_EMAIL_ADDRESS) TaskDef concurrentExecutionLimitedTask = new TaskDef() concurrentExecutionLimitedTask.name = "test_task_with_concurrency_limit" concurrentExecutionLimitedTask.concurrentExecLimit = 1 concurrentExecutionLimitedTask.ownerEmail = DEFAULT_EMAIL_ADDRESS TaskDef rateLimitedTask = new TaskDef() rateLimitedTask.name = 'test_task_with_rateLimits' rateLimitedTask.rateLimitFrequencyInSeconds = 10 rateLimitedTask.rateLimitPerFrequency = 1 rateLimitedTask.ownerEmail = DEFAULT_EMAIL_ADDRESS TaskDef rateLimitedSimpleTask = new TaskDef() rateLimitedSimpleTask.name = 'test_simple_task_with_rateLimits' rateLimitedSimpleTask.rateLimitFrequencyInSeconds = 10 rateLimitedSimpleTask.rateLimitPerFrequency = 1 rateLimitedSimpleTask.ownerEmail = DEFAULT_EMAIL_ADDRESS TaskDef eventTaskX = new TaskDef() eventTaskX.name = 'eventX' eventTaskX.timeoutSeconds = 10 eventTaskX.responseTimeoutSeconds = 10 eventTaskX.ownerEmail = DEFAULT_EMAIL_ADDRESS metadataService.registerTaskDef( [taskWithResponseTimeOut, optionalTask, simpleSubWorkflowTask, subWorkflowTask, waitTimeOutTask, userTask, eventTaskX, rateLimitedTask, rateLimitedSimpleTask, concurrentExecutionLimitedTask] ) } /** * This is an helper method that enables each test feature to run from a clean state * This method is intended to be used in the cleanup() or cleanupSpec() method of any spock specification. * By invoking this method all the running workflows are terminated. * @throws Exception When unable to terminate any running workflow */ void clearWorkflows() throws Exception { List workflowsWithVersion = metadataService.getWorkflowDefs() .collect { workflowDef -> workflowDef.getName() + ":" + workflowDef.getVersion() } for (String workflowWithVersion : workflowsWithVersion) { String workflowName = StringUtils.substringBefore(workflowWithVersion, ":") int version = Integer.parseInt(StringUtils.substringAfter(workflowWithVersion, ":")) List running = workflowExecutionService.getRunningWorkflows(workflowName, version) for (String workflowId : running) { WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, false) if (!workflow.getStatus().isTerminal()) { workflowExecutor.terminateWorkflow(workflowId, "cleanup") } } } queueDAO.queuesDetail().keySet() .forEach { queueDAO.flush(it) } new FileOutputStream(this.getClass().getResource(TEMP_FILE_PATH).getPath()).close() } /** * A helper method to retrieve a task definition that is persisted * @param taskDefName The name of the task for which the task definition is requested * @return an Optional of the TaskDefinition */ Optional getPersistedTaskDefinition(String taskDefName) { try { return Optional.of(metadataService.getTaskDef(taskDefName)) } catch(NotFoundException nfe) { return Optional.empty() } } /** * A helper methods that registers workflows based on the paths of the json file representing a workflow definition * @param workflowJsonPaths a comma separated var ags of the paths of the workflow definitions */ void registerWorkflows(String... workflowJsonPaths) { workflowJsonPaths.collect { readFile(it) } .forEach { metadataService.updateWorkflowDef(it) } } WorkflowDef readFile(String path) { InputStream inputStream = getClass().getClassLoader().getResourceAsStream(path) return objectMapper.readValue(inputStream, WorkflowDef.class) } /** * A helper method intended to be used in the when: block of the spock test feature * This method is intended to be used to poll and update the task status as failed * It also provides a delay to return if needed after the task has been updated to failed * @param taskName name of the task that needs to be polled and failed * @param workerId name of the worker id using which a task is polled * @param failureReason the reason to fail the task that will added to the task update * @param outputParams An optional output parameters if available will be added to the task before updating to failed * @param waitAtEndSeconds an optional delay before the method returns, if the value is 0 skips the delay * @return A Tuple of taskResult and acknowledgement of the poll */ Tuple pollAndFailTask(String taskName, String workerId, String failureReason, Map outputParams = null, int waitAtEndSeconds = 0) { def polledIntegrationTask = workflowExecutionService.poll(taskName, workerId) def taskResult = new TaskResult(polledIntegrationTask) taskResult.status = TaskResult.Status.FAILED taskResult.reasonForIncompletion = failureReason if (outputParams) { outputParams.forEach { k, v -> taskResult.outputData[k] = v } } workflowExecutionService.updateTask(taskResult) return waitAtEndSecondsAndReturn(waitAtEndSeconds, polledIntegrationTask) } /** * A helper method to introduce delay and convert the polledIntegrationTask and ackPolledIntegrationTask * into a tuple. This method is intended to be used by pollAndFailTask and pollAndCompleteTask * @param waitAtEndSeconds The total seconds of delay before the method returns * @param ackedTaskResult the task result created after ack * @return A Tuple of polledTask and acknowledgement of the poll */ static Tuple waitAtEndSecondsAndReturn(int waitAtEndSeconds, Task polledIntegrationTask) { if (waitAtEndSeconds > 0) { Thread.sleep(waitAtEndSeconds * 1000) } return new Tuple(polledIntegrationTask) } /** * A helper method intended to be used in the when: block of the spock test feature * This method is intended to be used to poll and update the task status as completed * It also provides a delay to return if needed after the task has been updated to completed * @param taskName name of the task that needs to be polled and completed * @param workerId name of the worker id using which a task is polled * @param outputParams An optional output parameters if available will be added to the task before updating to completed * @param waitAtEndSeconds waitAtEndSeconds an optional delay before the method returns, if the value is 0 skips the delay * @return A Tuple of polledTask and acknowledgement of the poll */ Tuple pollAndCompleteTask(String taskName, String workerId, Map outputParams = null, int waitAtEndSeconds = 0) { def polledIntegrationTask = workflowExecutionService.poll(taskName, workerId) if (polledIntegrationTask == null) { return new Tuple(null, null) } def taskResult = new TaskResult(polledIntegrationTask) taskResult.status = TaskResult.Status.COMPLETED if (outputParams) { outputParams.forEach { k, v -> taskResult.outputData[k] = v } } workflowExecutionService.updateTask(taskResult) return waitAtEndSecondsAndReturn(waitAtEndSeconds, polledIntegrationTask) } Tuple pollAndCompleteLargePayloadTask(String taskName, String workerId, String outputPayloadPath) { def polledIntegrationTask = workflowExecutionService.poll(taskName, workerId) def taskResult = new TaskResult(polledIntegrationTask) taskResult.status = TaskResult.Status.COMPLETED taskResult.outputData = null taskResult.externalOutputPayloadStoragePath = outputPayloadPath workflowExecutionService.updateTask(taskResult) return new Tuple(polledIntegrationTask) } Tuple pollAndUpdateTask(String taskName, String workerId, String outputPayloadPath, Map outputParams = null, int waitAtEndSeconds = 0) { def polledIntegrationTask = workflowExecutionService.poll(taskName, workerId) def taskResult = new TaskResult(polledIntegrationTask) taskResult.status = TaskResult.Status.IN_PROGRESS taskResult.callbackAfterSeconds = 1 if (outputPayloadPath) { taskResult.outputData = null taskResult.externalOutputPayloadStoragePath = outputPayloadPath } else if (outputParams) { outputParams.forEach { k, v -> taskResult.outputData[k] = v } } workflowExecutionService.updateTask(taskResult) return waitAtEndSecondsAndReturn(waitAtEndSeconds, polledIntegrationTask) } /** * A helper method intended to be used in the then: block of the spock test feature, ideally intended to be called after either: * pollAndCompleteTask function or pollAndFailTask function * @param completedTaskAndAck A Tuple of polledTask and acknowledgement of the poll * @param expectedTaskInputParams a map of input params that are verified against the polledTask that is part of the completedTaskAndAck tuple */ static void verifyPolledAndAcknowledgedTask(Tuple completedTaskAndAck, Map expectedTaskInputParams = null) { assert completedTaskAndAck[0]: "The task polled cannot be null" def polledIntegrationTask = completedTaskAndAck[0] as Task assert polledIntegrationTask if (expectedTaskInputParams) { expectedTaskInputParams.forEach { k, v -> assert polledIntegrationTask.inputData.containsKey(k) assert polledIntegrationTask.inputData[k] == v } } } static void verifyPolledAndAcknowledgedLargePayloadTask(Tuple completedTaskAndAck) { assert completedTaskAndAck[0]: "The task polled cannot be null" def polledIntegrationTask = completedTaskAndAck[0] as Task assert polledIntegrationTask } static void verifyPayload(Map expected, Map payload) { expected.forEach { k, v -> assert payload.containsKey(k) assert payload[k] == v } } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/ConductorTestApp.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor; import java.io.IOException; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; /** Copy of com.netflix.conductor.Conductor for use by @SpringBootTest in AbstractSpecification. */ // Prevents from the datasource beans to be loaded, AS they are needed only for specific databases. // In case that SQL database is selected this class will be imported back in the appropriate // database persistence module. @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) public class ConductorTestApp { public static void main(String[] args) throws IOException { SpringApplication.run(ConductorTestApp.class, args); } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.Reader; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Optional; import org.apache.http.HttpHost; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.test.context.TestPropertySource; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @TestPropertySource( properties = {"conductor.indexing.enabled=true", "conductor.elasticsearch.version=6"}) public abstract class AbstractEndToEndTest { private static final Logger log = LoggerFactory.getLogger(AbstractEndToEndTest.class); private static final String TASK_DEFINITION_PREFIX = "task_"; private static final String DEFAULT_DESCRIPTION = "description"; // Represents null value deserialized from the redis in memory db private static final String DEFAULT_NULL_VALUE = "null"; protected static final String DEFAULT_EMAIL_ADDRESS = "test@harness.com"; private static final ElasticsearchContainer container = new ElasticsearchContainer( DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss") .withTag("6.8.17")); // this should match the client version private static RestClient restClient; // Initialization happens in a static block so the container is initialized // only once for all the sub-class tests in a CI environment // container is stopped when JVM exits // https://www.testcontainers.org/test_framework_integration/manual_lifecycle_control/#singleton-containers static { container.start(); String httpHostAddress = container.getHttpHostAddress(); System.setProperty("conductor.elasticsearch.url", "http://" + httpHostAddress); log.info("Initialized Elasticsearch {}", container.getContainerId()); } @BeforeClass public static void initializeEs() { String httpHostAddress = container.getHttpHostAddress(); String host = httpHostAddress.split(":")[0]; int port = Integer.parseInt(httpHostAddress.split(":")[1]); RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "http")); restClient = restClientBuilder.build(); } @AfterClass public static void cleanupEs() throws Exception { // deletes all indices Response beforeResponse = restClient.performRequest(new Request("GET", "/_cat/indices")); Reader streamReader = new InputStreamReader(beforeResponse.getEntity().getContent()); BufferedReader bufferedReader = new BufferedReader(streamReader); String line; while ((line = bufferedReader.readLine()) != null) { String[] fields = line.split("\\s"); String endpoint = String.format("/%s", fields[2]); restClient.performRequest(new Request("DELETE", endpoint)); } if (restClient != null) { restClient.close(); } } @Test public void testEphemeralWorkflowsWithStoredTasks() { String workflowExecutionName = "testEphemeralWorkflow"; createAndRegisterTaskDefinitions("storedTaskDef", 5); WorkflowDef workflowDefinition = createWorkflowDefinition(workflowExecutionName); WorkflowTask workflowTask1 = createWorkflowTask("storedTaskDef1"); WorkflowTask workflowTask2 = createWorkflowTask("storedTaskDef2"); workflowDefinition.getTasks().addAll(Arrays.asList(workflowTask1, workflowTask2)); String workflowId = startWorkflow(workflowExecutionName, workflowDefinition); assertNotNull(workflowId); Workflow workflow = getWorkflow(workflowId, true); WorkflowDef ephemeralWorkflow = workflow.getWorkflowDefinition(); assertNotNull(ephemeralWorkflow); assertEquals(workflowDefinition, ephemeralWorkflow); } @Test public void testEphemeralWorkflowsWithEphemeralTasks() { String workflowExecutionName = "ephemeralWorkflowWithEphemeralTasks"; WorkflowDef workflowDefinition = createWorkflowDefinition(workflowExecutionName); WorkflowTask workflowTask1 = createWorkflowTask("ephemeralTask1"); TaskDef taskDefinition1 = createTaskDefinition("ephemeralTaskDef1"); workflowTask1.setTaskDefinition(taskDefinition1); WorkflowTask workflowTask2 = createWorkflowTask("ephemeralTask2"); TaskDef taskDefinition2 = createTaskDefinition("ephemeralTaskDef2"); workflowTask2.setTaskDefinition(taskDefinition2); workflowDefinition.getTasks().addAll(Arrays.asList(workflowTask1, workflowTask2)); String workflowId = startWorkflow(workflowExecutionName, workflowDefinition); assertNotNull(workflowId); Workflow workflow = getWorkflow(workflowId, true); WorkflowDef ephemeralWorkflow = workflow.getWorkflowDefinition(); assertNotNull(ephemeralWorkflow); assertEquals(workflowDefinition, ephemeralWorkflow); List ephemeralTasks = ephemeralWorkflow.getTasks(); assertEquals(2, ephemeralTasks.size()); for (WorkflowTask ephemeralTask : ephemeralTasks) { assertNotNull(ephemeralTask.getTaskDefinition()); } } @Test public void testEphemeralWorkflowsWithEphemeralAndStoredTasks() { createAndRegisterTaskDefinitions("storedTask", 1); WorkflowDef workflowDefinition = createWorkflowDefinition("testEphemeralWorkflowsWithEphemeralAndStoredTasks"); WorkflowTask workflowTask1 = createWorkflowTask("ephemeralTask1"); TaskDef taskDefinition1 = createTaskDefinition("ephemeralTaskDef1"); workflowTask1.setTaskDefinition(taskDefinition1); WorkflowTask workflowTask2 = createWorkflowTask("storedTask0"); workflowDefinition.getTasks().add(workflowTask1); workflowDefinition.getTasks().add(workflowTask2); String workflowExecutionName = "ephemeralWorkflowWithEphemeralAndStoredTasks"; String workflowId = startWorkflow(workflowExecutionName, workflowDefinition); assertNotNull(workflowId); Workflow workflow = getWorkflow(workflowId, true); WorkflowDef ephemeralWorkflow = workflow.getWorkflowDefinition(); assertNotNull(ephemeralWorkflow); assertEquals(workflowDefinition, ephemeralWorkflow); TaskDef storedTaskDefinition = getTaskDefinition("storedTask0"); List tasks = ephemeralWorkflow.getTasks(); assertEquals(2, tasks.size()); assertEquals(workflowTask1, tasks.get(0)); TaskDef currentStoredTaskDefinition = tasks.get(1).getTaskDefinition(); assertNotNull(currentStoredTaskDefinition); assertEquals(storedTaskDefinition, currentStoredTaskDefinition); } @Test public void testEventHandler() { String eventName = "conductor:test_workflow:complete_task_with_event"; EventHandler eventHandler = new EventHandler(); eventHandler.setName("test_complete_task_event"); EventHandler.Action completeTaskAction = new EventHandler.Action(); completeTaskAction.setAction(EventHandler.Action.Type.complete_task); completeTaskAction.setComplete_task(new EventHandler.TaskDetails()); completeTaskAction.getComplete_task().setTaskRefName("test_task"); completeTaskAction.getComplete_task().setWorkflowId("test_id"); completeTaskAction.getComplete_task().setOutput(new HashMap<>()); eventHandler.getActions().add(completeTaskAction); eventHandler.setEvent(eventName); eventHandler.setActive(true); registerEventHandler(eventHandler); Iterator it = getEventHandlers(eventName, true); EventHandler result = it.next(); assertFalse(it.hasNext()); assertEquals(eventHandler.getName(), result.getName()); } protected WorkflowTask createWorkflowTask(String name) { WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName(name); workflowTask.setWorkflowTaskType(TaskType.SIMPLE); workflowTask.setTaskReferenceName(name); workflowTask.setDescription(getDefaultDescription(name)); workflowTask.setDynamicTaskNameParam(DEFAULT_NULL_VALUE); workflowTask.setCaseValueParam(DEFAULT_NULL_VALUE); workflowTask.setCaseExpression(DEFAULT_NULL_VALUE); workflowTask.setDynamicForkTasksParam(DEFAULT_NULL_VALUE); workflowTask.setDynamicForkTasksInputParamName(DEFAULT_NULL_VALUE); workflowTask.setSink(DEFAULT_NULL_VALUE); workflowTask.setEvaluatorType(DEFAULT_NULL_VALUE); workflowTask.setExpression(DEFAULT_NULL_VALUE); return workflowTask; } protected TaskDef createTaskDefinition(String name) { TaskDef taskDefinition = new TaskDef(); taskDefinition.setName(name); return taskDefinition; } protected WorkflowDef createWorkflowDefinition(String workflowName) { WorkflowDef workflowDefinition = new WorkflowDef(); workflowDefinition.setName(workflowName); workflowDefinition.setDescription(getDefaultDescription(workflowName)); workflowDefinition.setFailureWorkflow(DEFAULT_NULL_VALUE); workflowDefinition.setOwnerEmail(DEFAULT_EMAIL_ADDRESS); return workflowDefinition; } protected List createAndRegisterTaskDefinitions( String prefixTaskDefinition, int numberOfTaskDefinitions) { String prefix = Optional.ofNullable(prefixTaskDefinition).orElse(TASK_DEFINITION_PREFIX); List definitions = new LinkedList<>(); for (int i = 0; i < numberOfTaskDefinitions; i++) { TaskDef def = new TaskDef( prefix + i, "task " + i + DEFAULT_DESCRIPTION, DEFAULT_EMAIL_ADDRESS, 3, 60, 60); def.setTimeoutPolicy(TaskDef.TimeoutPolicy.RETRY); definitions.add(def); } this.registerTaskDefinitions(definitions); return definitions; } private String getDefaultDescription(String nameResource) { return nameResource + " " + DEFAULT_DESCRIPTION; } protected abstract String startWorkflow( String workflowExecutionName, WorkflowDef workflowDefinition); protected abstract Workflow getWorkflow(String workflowId, boolean includeTasks); protected abstract TaskDef getTaskDefinition(String taskName); protected abstract void registerTaskDefinitions(List taskDefinitionList); protected abstract void registerWorkflowDefinition(WorkflowDef workflowDefinition); protected abstract void registerEventHandler(EventHandler eventHandler); protected abstract Iterator getEventHandlers(String event, boolean activeOnly); } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration.grpc; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; import com.netflix.conductor.client.grpc.WorkflowClient; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskDef.TimeoutPolicy; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.Workflow.WorkflowStatus; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.test.integration.AbstractEndToEndTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @RunWith(SpringRunner.class) @SpringBootTest( properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"}) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest { protected static TaskClient taskClient; protected static WorkflowClient workflowClient; protected static MetadataClient metadataClient; protected static EventClient eventClient; @Override protected String startWorkflow(String workflowExecutionName, WorkflowDef workflowDefinition) { StartWorkflowRequest workflowRequest = new StartWorkflowRequest() .withName(workflowExecutionName) .withWorkflowDef(workflowDefinition); return workflowClient.startWorkflow(workflowRequest); } @Override protected Workflow getWorkflow(String workflowId, boolean includeTasks) { return workflowClient.getWorkflow(workflowId, includeTasks); } @Override protected TaskDef getTaskDefinition(String taskName) { return metadataClient.getTaskDef(taskName); } @Override protected void registerTaskDefinitions(List taskDefinitionList) { metadataClient.registerTaskDefs(taskDefinitionList); } @Override protected void registerWorkflowDefinition(WorkflowDef workflowDefinition) { metadataClient.registerWorkflowDef(workflowDefinition); } @Override protected void registerEventHandler(EventHandler eventHandler) { eventClient.registerEventHandler(eventHandler); } @Override protected Iterator getEventHandlers(String event, boolean activeOnly) { return eventClient.getEventHandlers(event, activeOnly); } @Test public void testAll() throws Exception { assertNotNull(taskClient); List defs = new LinkedList<>(); for (int i = 0; i < 5; i++) { TaskDef def = new TaskDef("t" + i, "task " + i, DEFAULT_EMAIL_ADDRESS, 3, 60, 60); def.setTimeoutPolicy(TimeoutPolicy.RETRY); defs.add(def); } metadataClient.registerTaskDefs(defs); for (int i = 0; i < 5; i++) { final String taskName = "t" + i; TaskDef def = metadataClient.getTaskDef(taskName); assertNotNull(def); assertEquals(taskName, def.getName()); } WorkflowDef def = createWorkflowDefinition("test"); WorkflowTask t0 = createWorkflowTask("t0"); WorkflowTask t1 = createWorkflowTask("t1"); def.getTasks().add(t0); def.getTasks().add(t1); metadataClient.registerWorkflowDef(def); WorkflowDef found = metadataClient.getWorkflowDef(def.getName(), null); assertNotNull(found); assertEquals(def, found); String correlationId = "test_corr_id"; StartWorkflowRequest startWf = new StartWorkflowRequest(); startWf.setName(def.getName()); startWf.setCorrelationId(correlationId); String workflowId = workflowClient.startWorkflow(startWf); assertNotNull(workflowId); Workflow workflow = workflowClient.getWorkflow(workflowId, false); assertEquals(0, workflow.getTasks().size()); assertEquals(workflowId, workflow.getWorkflowId()); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(1, workflow.getTasks().size()); assertEquals(t0.getTaskReferenceName(), workflow.getTasks().get(0).getReferenceTaskName()); assertEquals(workflowId, workflow.getWorkflowId()); List runningIds = workflowClient.getRunningWorkflow(def.getName(), def.getVersion()); assertNotNull(runningIds); assertEquals(1, runningIds.size()); assertEquals(workflowId, runningIds.get(0)); List polled = taskClient.batchPollTasksByTaskType("non existing task", "test", 1, 100); assertNotNull(polled); assertEquals(0, polled.size()); polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); assertNotNull(polled); assertEquals(1, polled.size()); assertEquals(t0.getName(), polled.get(0).getTaskDefName()); Task task = polled.get(0); task.getOutputData().put("key1", "value1"); task.setStatus(Status.COMPLETED); taskClient.updateTask(new TaskResult(task)); polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); assertNotNull(polled); assertTrue(polled.toString(), polled.isEmpty()); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(2, workflow.getTasks().size()); assertEquals(t0.getTaskReferenceName(), workflow.getTasks().get(0).getReferenceTaskName()); assertEquals(t1.getTaskReferenceName(), workflow.getTasks().get(1).getReferenceTaskName()); assertEquals(Status.COMPLETED, workflow.getTasks().get(0).getStatus()); assertEquals(Status.SCHEDULED, workflow.getTasks().get(1).getStatus()); Task taskById = taskClient.getTaskDetails(task.getTaskId()); assertNotNull(taskById); assertEquals(task.getTaskId(), taskById.getTaskId()); Thread.sleep(1000); SearchResult searchResult = workflowClient.search("workflowType='" + def.getName() + "'"); assertNotNull(searchResult); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow.getWorkflowId(), searchResult.getResults().get(0).getWorkflowId()); SearchResult searchResultV2 = workflowClient.searchV2("workflowType='" + def.getName() + "'"); assertNotNull(searchResultV2); assertEquals(1, searchResultV2.getTotalHits()); assertEquals(workflow.getWorkflowId(), searchResultV2.getResults().get(0).getWorkflowId()); SearchResult searchResultAdvanced = workflowClient.search(0, 1, null, null, "workflowType='" + def.getName() + "'"); assertNotNull(searchResultAdvanced); assertEquals(1, searchResultAdvanced.getTotalHits()); assertEquals( workflow.getWorkflowId(), searchResultAdvanced.getResults().get(0).getWorkflowId()); SearchResult searchResultV2Advanced = workflowClient.searchV2(0, 1, null, null, "workflowType='" + def.getName() + "'"); assertNotNull(searchResultV2Advanced); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals( workflow.getWorkflowId(), searchResultV2Advanced.getResults().get(0).getWorkflowId()); SearchResult taskSearchResult = taskClient.search("taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResult); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals(t0.getName(), taskSearchResult.getResults().get(0).getTaskDefName()); SearchResult taskSearchResultAdvanced = taskClient.search(0, 1, null, null, "taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultAdvanced); assertEquals(1, taskSearchResultAdvanced.getTotalHits()); assertEquals(t0.getName(), taskSearchResultAdvanced.getResults().get(0).getTaskDefName()); SearchResult taskSearchResultV2 = taskClient.searchV2("taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultV2); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals( t0.getTaskReferenceName(), taskSearchResultV2.getResults().get(0).getReferenceTaskName()); SearchResult taskSearchResultV2Advanced = taskClient.searchV2(0, 1, null, null, "taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultV2Advanced); assertEquals(1, taskSearchResultV2Advanced.getTotalHits()); assertEquals( t0.getTaskReferenceName(), taskSearchResultV2Advanced.getResults().get(0).getReferenceTaskName()); workflowClient.terminateWorkflow(workflowId, "terminate reason"); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.TERMINATED, workflow.getStatus()); workflowClient.restart(workflowId, false); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(1, workflow.getTasks().size()); } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/GrpcEndToEndTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration.grpc; import org.junit.Before; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; import com.netflix.conductor.client.grpc.WorkflowClient; public class GrpcEndToEndTest extends AbstractGrpcEndToEndTest { @Before public void init() { taskClient = new TaskClient("localhost", 8092); workflowClient = new WorkflowClient("localhost", 8092); metadataClient = new MetadataClient("localhost", 8092); eventClient = new EventClient("localhost", 8092); } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java ================================================ /* * Copyright 2020 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration.http; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.boot.web.server.LocalServerPort; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.EventClient; import com.netflix.conductor.client.http.MetadataClient; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.http.WorkflowClient; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.Task.Status; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.Workflow; import com.netflix.conductor.common.run.Workflow.WorkflowStatus; import com.netflix.conductor.common.run.WorkflowSummary; import com.netflix.conductor.common.validation.ValidationError; import com.netflix.conductor.test.integration.AbstractEndToEndTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractHttpEndToEndTest extends AbstractEndToEndTest { @LocalServerPort protected int port; protected static String apiRoot; protected static TaskClient taskClient; protected static WorkflowClient workflowClient; protected static MetadataClient metadataClient; protected static EventClient eventClient; @Override protected String startWorkflow(String workflowExecutionName, WorkflowDef workflowDefinition) { StartWorkflowRequest workflowRequest = new StartWorkflowRequest() .withName(workflowExecutionName) .withWorkflowDef(workflowDefinition); return workflowClient.startWorkflow(workflowRequest); } @Override protected Workflow getWorkflow(String workflowId, boolean includeTasks) { return workflowClient.getWorkflow(workflowId, includeTasks); } @Override protected TaskDef getTaskDefinition(String taskName) { return metadataClient.getTaskDef(taskName); } @Override protected void registerTaskDefinitions(List taskDefinitionList) { metadataClient.registerTaskDefs(taskDefinitionList); } @Override protected void registerWorkflowDefinition(WorkflowDef workflowDefinition) { metadataClient.registerWorkflowDef(workflowDefinition); } @Override protected void registerEventHandler(EventHandler eventHandler) { eventClient.registerEventHandler(eventHandler); } @Override protected Iterator getEventHandlers(String event, boolean activeOnly) { return eventClient.getEventHandlers(event, activeOnly).iterator(); } @Test public void testAll() throws Exception { createAndRegisterTaskDefinitions("t", 5); WorkflowDef def = new WorkflowDef(); def.setName("test"); def.setOwnerEmail(DEFAULT_EMAIL_ADDRESS); WorkflowTask t0 = new WorkflowTask(); t0.setName("t0"); t0.setWorkflowTaskType(TaskType.SIMPLE); t0.setTaskReferenceName("t0"); WorkflowTask t1 = new WorkflowTask(); t1.setName("t1"); t1.setWorkflowTaskType(TaskType.SIMPLE); t1.setTaskReferenceName("t1"); def.getTasks().add(t0); def.getTasks().add(t1); metadataClient.registerWorkflowDef(def); WorkflowDef workflowDefinitionFromSystem = metadataClient.getWorkflowDef(def.getName(), null); assertNotNull(workflowDefinitionFromSystem); assertEquals(def, workflowDefinitionFromSystem); String correlationId = "test_corr_id"; StartWorkflowRequest startWorkflowRequest = new StartWorkflowRequest() .withName(def.getName()) .withCorrelationId(correlationId) .withPriority(50) .withInput(new HashMap<>()); String workflowId = workflowClient.startWorkflow(startWorkflowRequest); assertNotNull(workflowId); Workflow workflow = workflowClient.getWorkflow(workflowId, false); assertEquals(0, workflow.getTasks().size()); assertEquals(workflowId, workflow.getWorkflowId()); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(1, workflow.getTasks().size()); assertEquals(t0.getTaskReferenceName(), workflow.getTasks().get(0).getReferenceTaskName()); assertEquals(workflowId, workflow.getWorkflowId()); int queueSize = taskClient.getQueueSizeForTask(workflow.getTasks().get(0).getTaskType()); assertEquals(1, queueSize); List runningIds = workflowClient.getRunningWorkflow(def.getName(), def.getVersion()); assertNotNull(runningIds); assertEquals(1, runningIds.size()); assertEquals(workflowId, runningIds.get(0)); List polled = taskClient.batchPollTasksByTaskType("non existing task", "test", 1, 100); assertNotNull(polled); assertEquals(0, polled.size()); polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); assertNotNull(polled); assertEquals(1, polled.size()); assertEquals(t0.getName(), polled.get(0).getTaskDefName()); Task task = polled.get(0); task.getOutputData().put("key1", "value1"); task.setStatus(Status.COMPLETED); taskClient.updateTask(new TaskResult(task)); polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); assertNotNull(polled); assertTrue(polled.toString(), polled.isEmpty()); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(2, workflow.getTasks().size()); assertEquals(t0.getTaskReferenceName(), workflow.getTasks().get(0).getReferenceTaskName()); assertEquals(t1.getTaskReferenceName(), workflow.getTasks().get(1).getReferenceTaskName()); assertEquals(Task.Status.COMPLETED, workflow.getTasks().get(0).getStatus()); assertEquals(Task.Status.SCHEDULED, workflow.getTasks().get(1).getStatus()); Task taskById = taskClient.getTaskDetails(task.getTaskId()); assertNotNull(taskById); assertEquals(task.getTaskId(), taskById.getTaskId()); queueSize = taskClient.getQueueSizeForTask(workflow.getTasks().get(1).getTaskType()); assertEquals(1, queueSize); Thread.sleep(1000); SearchResult searchResult = workflowClient.search("workflowType='" + def.getName() + "'"); assertNotNull(searchResult); assertEquals(1, searchResult.getTotalHits()); assertEquals(workflow.getWorkflowId(), searchResult.getResults().get(0).getWorkflowId()); SearchResult searchResultV2 = workflowClient.searchV2("workflowType='" + def.getName() + "'"); assertNotNull(searchResultV2); assertEquals(1, searchResultV2.getTotalHits()); assertEquals(workflow.getWorkflowId(), searchResultV2.getResults().get(0).getWorkflowId()); SearchResult searchResultAdvanced = workflowClient.search(0, 1, null, null, "workflowType='" + def.getName() + "'"); assertNotNull(searchResultAdvanced); assertEquals(1, searchResultAdvanced.getTotalHits()); assertEquals( workflow.getWorkflowId(), searchResultAdvanced.getResults().get(0).getWorkflowId()); SearchResult searchResultV2Advanced = workflowClient.searchV2(0, 1, null, null, "workflowType='" + def.getName() + "'"); assertNotNull(searchResultV2Advanced); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals( workflow.getWorkflowId(), searchResultV2Advanced.getResults().get(0).getWorkflowId()); SearchResult taskSearchResult = taskClient.search("taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResult); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals(t0.getName(), taskSearchResult.getResults().get(0).getTaskDefName()); SearchResult taskSearchResultAdvanced = taskClient.search(0, 1, null, null, "taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultAdvanced); assertEquals(1, taskSearchResultAdvanced.getTotalHits()); assertEquals(t0.getName(), taskSearchResultAdvanced.getResults().get(0).getTaskDefName()); SearchResult taskSearchResultV2 = taskClient.searchV2("taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultV2); assertEquals(1, searchResultV2Advanced.getTotalHits()); assertEquals( t0.getTaskReferenceName(), taskSearchResultV2.getResults().get(0).getReferenceTaskName()); SearchResult taskSearchResultV2Advanced = taskClient.searchV2(0, 1, null, null, "taskType='" + t0.getName() + "'"); assertNotNull(taskSearchResultV2Advanced); assertEquals(1, taskSearchResultV2Advanced.getTotalHits()); assertEquals( t0.getTaskReferenceName(), taskSearchResultV2Advanced.getResults().get(0).getReferenceTaskName()); workflowClient.terminateWorkflow(workflowId, "terminate reason"); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.TERMINATED, workflow.getStatus()); workflowClient.restart(workflowId, false); workflow = workflowClient.getWorkflow(workflowId, true); assertNotNull(workflow); assertEquals(WorkflowStatus.RUNNING, workflow.getStatus()); assertEquals(1, workflow.getTasks().size()); workflowClient.skipTaskFromWorkflow(workflowId, "t1"); } @Test(expected = ConductorClientException.class) public void testMetadataWorkflowDefinition() { String workflowDefName = "testWorkflowDefMetadata"; WorkflowDef def = new WorkflowDef(); def.setName(workflowDefName); def.setVersion(1); WorkflowTask t0 = new WorkflowTask(); t0.setName("t0"); t0.setWorkflowTaskType(TaskType.SIMPLE); t0.setTaskReferenceName("t0"); WorkflowTask t1 = new WorkflowTask(); t1.setName("t1"); t1.setWorkflowTaskType(TaskType.SIMPLE); t1.setTaskReferenceName("t1"); def.getTasks().add(t0); def.getTasks().add(t1); metadataClient.registerWorkflowDef(def); metadataClient.unregisterWorkflowDef(workflowDefName, 1); try { metadataClient.getWorkflowDef(workflowDefName, 1); } catch (ConductorClientException e) { int statusCode = e.getStatus(); String errorMessage = e.getMessage(); boolean retryable = e.isRetryable(); assertEquals(404, statusCode); assertEquals( "No such workflow found by name: testWorkflowDefMetadata, version: 1", errorMessage); assertFalse(retryable); throw e; } } @Test(expected = ConductorClientException.class) public void testInvalidResource() { MetadataClient metadataClient = new MetadataClient(); metadataClient.setRootURI(String.format("%sinvalid", apiRoot)); WorkflowDef def = new WorkflowDef(); def.setName("testWorkflowDel"); def.setVersion(1); try { metadataClient.registerWorkflowDef(def); } catch (ConductorClientException e) { int statusCode = e.getStatus(); boolean retryable = e.isRetryable(); assertEquals(404, statusCode); assertFalse(retryable); throw e; } } @Test(expected = ConductorClientException.class) public void testUpdateWorkflow() { TaskDef taskDef = new TaskDef(); taskDef.setName("taskUpdate"); ArrayList tasks = new ArrayList<>(); tasks.add(taskDef); metadataClient.registerTaskDefs(tasks); WorkflowDef def = new WorkflowDef(); def.setName("testWorkflowDel"); def.setVersion(1); WorkflowTask workflowTask = new WorkflowTask(); workflowTask.setName("taskUpdate"); workflowTask.setTaskReferenceName("taskUpdate"); List workflowTaskList = new ArrayList<>(); workflowTaskList.add(workflowTask); def.setTasks(workflowTaskList); List workflowList = new ArrayList<>(); workflowList.add(def); metadataClient.registerWorkflowDef(def); def.setVersion(2); metadataClient.updateWorkflowDefs(workflowList); WorkflowDef def1 = metadataClient.getWorkflowDef(def.getName(), 2); assertNotNull(def1); try { metadataClient.getTaskDef("test"); } catch (ConductorClientException e) { int statuCode = e.getStatus(); assertEquals(404, statuCode); assertEquals("No such taskType found by name: test", e.getMessage()); assertFalse(e.isRetryable()); throw e; } } @Test public void testStartWorkflow() { StartWorkflowRequest startWorkflowRequest = new StartWorkflowRequest(); try { workflowClient.startWorkflow(startWorkflowRequest); fail("StartWorkflow#name is null but NullPointerException was not thrown"); } catch (NullPointerException e) { assertEquals("Workflow name cannot be null or empty", e.getMessage()); } catch (Exception e) { fail("StartWorkflow#name is null but NullPointerException was not thrown"); } } @Test(expected = ConductorClientException.class) public void testUpdateTask() { TaskResult taskResult = new TaskResult(); try { taskClient.updateTask(taskResult); } catch (ConductorClientException e) { assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertEquals(2, errors.size()); assertTrue(errorMessages.contains("Workflow Id cannot be null or empty")); throw e; } } @Test(expected = ConductorClientException.class) public void testGetWorfklowNotFound() { try { workflowClient.getWorkflow("w123", true); } catch (ConductorClientException e) { assertEquals(404, e.getStatus()); assertEquals("No such workflow found by id: w123", e.getMessage()); assertFalse(e.isRetryable()); throw e; } } @Test(expected = ConductorClientException.class) public void testEmptyCreateWorkflowDef() { try { WorkflowDef workflowDef = new WorkflowDef(); metadataClient.registerWorkflowDef(workflowDef); } catch (ConductorClientException e) { assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertTrue(errorMessages.contains("WorkflowDef name cannot be null or empty")); assertTrue(errorMessages.contains("WorkflowTask list cannot be empty")); throw e; } } @Test(expected = ConductorClientException.class) public void testUpdateWorkflowDef() { try { WorkflowDef workflowDef = new WorkflowDef(); List workflowDefList = new ArrayList<>(); workflowDefList.add(workflowDef); metadataClient.updateWorkflowDefs(workflowDefList); } catch (ConductorClientException e) { assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertEquals(3, errors.size()); assertTrue(errorMessages.contains("WorkflowTask list cannot be empty")); assertTrue(errorMessages.contains("WorkflowDef name cannot be null or empty")); assertTrue(errorMessages.contains("ownerEmail cannot be empty")); throw e; } } @Test public void testTaskByTaskId() { try { taskClient.getTaskDetails("test999"); } catch (ConductorClientException e) { assertEquals(404, e.getStatus()); assertEquals("No such task found by taskId: test999", e.getMessage()); } } @Test public void testListworkflowsByCorrelationId() { workflowClient.getWorkflows("test", "test12", false, false); } @Test(expected = ConductorClientException.class) public void testCreateInvalidWorkflowDef() { try { WorkflowDef workflowDef = new WorkflowDef(); List workflowDefList = new ArrayList<>(); workflowDefList.add(workflowDef); metadataClient.registerWorkflowDef(workflowDef); } catch (ConductorClientException e) { assertEquals(3, e.getValidationErrors().size()); assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertTrue(errorMessages.contains("WorkflowDef name cannot be null or empty")); assertTrue(errorMessages.contains("WorkflowTask list cannot be empty")); assertTrue(errorMessages.contains("ownerEmail cannot be empty")); throw e; } } @Test(expected = ConductorClientException.class) public void testUpdateTaskDefNameNull() { TaskDef taskDef = new TaskDef(); try { metadataClient.updateTaskDef(taskDef); } catch (ConductorClientException e) { assertEquals(2, e.getValidationErrors().size()); assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertTrue(errorMessages.contains("TaskDef name cannot be null or empty")); assertTrue(errorMessages.contains("ownerEmail cannot be empty")); throw e; } } @Test(expected = IllegalArgumentException.class) public void testGetTaskDefNotExisting() { metadataClient.getTaskDef(""); } @Test(expected = ConductorClientException.class) public void testUpdateWorkflowDefNameNull() { WorkflowDef workflowDef = new WorkflowDef(); List list = new ArrayList<>(); list.add(workflowDef); try { metadataClient.updateWorkflowDefs(list); } catch (ConductorClientException e) { assertEquals(3, e.getValidationErrors().size()); assertEquals(400, e.getStatus()); assertEquals("Validation failed, check below errors for detail.", e.getMessage()); assertFalse(e.isRetryable()); List errors = e.getValidationErrors(); List errorMessages = errors.stream().map(ValidationError::getMessage).collect(Collectors.toList()); assertTrue(errorMessages.contains("WorkflowDef name cannot be null or empty")); assertTrue(errorMessages.contains("WorkflowTask list cannot be empty")); assertTrue(errorMessages.contains("ownerEmail cannot be empty")); throw e; } } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/integration/http/HttpEndToEndTest.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.integration.http; import org.junit.Before; import com.netflix.conductor.client.http.EventClient; import com.netflix.conductor.client.http.MetadataClient; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.http.WorkflowClient; public class HttpEndToEndTest extends AbstractHttpEndToEndTest { @Before public void init() { apiRoot = String.format("http://localhost:%d/api/", port); taskClient = new TaskClient(); taskClient.setRootURI(apiRoot); workflowClient = new WorkflowClient(); workflowClient.setRootURI(apiRoot); metadataClient = new MetadataClient(); metadataClient.setRootURI(apiRoot); eventClient = new EventClient(); eventClient.setRootURI(apiRoot); } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/utils/MockExternalPayloadStorage.java ================================================ /* * Copyright 2021 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.utils; import java.io.*; import java.nio.file.Files; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.fasterxml.jackson.databind.ObjectMapper; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SIMPLE; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; /** A {@link ExternalPayloadStorage} implementation that stores payload in file. */ @ConditionalOnProperty(name = "conductor.external-payload-storage.type", havingValue = "mock") @Component public class MockExternalPayloadStorage implements ExternalPayloadStorage { private static final Logger LOGGER = LoggerFactory.getLogger(MockExternalPayloadStorage.class); private final ObjectMapper objectMapper; private final File payloadDir; @Autowired public MockExternalPayloadStorage(ObjectMapper objectMapper) throws IOException { this.objectMapper = objectMapper; this.payloadDir = Files.createTempDirectory("payloads").toFile(); LOGGER.info( "{} initialized in directory: {}", this.getClass().getSimpleName(), payloadDir.getAbsolutePath()); } @Override public ExternalStorageLocation getLocation( Operation operation, PayloadType payloadType, String path) { ExternalStorageLocation location = new ExternalStorageLocation(); location.setPath(UUID.randomUUID() + ".json"); return location; } @Override public void upload(String path, InputStream payload, long payloadSize) { File file = new File(payloadDir, path); String filePath = file.getAbsolutePath(); try { if (!file.exists() && file.createNewFile()) { LOGGER.debug("Created file: {}", filePath); } IOUtils.copy(payload, new FileOutputStream(file)); LOGGER.debug("Written to {}", filePath); } catch (IOException e) { // just handle this exception here and return empty map so that test will fail in case // this exception is thrown LOGGER.error("Error writing to {}", filePath); } finally { try { if (payload != null) { payload.close(); } } catch (IOException e) { LOGGER.warn("Unable to close input stream when writing to file"); } } } @Override public InputStream download(String path) { try { LOGGER.debug("Reading from {}", path); return new FileInputStream(new File(payloadDir, path)); } catch (IOException e) { LOGGER.error("Error reading {}", path, e); return null; } } public void upload(String path, Map payload) { try { InputStream bais = new ByteArrayInputStream(objectMapper.writeValueAsBytes(payload)); upload(path, bais, 0); } catch (IOException e) { LOGGER.error("Error serializing map to json", e); } } public InputStream readOutputDotJson() { return MockExternalPayloadStorage.class.getResourceAsStream("/output.json"); } @SuppressWarnings("unchecked") public Map curateDynamicForkLargePayload() { Map dynamicForkLargePayload = new HashMap<>(); try { InputStream inputStream = readOutputDotJson(); Map largePayload = objectMapper.readValue(inputStream, Map.class); WorkflowTask simpleWorkflowTask = new WorkflowTask(); simpleWorkflowTask.setName("integration_task_10"); simpleWorkflowTask.setTaskReferenceName("t10"); simpleWorkflowTask.setType(TASK_TYPE_SIMPLE); simpleWorkflowTask.setInputParameters( Collections.singletonMap("p1", "${workflow.input.imageType}")); WorkflowDef subWorkflowDef = new WorkflowDef(); subWorkflowDef.setName("one_task_workflow"); subWorkflowDef.setVersion(1); subWorkflowDef.setTasks(Collections.singletonList(simpleWorkflowTask)); SubWorkflowParams subWorkflowParams = new SubWorkflowParams(); subWorkflowParams.setName("one_task_workflow"); subWorkflowParams.setVersion(1); subWorkflowParams.setWorkflowDef(subWorkflowDef); WorkflowTask subWorkflowTask = new WorkflowTask(); subWorkflowTask.setName("large_payload_subworkflow"); subWorkflowTask.setType(TASK_TYPE_SUB_WORKFLOW); subWorkflowTask.setTaskReferenceName("large_payload_subworkflow"); subWorkflowTask.setInputParameters(largePayload); subWorkflowTask.setSubWorkflowParam(subWorkflowParams); dynamicForkLargePayload.put("dynamicTasks", List.of(subWorkflowTask)); dynamicForkLargePayload.put( "dynamicTasksInput", Map.of("large_payload_subworkflow", largePayload)); } catch (IOException e) { // just handle this exception here and return empty map so that test will fail in case // this exception is thrown } return dynamicForkLargePayload; } public Map downloadPayload(String path) { InputStream inputStream = download(path); if (inputStream != null) { try { Map largePayload = objectMapper.readValue(inputStream, Map.class); return largePayload; } catch (IOException e) { LOGGER.error("Error in downloading payload for path {}", path, e); } } return new HashMap<>(); } public Map createLargePayload(int repeat) { Map largePayload = new HashMap<>(); try { InputStream inputStream = readOutputDotJson(); Map payload = objectMapper.readValue(inputStream, Map.class); for (int i = 0; i < repeat; i++) { largePayload.put(String.valueOf(i), payload); } } catch (IOException e) { // just handle this exception here and return empty map so that test will fail in case // this exception is thrown } return largePayload; } } ================================================ FILE: test-harness/src/test/java/com/netflix/conductor/test/utils/UserTask.java ================================================ /* * Copyright 2022 Netflix, Inc. *

    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.netflix.conductor.test.utils; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Uninterruptibles; @Component(UserTask.NAME) public class UserTask extends WorkflowSystemTask { private static final Logger LOGGER = LoggerFactory.getLogger(UserTask.class); public static final String NAME = "USER_TASK"; private final ObjectMapper objectMapper; private static final TypeReference>>> mapStringListObjects = new TypeReference<>() {}; @Autowired public UserTask(ObjectMapper objectMapper) { super(NAME); this.objectMapper = objectMapper; LOGGER.info("Initialized system task - {}", getClass().getCanonicalName()); } @Override public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); if (task.getWorkflowTask().isAsyncComplete()) { task.setStatus(TaskModel.Status.IN_PROGRESS); } else { Map>> map = objectMapper.convertValue(task.getInputData(), mapStringListObjects); Map output = new HashMap<>(); Map> defaultLargeInput = new HashMap<>(); defaultLargeInput.put("TEST_SAMPLE", Collections.singletonList("testDefault")); output.put( "size", map.getOrDefault("largeInput", defaultLargeInput).get("TEST_SAMPLE").size()); task.setOutputData(output); task.setStatus(TaskModel.Status.COMPLETED); } } @Override public boolean isAsync() { return true; } } ================================================ FILE: test-harness/src/test/resources/application-integrationtest.properties ================================================ # # /* # * Copyright 2021 Netflix, Inc. # *

    # * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with # * the License. You may obtain a copy of the License at # *

    # * http://www.apache.org/licenses/LICENSE-2.0 # *

    # * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on # * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the # * specific language governing permissions and limitations under the License. # */ # conductor.db.type=memory conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false conductor.app.stack=test conductor.app.appId=conductor conductor.app.workflow-offset-timeout=30s conductor.system-task-workers.enabled=false conductor.app.system-task-worker-callback-duration=0 conductor.app.event-message-indexing-enabled=true conductor.app.event-execution-indexing-enabled=true conductor.workflow-reconciler.enabled=true conductor.workflow-repair-service.enabled=false conductor.app.workflow-execution-lock-enabled=false conductor.app.workflow-input-payload-size-threshold=10KB conductor.app.max-workflow-input-payload-size-threshold=10240KB conductor.app.workflow-output-payload-size-threshold=10KB conductor.app.max-workflow-output-payload-size-threshold=10240KB conductor.app.task-input-payload-size-threshold=10KB conductor.app.max-task-input-payload-size-threshold=10240KB conductor.app.task-output-payload-size-threshold=10KB conductor.app.max-task-output-payload-size-threshold=10240KB conductor.app.max-workflow-variables-payload-size-threshold=2KB conductor.redis.availability-zone=us-east-1c conductor.redis.data-center-region=us-east-1 conductor.redis.workflow-namespace-prefix=integration-test conductor.redis.queue-namespace-prefix=integtest conductor.elasticsearch.index-prefix=conductor conductor.elasticsearch.cluster-health-color=yellow management.metrics.export.datadog.enabled=false ================================================ FILE: test-harness/src/test/resources/concurrency_limited_task_workflow_integration_test.json ================================================ { "name": "test_concurrency_limits_workflow", "version": 1, "tasks": [ { "name": "test_task_with_concurrency_limit", "taskReferenceName": "test_task_with_concurrency_limit", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/conditional_switch_task_workflow_integration_test.json ================================================ { "name": "ConditionalTaskWF", "description": "ConditionalTaskWF", "version": 1, "tasks": [ { "name": "conditional", "taskReferenceName": "conditional", "inputParameters": { "case": "${workflow.input.param1}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "nested": [ { "name": "nestedCondition", "taskReferenceName": "nestedCondition", "inputParameters": { "case": "${workflow.input.param2}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "one": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "three": [ { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp3": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_10", "taskReferenceName": "t10", "inputParameters": { "tp10": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "finalcondition", "taskReferenceName": "finalCase", "inputParameters": { "finalCase": "${workflow.input.finalCase}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "finalCase", "decisionCases": { "notify": [ { "name": "integration_task_4", "taskReferenceName": "integration_task_4", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/conditional_system_task_workflow_integration_test.json ================================================ { "name": "ConditionalSystemWorkflow", "description": "ConditionalSystemWorkflow", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "tp11": "${workflow.input.param1}", "tp12": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decision", "taskReferenceName": "decision", "inputParameters": { "case": "${t1.output.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "one": [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp21": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "user_task", "taskReferenceName": "user_task", "inputParameters": { "largeInput": "${t1.output.op}" }, "type": "USER_TASK", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp31": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o2": "${t1.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/conditional_task_workflow_integration_test.json ================================================ { "name": "ConditionalTaskWF", "description": "ConditionalTaskWF", "version": 1, "tasks": [ { "name": "conditional", "taskReferenceName": "conditional", "inputParameters": { "case": "${workflow.input.param1}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "nested": [ { "name": "nestedCondition", "taskReferenceName": "nestedCondition", "inputParameters": { "case": "${workflow.input.param2}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "one": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "three": [ { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp3": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_10", "taskReferenceName": "t10", "inputParameters": { "tp10": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "finalcondition", "taskReferenceName": "finalCase", "inputParameters": { "finalCase": "${workflow.input.finalCase}" }, "type": "DECISION", "caseValueParam": "finalCase", "decisionCases": { "notify": [ { "name": "integration_task_4", "taskReferenceName": "integration_task_4", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/decision_and_fork_join_integration_test.json ================================================ { "name": "ForkConditionalTest", "description": "ForkConditionalTest", "version": 1, "tasks": [ { "name": "forkTask", "taskReferenceName": "forkTask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "decisionTask", "taskReferenceName": "decisionTask", "inputParameters": { "case": "${workflow.input.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "c": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_5", "taskReferenceName": "t5", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_10", "taskReferenceName": "t10", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "joinTask", "taskReferenceName": "joinTask", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t20", "t10" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/decision_and_terminate_integration_test.json ================================================ { "name": "ConditionalTerminateWorkflow", "description": "ConditionalTerminateWorkflow", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "tp11": "${workflow.input.param1}", "tp12": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decision", "taskReferenceName": "decision", "inputParameters": { "case": "${workflow.input.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "one": [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp21": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "FAILED", "workflowOutput": "${t1.output.op}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp31": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o2": "${t3.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_as_subtask_integration_test.json ================================================ { "name": "Do_While_SubTask", "description": "Do_While_SubTask", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fork", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", "loopOver": [ { "name": "integration_task_0", "taskReferenceName": "integration_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] } ], [ { "name": "integration_task_2", "taskReferenceName": "integration_task_2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "loopTask", "integration_task_2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_five_loop_over_integration_test.json ================================================ { "name": "do_while_five_loop_over_integration_test", "description": "do_while with a mix of 5, simple and system tasks", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value ) { true;} else {false;} ", "loopOver": [ { "name": "LAMBDA_TASK", "taskReferenceName": "lambda_locs", "inputParameters": { "scriptExpression": "return {locationRanId: 'some location id'}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "jq_add_location", "taskReferenceName": "jq_add_location", "inputParameters": { "locationIdValue": "${lambda_locs.output.result.locationRanId}", "queryExpression": "{ out: ({ \"locationId\": .locationIdValue }) }" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "jq_create_hydrus_input", "taskReferenceName": "jq_create_hydrus_input", "inputParameters": { "locationIdValue": "${lambda_locs.output.result.locationRanId}", "queryExpression": "{ out: ({ \"locationId\": .locationIdValue }) }" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "integration_task_2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, { "name": "integration_task_3", "taskReferenceName": "integration_task_3", "inputParameters": {}, "type": "SIMPLE" } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_integration_test.json ================================================ { "name": "Do_While_Workflow", "description": "Do_While_Workflow", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", "loopOver": [ { "name": "integration_task_0", "taskReferenceName": "integration_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork", "taskReferenceName": "fork", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_2", "taskReferenceName": "integration_task_2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "integration_task_1", "integration_task_2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_iteration_fix_test.json ================================================ { "name": "Do_While_Workflow_Iteration_Fix", "description": "Do_While_Workflow_Iteration_Fix", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", "loopOver": [ { "name": "form_uri", "taskReferenceName": "form_uri", "inputParameters": { "index" : "${loopTask['iteration']}", "scriptExpression": "return $.index - 1;" }, "type": "LAMBDA" } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_multiple_integration_test.json ================================================ { "name": "Do_While_Multiple", "description": "Do_While_Multiple", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value ) { true;} else {false;} ", "loopOver": [ { "name": "integration_task_0", "taskReferenceName": "integration_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork", "taskReferenceName": "fork", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_2", "taskReferenceName": "integration_task_2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "integration_task_1", "integration_task_2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, { "name": "loopTask2", "taskReferenceName": "loopTask2", "inputParameters": { "value": "${workflow.input.loop2}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask2['iteration'] < $.value) { true; } else { false; }", "loopOver": [ { "name": "integration_task_3", "taskReferenceName": "integration_task_3", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_set_variable_fix.json ================================================ { "name": "do_while_Set_variable_fix", "description": "do_while with set variable task fix", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.variables.value}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.value > 0) { true; } else { false; } ", "loopOver": [ { "name": "set_variable", "taskReferenceName": "set_variable", "inputParameters": { "value": "0" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_sub_workflow_integration_test.json ================================================ { "name": "Do_While_Sub_Workflow", "description": "Do_While_Sub_Workflow", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", "loopOver": [ { "name": "integration_task_0", "taskReferenceName": "integration_task_0", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork", "taskReferenceName": "fork", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_2", "taskReferenceName": "integration_task_2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "integration_task_1", "integration_task_2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sub_workflow_task", "taskReferenceName": "st1", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_workflow" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_system_tasks.json ================================================ { "name": "do_while_system_tasks", "description": "do_while with a mix of 5, simple and system tasks", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value ) { true;} else {false;} ", "loopOver": [ { "name": "LAMBDA_TASK", "taskReferenceName": "lambda_locs", "inputParameters": { "scriptExpression": "return {locationRanId: 'some location id'}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "jq_add_location", "taskReferenceName": "jq_add_location", "inputParameters": { "locationIdValue": "${lambda_locs.output.result.locationRanId}", "queryExpression": "{ out: ({ \"locationId\": .locationIdValue }) }" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "jq_create_hydrus_input", "taskReferenceName": "jq_create_hydrus_input", "inputParameters": { "locationIdValue": "${lambda_locs.output.result.locationRanId}", "queryExpression": "{ out: ({ \"locationId\": .locationIdValue }) }" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, { "name": "integration_task_1", "taskReferenceName": "integration_task_1", "inputParameters": {}, "type": "SIMPLE" } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/do_while_with_decision_task.json ================================================ { "name": "DO_While_with_Decision_task", "description": "Program for testing loop behaviour", "version": 1, "schemaVersion": 2, "ownerEmail": "xyz@company.eu", "tasks": [ { "name": "LoopTask", "taskReferenceName": "LoopTask", "type": "DO_WHILE", "inputParameters": { "list": "${workflow.input.list}" }, "loopCondition": "$.LoopTask['iteration'] < $.list.length", "loopOver": [ { "name": "GetNumberAtIndex", "taskReferenceName": "GetNumberAtIndex", "type": "INLINE", "inputParameters": { "evaluatorType": "javascript", "list": "${workflow.input.list}", "iterator": "${LoopTask.output.iteration}", "expression": "function getElement() { return $.list.get($.iterator - 1); } getElement();" } }, { "name": "SwitchTask", "taskReferenceName": "SwitchTask", "type": "SWITCH", "evaluatorType": "javascript", "inputParameters": { "param": "${GetNumberAtIndex.output.result}" }, "expression": "$.param > 0", "decisionCases": { "true": [ { "name": "WaitTask", "taskReferenceName": "WaitTask", "type": "WAIT", "inputParameters": { } }, { "name": "ComputeNumber", "taskReferenceName": "ComputeNumber", "type": "INLINE", "inputParameters": { "evaluatorType": "javascript", "number": "${GetNumberAtIndex.output.result.number}", "expression": "function compute() { return $.number+10; } compute();" } } ] } } ] } ] } ================================================ FILE: test-harness/src/test/resources/dynamic_fork_join_integration_test.json ================================================ { "name": "DynamicFanInOutTest", "description": "DynamicFanInOutTest", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "dt1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createdBy": "integration_app", "name": "integration_task_1", "description": "integration_task_1", "retryCount": 1, "timeoutSeconds": 120, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "fork", "taskReferenceName": "dynamicfanouttask", "inputParameters": { "dynamicTasks": "${dt1.output.dynamicTasks}", "dynamicTasksInput": "${dt1.output.dynamicTasksInput}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "dynamicfanouttask_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_4", "taskReferenceName": "task4", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createdBy": "integration_app", "name": "integration_task_4", "description": "integration_task_4", "retryCount": 1, "timeoutSeconds": 120, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 3600, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/event_workflow_integration_test.json ================================================ { "name": "test_event_workflow", "version": 1, "tasks": [ { "name": "eventX", "taskReferenceName": "wait0", "inputParameters": {}, "type": "EVENT", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "sink": "conductor", "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/exclusive_join_integration_test.json ================================================ { "name": "ExclusiveJoinTestWorkflow", "description": "Exclusive Join Test Workflow", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "task1", "inputParameters": { "payload": "${workflow.input.payload}" }, "type": "SIMPLE", "startDelay": 0, "optional": false }, { "name": "decide_task", "taskReferenceName": "decision1", "inputParameters": { "decision_1": "${workflow.input.decision_1}" }, "type": "DECISION", "caseValueParam": "decision_1", "decisionCases": { "true": [ { "name": "integration_task_2", "taskReferenceName": "task2", "inputParameters": { "payload": "${task1.output.payload}" }, "type": "SIMPLE", "startDelay": 0, "optional": false }, { "name": "decide_task", "taskReferenceName": "decision2", "inputParameters": { "decision_2": "${workflow.input.decision_2}" }, "type": "DECISION", "caseValueParam": "decision_2", "decisionCases": { "true": [ { "name": "integration_task_3", "taskReferenceName": "task3", "inputParameters": { "payload": "${task2.output.payload}" }, "type": "SIMPLE", "startDelay": 0, "optional": false } ] } } ], "false": [ { "name": "integration_task_4", "taskReferenceName": "task4", "inputParameters": { "payload": "${task1.output.payload}" }, "type": "SIMPLE", "startDelay": 0, "optional": false }, { "name": "decide_task", "taskReferenceName": "decision3", "inputParameters": { "decision_3": "${workflow.input.decision_3}" }, "type": "DECISION", "caseValueParam": "decision_3", "decisionCases": { "true": [ { "name": "integration_task_5", "taskReferenceName": "task5", "inputParameters": { "payload": "${task4.output.payload}" }, "type": "SIMPLE", "startDelay": 0, "optional": false } ] } } ] } }, { "name": "exclusive_join", "taskReferenceName": "exclusiveJoin", "type": "EXCLUSIVE_JOIN", "joinOn": [ "task3", "task5" ], "defaultExclusiveJoinTask": [ "task2", "task4", "task1" ] } ], "schemaVersion": 2, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/failure_workflow_for_terminate_task_workflow.json ================================================ { "name": "failure_workflow", "version": 1, "tasks": [ { "name": "lambda", "taskReferenceName": "lambda0", "inputParameters": { "input": "${workflow.input}", "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false}}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/fork_join_integration_test.json ================================================ { "name": "FanInOutTest", "description": "FanInOutTest", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fanouttask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "workflow.input.param1", "p2": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "p1": "workflow.input.param1", "p2": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "workflow.input.param1" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "fanouttask_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t3", "t2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_4", "taskReferenceName": "t4", "inputParameters": { "tp1": "workflow.input.param1" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/fork_join_sub_workflow.json ================================================ { "name": "integration_test_fork_join_sw", "description": "integration_test_fork_join_sw", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fanouttask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "sub_workflow_task", "taskReferenceName": "st1", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_workflow" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "fanouttask_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "st1", "t2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/fork_join_with_no_task_retry_integration_test.json ================================================ { "name": "FanInOutTest_2", "description": "FanInOutTest_2", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fanouttask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_0_RT_1", "taskReferenceName": "t1", "inputParameters": { "p1": "workflow.input.param1", "p2": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_0_RT_3", "taskReferenceName": "t3", "inputParameters": { "p1": "workflow.input.param1", "p2": "workflow.input.param2" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_0_RT_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "workflow.input.param1" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "fanouttask_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t3", "t2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_0_RT_4", "taskReferenceName": "t4", "inputParameters": { "tp1": "workflow.input.param1" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/fork_join_with_optional_sub_workflow_forks_integration_test.json ================================================ { "name": "integration_test_fork_join_optional_sw", "description": "integration_test_fork_join_optional_sw", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fanouttask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "st1", "taskReferenceName": "st1", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_workflow" }, "joinOn": [], "optional": true, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "st2", "taskReferenceName": "st2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "sub_workflow" }, "joinOn": [], "optional": true, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "fanouttask_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "st1", "st2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/hierarchical_fork_join_swf.json ================================================ { "name": "hierarchical_fork_join_swf", "description": "hierarchical_fork_join_swf", "version": 1, "tasks": [ { "name": "fork", "taskReferenceName": "fanouttask", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "sub_workflow_task", "taskReferenceName": "st1", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "${workflow.input.subwf}", "version": 1 }, "retryCount": 0 } ], [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "retryCount": 0 } ] ] }, { "name": "join", "taskReferenceName": "fanouttask_join", "inputParameters": {}, "type": "JOIN", "joinOn": [ "st1", "t2" ] } ], "inputParameters": [ "param1", "param2", "subwf" ], "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/input.json ================================================ ================================================ FILE: test-harness/src/test/resources/json_jq_transform_result_integration_test.json ================================================ { "name": "json_jq_transform_result_wf", "version": 1, "tasks": [ { "name": "json_jq_1", "taskReferenceName": "json_jq_1", "description": "json_jq_1", "inputParameters": { "data": [], "queryExpression": "if(.data | length >0) then \"EXISTS\" else \"CREATE\" end" }, "type": "JSON_JQ_TRANSFORM", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "decide_1", "taskReferenceName": "decide_1", "inputParameters": { "outcome": "${json_jq_1.output.result}" }, "type": "DECISION", "caseValueParam": "outcome", "decisionCases": { "CREATE": [ { "name": "json_jq_2", "taskReferenceName": "json_jq_2", "description": "json_jq_2", "inputParameters": { "inputData": { "request": { "transitions": [ { "name": "redeliver" }, { "name": "redeliver_from_validation_error" }, { "name": "redelivery" } ] } }, "queryExpression": ".inputData.request.transitions | map(.name)" }, "type": "JSON_JQ_TRANSFORM", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "decide_2", "taskReferenceName": "decide_2", "inputParameters": { "requestedAction": "${workflow.input.requestedAction}", "availableActions": "${json_jq_2.output.result}" }, "type": "DECISION", "caseExpression": "if ($.availableActions.indexOf($.requestedAction) >= 0) { \"true\" } else { \"false\" }", "decisionCases": { "false": [ { "name": "get_population_data", "taskReferenceName": "get_population_data", "inputParameters": { "http_request": { "uri": "https://datausa.io/api/data?drilldowns=Nation&measures=Population", "method": "GET" } }, "type": "HTTP", "startDelay": 0, "optional": false, "asyncComplete": false } ] } } ] } } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/nested_fork_join_integration_test.json ================================================ { "name": "FanInOutNestedTest", "description": "FanInOutNestedTest", "version": 1, "tasks": [ { "name": "fork1", "taskReferenceName": "fork1", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_11", "taskReferenceName": "t11", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "fork2", "taskReferenceName": "fork2", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_12", "taskReferenceName": "t12", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_14", "taskReferenceName": "t14", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_13", "taskReferenceName": "t13", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "Decision", "taskReferenceName": "d1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "a": [ { "name": "integration_task_16", "taskReferenceName": "t16", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_19", "taskReferenceName": "t19", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "b": [ { "name": "integration_task_17", "taskReferenceName": "t17", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20b", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_18", "taskReferenceName": "t18", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20def", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join2", "taskReferenceName": "join2", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t14", "t20" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join1", "taskReferenceName": "join1", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t11", "join2" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_15", "taskReferenceName": "t15", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/nested_fork_join_swf.json ================================================ { "name": "nested_fork_join_swf", "description": "nested_fork_join_swf", "version": 1, "tasks": [ { "name": "outer_fork", "taskReferenceName": "outer_fork", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "FORK_JOIN", "forkTasks": [ [ { "name": "inner_fork", "taskReferenceName": "inner_fork", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "FORK_JOIN", "forkTasks": [ [ { "name": "sub_workflow_task", "taskReferenceName": "st1", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "${workflow.input.subwf}", "version": 1 }, "retryCount": 0 } ], [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "retryCount": 0 } ] ] }, { "name": "inner_join", "taskReferenceName": "inner_join", "type": "JOIN", "joinOn": [ "st1", "t2" ] } ], [ { "name": "integration_task_2", "taskReferenceName": "t3", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "retryCount": 0 } ] ] }, { "name": "join", "taskReferenceName": "outer_join", "inputParameters": {}, "type": "JOIN", "joinOn": [ "inner_join", "t3" ] } ], "inputParameters": [ "param1", "param2", "subwf" ], "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/nested_fork_join_with_sub_workflow_integration_test.json ================================================ { "name": "FanInOutNestedSubWorkflowTest", "description": "FanInOutNestedSubWorkflowTest", "version": 1, "tasks": [ { "name": "fork1", "taskReferenceName": "fork1", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_11", "taskReferenceName": "t11", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "fork2", "taskReferenceName": "fork2", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "integration_task_12", "taskReferenceName": "t12", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_14", "taskReferenceName": "t14", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_13", "taskReferenceName": "t13", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "Decision", "taskReferenceName": "d1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "a": [ { "name": "integration_task_16", "taskReferenceName": "t16", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_19", "taskReferenceName": "t19", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "b": [ { "name": "integration_task_17", "taskReferenceName": "t17", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20b", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_18", "taskReferenceName": "t18", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20def", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join2", "taskReferenceName": "join2", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t14", "t20" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "sw1", "taskReferenceName": "sw1", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "integration_test_wf" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join1", "taskReferenceName": "join1", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t11", "join2", "sw1" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_15", "taskReferenceName": "t15", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "case": "${workflow.input.case}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/output.json ================================================ { "imageType": "TEST_SAMPLE", "case": "two", "op": { "TEST_SAMPLE": [ { "sourceId": "1413900_10830", "url": "file/location/a0bdc4d0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_50241", "url": "file/location/cd4e00a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-55ee8663-85c2-42d3-aca2-4076707e6d4e", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-14056154-1544-4350-81db-b3751fe44777", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-0b0ae5ea-d5c5-410c-adc9-bf16d2909c2e", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-08869779-614d-417c-bfea-36a3f8f199da", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-e117db45-1c48-45d0-b751-89386eb2d81d", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0221421-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/4a009209-002f-4b58-8b96-cb2198f8ba3c" }, { "sourceId": "f0252161-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/55b56298-5e7a-4949-b919-88c5c9557e8e" }, { "sourceId": "f038d070-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/3c4804f4-e826-436f-90c9-52b8d9266d52" }, { "sourceId": "f04e0621-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/689283a1-1816-48ef-83da-7f9ac874bf45" }, { "sourceId": "f04ddf10-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/586666ae-7321-445a-80b6-323c8c241ecd" }, { "sourceId": "f05950c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/31795cc4-2590-4b20-a617-deaa18301f99" }, { "sourceId": "1413900_46819", "url": "file/location/c74497a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_11177", "url": "file/location/a231c730-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48713", "url": "file/location/ca638ae0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48525", "url": "file/location/ca0c9140-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_73303", "url": "file/location/d5943a40-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_55202", "url": "file/location/d1a4d7a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-61413adf-3c10-4484-b25d-e238df898f45", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-addca397-f050-4339-ae86-9ba8c4e1b0d5", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e4de9810-0f69-4593-8926-01ed82cbebcb", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e16e2074-7af6-4700-ab05-ca41ba9c9ab4", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-341c86f8-57a5-40e1-8842-3eb41dd9f528", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-88c2ea9b-cef7-4120-8043-b92713d8fade", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3f6a731f-3c92-4677-9923-f80b8a6be632", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-1508b871-64de-47ce-8b07-76c5cb3f3e1e", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "generated-1406dce8-7b9c-4956-a7e8-78721c476ce9", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "f0206671-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35ebee36-3072-44c5-abb5-702a5a3b1a91" }, { "sourceId": "f01f5501-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d3a9133d-c681-4910-a769-8195526ae634" }, { "sourceId": "f022b060-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8fc1413d-170e-4644-a554-5e0c596b225c" }, { "sourceId": "f02fa8b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35bed0a2-7def-457b-bded-4f4d7d94f76e" }, { "sourceId": "f031f2a0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a5a2ea1f-8d13-429c-a44d-3057d21f608a" }, { "sourceId": "f0424650-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/1c599ffc-4f10-4c0b-8d9a-ae41c7256113" }, { "sourceId": "f04ec970-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8404a421-e1a6-41cf-af63-a35ccb474457" }, { "sourceId": "1413900_47197", "url": "file/location/c81b6fa0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-2a63c0c8-62ea-44a4-a33b-f0b3047e8b00", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-b27face7-3589-4209-944a-5153b20c5996", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-144675b3-9321-48d2-8b5b-e19a40d30ef2", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-8cbe821e-b1fb-48ce-beb5-735319af4db6", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-ecc4ea47-9bad-4b91-97c7-35f4ea6fb479", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-c1eb9ed0-8560-4e09-a748-f926edb7cdc2", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-6bed81fd-c777-4c61-8da1-0bb7f7cf0082", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-852e5510-dd5d-4900-a614-854148fcc716", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-f4dedcb7-37c9-4ba9-ab37-64ec9be7c882", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0259691-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/721bc0de-e75f-4386-8b2e-ca84eb653596" }, { "sourceId": "f02b3be1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d2043b17-8ce5-42ee-a5e4-81c68f0c4838" }, { "sourceId": "f02b62f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/63931561-3b5b-4ffe-af47-da2c9de94684" }, { "sourceId": "f0315660-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d99ed629-2885-4e4a-8a1b-22e487b875fa" }, { "sourceId": "f0306c00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6f8e673a-7003-44aa-96b9-e2ed8a4654ff" }, { "sourceId": "f033c760-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/627c00f9-14b3-4057-b6e2-0f962ad0308e" }, { "sourceId": "f03526f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fafabaf9-fe58-4a9a-b555-026521aeb2fe" }, { "sourceId": "f03acc41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6c9fed2c-558a-4db3-8360-659b5e8c46e4" }, { "sourceId": "f0463df1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e9fb83d2-5f14-4442-92b5-67e613f2e35f" }, { "sourceId": "f04fb3d0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e7a0f82f-be8d-4ada-a4b1-13e8165e08be" }, { "sourceId": "f05272f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9aba488a-22b3-4932-85a7-52c461203541" }, { "sourceId": "f0581841-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/457415f6-6d0c-4304-8533-0d5b43fac564" }, { "sourceId": "generated-8fefb48c-6fde-4fd6-8f33-a1f3f3b62105", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-30c61aa5-f5bd-4077-8c32-336b87acbe96", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-d5da37db-d486-46d4-8f7d-1e0710a77eb5", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-77af26fe-9e22-48af-99e3-f63f10fbe6de", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-2e807016-3d11-4b60-bec7-c380a608b67d", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-615d02e9-62c2-43ab-9df7-753b6b8e2c22", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3e1600fd-a626-4ee6-972b-5f0187e96c38", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-1dcb208c-6a58-4334-a60c-6fb54c8a2af5", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f024ac30-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0af2107b-4231-4d23-bef3-4e417ac6c5d3" }, { "sourceId": "f0282ea1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0f592681-fd23-4194-ae43-42f61c664485" }, { "sourceId": "f02c4d50-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ec46b9a3-99af-410a-af7d-726f8854909f" }, { "sourceId": "f02b8a00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aed7e5da-b524-4d41-b264-28ce615ec826" }, { "sourceId": "f02b14d1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/b88c9055-ab0d-4d27-a405-265ba2a15f0c" }, { "sourceId": "f03044f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb8c4df9-d59e-4ac3-880e-4ea94cd880a4" }, { "sourceId": "f034ffe1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/59f3fbe8-b300-4861-9b2f-dac7b15aea7d" }, { "sourceId": "f03c2bd0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/19a06d54-41ed-419d-9947-f10cd5f0d85c" }, { "sourceId": "f03fae41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a9a48a62-7d62-4f67-b281-cc6fdc1e722c" }, { "sourceId": "f0455390-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0aeffc0a-a5ad-46ff-abab-1b3bc6a5840a" }, { "sourceId": "f04b1ff1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9a08aaed-c125-48f7-9d1d-fd11266c2b12" }, { "sourceId": "f04cf4b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/17a6e0f9-aa64-411f-9af7-837c84f7443f" }, { "sourceId": "f0511360-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb633c73-cb33-4806-bc08-049024644856" }, { "sourceId": "f0538460-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a7012248-6769-42da-a6c8-d4b831f6efce" }, { "sourceId": "f058db91-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/bcf71522-6168-48c4-86c9-995bca60ae51" }, { "sourceId": "generated-adf005c4-95c1-4904-9968-09cc19a26bfe", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-c4d367a4-4cdc-412e-af79-09b227f2e3ba", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-48dba018-f884-49db-b87e-67274e244c8f", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "generated-26700b83-4892-420e-8b46-1ee21eba75fb", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-632f3198-c0dc-4348-974f-51684d4e443e", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-86e2dd1d-1aa4-4dbe-b37b-b488f5dd1c70", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f04134e0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ff8f59bf-7757-4d51-a7e4-619f3e8ffaf2" }, { "sourceId": "f04f65b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d66467d1-3ac6-4041-8d15-e722ee07231f" }, { "sourceId": "1413900_15255", "url": "file/location/a9e20260-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-e953493b-cbe3-4319-885e-00c82089c76c", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-65c54676-3adb-4ef0-b65e-8e2a49533cbf", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "f02ac6b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/21568877-07a5-411f-9715-5e92806c4448" }, { "sourceId": "f02fcfc1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/f3b1f1a2-48d3-475d-a607-2e5a1fe532e7" }, { "sourceId": "f03526f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/84a40c66-d925-4a4a-ba62-8491d26e29e9" }, { "sourceId": "f03e75c1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e84c00e8-a148-46cf-9a0b-431c4c2aeb08" }, { "sourceId": "f0429471-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/178de9fa-7cc8-457a-8fb6-5c080e6163ea" }, { "sourceId": "f047eba0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/18d153aa-e13b-4264-ae03-f3da75eb425b" }, { "sourceId": "f04fdae0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/7c843e53-8d87-47cf-bca5-1a02e7f5e33f" }, { "sourceId": "f0553210-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/26bacd65-9082-4d83-9506-90e5f1ccd16a" }, { "sourceId": "1413900_84904", "url": "file/location/d8f7b090-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-84adc784-8d7d-4088-ba51-16fde57fbc21", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-9e49c58b-0b33-4daf-a39a-8fc91e302328", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "f02dd3f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8937b328-8f0d-4762-8d1f-7d7bc80c3d2e" }, { "sourceId": "f03240c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aab6e386-4d59-4b40-b257-9aed12a45446" } ] } } ================================================ FILE: test-harness/src/test/resources/rate_limited_simple_task_workflow_integration_test.json ================================================ { "name": "test_rate_limit_simple_task_workflow", "version": 1, "tasks": [ { "name": "test_simple_task_with_rateLimits", "taskReferenceName": "test_simple_task_with_rateLimits", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/rate_limited_system_task_workflow_integration_test.json ================================================ { "name": "test_rate_limit_system_task_workflow", "version": 1, "tasks": [ { "name": "test_task_with_rateLimits", "taskReferenceName": "test_task_with_rateLimits", "inputParameters": {}, "type": "USER_TASK", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/sequential_json_jq_transform_integration_test.json ================================================ { "name": "sequential_json_jq_transform_wf", "version": 1, "tasks": [ { "name": "default_variables", "taskReferenceName": "default_variables", "description": "default_variables", "inputParameters": { "input": "${workflow.input}", "queryExpression": "{ requestTransform: .input.requestTransform // \".body\" , responseTransform: .input.responseTransform // \".response.body\", method: .input.method // \"GET\", document: .input.document // \"rgt_results\", successExpression: .input.successExpression // \"true\" }" }, "type": "JSON_JQ_TRANSFORM", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "request_transform", "taskReferenceName": "request_transform", "description": "request_transform", "inputParameters": { "body": "${workflow.input.body}", "queryExpression": "${default_variables.output.result.requestTransform}" }, "type": "JSON_JQ_TRANSFORM", "startDelay": 0, "optional": false, "asyncComplete": false } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/set_variable_workflow_integration_test.json ================================================ { "name": "set_variable_workflow_integration_test", "version": 1, "tasks": [ { "name": "simple", "taskReferenceName": "simple", "description": "simple", "inputParameters": { }, "type": "SIMPLE", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "set_variable", "taskReferenceName": "set_variable_1", "inputParameters": { "var": "${workflow.input.var}" }, "type": "SET_VARIABLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "wait", "taskReferenceName": "wait0", "inputParameters": {}, "type": "WAIT", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "variables": "${workflow.variables}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_decision_task_integration_test.json ================================================ { "name": "DecisionWorkflow", "description": "DecisionWorkflow", "version": 1, "tasks": [ { "name": "decisionTask", "taskReferenceName": "decisionTask", "inputParameters": { "case": "${workflow.input.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "c": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_5", "taskReferenceName": "t5", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_json_jq_transform_integration_test.json ================================================ { "name": "test_json_jq_transform_wf", "version": 1, "tasks": [ { "name": "jq", "taskReferenceName": "jq_1", "inputParameters": { "input": "${workflow.input}", "queryExpression": ".input as $_ | { out: ($_.in1.array + $_.in2.array) }" }, "type": "JSON_JQ_TRANSFORM", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_lambda_workflow_integration_test.json ================================================ { "name": "test_lambda_wf", "version": 1, "tasks": [ { "name": "lambda", "taskReferenceName": "lambda0", "inputParameters": { "input": "${workflow.input}", "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false} }" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_one_task_sub_workflow_integration_test.json ================================================ { "name": "sub_workflow", "description": "sub_workflow", "version": 1, "tasks": [ { "name": "simple_task_in_sub_wf", "taskReferenceName": "t1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_set_variable_workflow_integration_test.json ================================================ { "name": "test_set_variable_wf", "version": 1, "tasks": [ { "name": "set_variable", "taskReferenceName": "set_variable_1", "inputParameters": { "var": "${workflow.input.var}" }, "type": "SET_VARIABLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "variables": "${workflow.variables}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_switch_task_integration_test.json ================================================ { "name": "SwitchWorkflow", "description": "SwitchWorkflow", "version": 1, "tasks": [ { "name": "switchTask", "taskReferenceName": "switchTask", "inputParameters": { "case": "${workflow.input.case}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "c": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_5", "taskReferenceName": "t5", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_wait_task_workflow_integration_test.json ================================================ { "name": "test_wait_timeout", "version": 1, "tasks": [ { "name": "waitTimeout", "taskReferenceName": "wait0", "inputParameters": {}, "type": "WAIT", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_1_input_template_integration_test.json ================================================ { "name": "integration_test_template_wf", "description": "Test a simple workflow with an input template", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "p3": "${CPEWF_TASK_ID}", "someNullKey": null }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2", "param3", "param4" ], "inputTemplate": { "param1": { "nested_object": { "nested_key": "nested_value" } }, "param2": ["list", "of", "strings"], "param3": "string" }, "outputParameters": { "output": "${t1.output.op}", "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "param3": "${workflow.input.param3}" }, "failureWorkflow": "$workflow.input.failureWfName", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_1_integration_test.json ================================================ { "name": "integration_test_wf", "description": "integration_test_wf", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "p3": "${CPEWF_TASK_ID}", "someNullKey": null }, "type": "SIMPLE" }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}", "tp3": "${CPEWF_TASK_ID}" }, "type": "SIMPLE" } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o1": "${workflow.input.param1}", "o2": "${t2.output.uuid}", "o3": "${t1.output.op}" }, "failureWorkflow": "$workflow.input.failureWfName", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_3_integration_test.json ================================================ { "name": "integration_test_wf3", "description": "integration_test_wf3", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "someNullKey": null }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_with_async_complete_system_task_integration_test.json ================================================ { "name": "async_complete_integration_test_wf", "description": "async_complete_integration_test_wf", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "p3": "${CPEWF_TASK_ID}", "someNullKey": null }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "user_task", "taskReferenceName": "user_task", "inputParameters": { "input": "${t1.output.op}" }, "type": "USER_TASK", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": true, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o1": "${workflow.input.param1}", "o2": "${user_task.output.uuid}", "o3": "${t1.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_with_optional_task_integration_test.json ================================================ { "name": "optional_task_wf", "description": "optional_task_wf", "version": 1, "tasks": [ { "name": "task_optional", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": true, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o1": "${workflow.input.param1}", "o2": "${t2.output.uuid}", "o3": "${t1.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_with_resp_time_out_integration_test.json ================================================ { "name": "RTOWF", "description": "RTOWF", "version": 1, "tasks": [ { "name": "task_rt", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o1": "${workflow.input.param1}", "o2": "${t2.output.uuid}", "o3": "${t1.output.op}" }, "failureWorkflow": "$workflow.input.failureWfName", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/simple_workflow_with_sub_workflow_inline_def_integration_test.json ================================================ { "name": "WorkflowWithInlineSubWorkflow", "description": "WorkflowWithInlineSubWorkflow", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "tp11": "${workflow.input.param1}", "tp12": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "swt", "taskReferenceName": "swt", "inputParameters": { "op": "${t1.output.op}", "imageType": "${t1.output.imageType}" }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "one_task_workflow", "version": 1, "workflowDefinition": { "name": "one_task_workflow", "version": 1, "tasks": [ { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "p1": "${workflow.input.imageType}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "imageType", "op" ], "outputParameters": { "op": "${t3.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0 } }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "op": "${t1.output.op}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o3": "${t1.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/start_workflow_input.json ================================================ { "startWorkflow": { "name": "integration_test_wf", "input": { "op": { "TEST_SAMPLE": [ { "sourceId": "1413900_10830", "url": "file/location/a0bdc4d0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_50241", "url": "file/location/cd4e00a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-55ee8663-85c2-42d3-aca2-4076707e6d4e", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-14056154-1544-4350-81db-b3751fe44777", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-0b0ae5ea-d5c5-410c-adc9-bf16d2909c2e", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-08869779-614d-417c-bfea-36a3f8f199da", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-e117db45-1c48-45d0-b751-89386eb2d81d", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0221421-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/4a009209-002f-4b58-8b96-cb2198f8ba3c" }, { "sourceId": "f0252161-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/55b56298-5e7a-4949-b919-88c5c9557e8e" }, { "sourceId": "f038d070-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/3c4804f4-e826-436f-90c9-52b8d9266d52" }, { "sourceId": "f04e0621-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/689283a1-1816-48ef-83da-7f9ac874bf45" }, { "sourceId": "f04ddf10-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/586666ae-7321-445a-80b6-323c8c241ecd" }, { "sourceId": "f05950c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/31795cc4-2590-4b20-a617-deaa18301f99" }, { "sourceId": "1413900_46819", "url": "file/location/c74497a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_11177", "url": "file/location/a231c730-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48713", "url": "file/location/ca638ae0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_48525", "url": "file/location/ca0c9140-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_73303", "url": "file/location/d5943a40-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "1413900_55202", "url": "file/location/d1a4d7a0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-61413adf-3c10-4484-b25d-e238df898f45", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-addca397-f050-4339-ae86-9ba8c4e1b0d5", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e4de9810-0f69-4593-8926-01ed82cbebcb", "url": "file/sample/location/838a0ddb-a315-453a-8b8a-fa795f9d7691" }, { "sourceId": "generated-e16e2074-7af6-4700-ab05-ca41ba9c9ab4", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-341c86f8-57a5-40e1-8842-3eb41dd9f528", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-88c2ea9b-cef7-4120-8043-b92713d8fade", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3f6a731f-3c92-4677-9923-f80b8a6be632", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-1508b871-64de-47ce-8b07-76c5cb3f3e1e", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "generated-1406dce8-7b9c-4956-a7e8-78721c476ce9", "url": "file/sample/location/a2e4195f-3900-45b4-9335-45f85fca6467" }, { "sourceId": "f0206671-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35ebee36-3072-44c5-abb5-702a5a3b1a91" }, { "sourceId": "f01f5501-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d3a9133d-c681-4910-a769-8195526ae634" }, { "sourceId": "f022b060-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8fc1413d-170e-4644-a554-5e0c596b225c" }, { "sourceId": "f02fa8b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/35bed0a2-7def-457b-bded-4f4d7d94f76e" }, { "sourceId": "f031f2a0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a5a2ea1f-8d13-429c-a44d-3057d21f608a" }, { "sourceId": "f0424650-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/1c599ffc-4f10-4c0b-8d9a-ae41c7256113" }, { "sourceId": "f04ec970-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8404a421-e1a6-41cf-af63-a35ccb474457" }, { "sourceId": "1413900_47197", "url": "file/location/c81b6fa0-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-2a63c0c8-62ea-44a4-a33b-f0b3047e8b00", "url": "file/sample/location/e008d018-63d7-44b2-b07e-c7435430ac71" }, { "sourceId": "generated-b27face7-3589-4209-944a-5153b20c5996", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-144675b3-9321-48d2-8b5b-e19a40d30ef2", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-8cbe821e-b1fb-48ce-beb5-735319af4db6", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-ecc4ea47-9bad-4b91-97c7-35f4ea6fb479", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-c1eb9ed0-8560-4e09-a748-f926edb7cdc2", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-6bed81fd-c777-4c61-8da1-0bb7f7cf0082", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-852e5510-dd5d-4900-a614-854148fcc716", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-f4dedcb7-37c9-4ba9-ab37-64ec9be7c882", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f0259691-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/721bc0de-e75f-4386-8b2e-ca84eb653596" }, { "sourceId": "f02b3be1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d2043b17-8ce5-42ee-a5e4-81c68f0c4838" }, { "sourceId": "f02b62f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/63931561-3b5b-4ffe-af47-da2c9de94684" }, { "sourceId": "f0315660-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d99ed629-2885-4e4a-8a1b-22e487b875fa" }, { "sourceId": "f0306c00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6f8e673a-7003-44aa-96b9-e2ed8a4654ff" }, { "sourceId": "f033c760-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/627c00f9-14b3-4057-b6e2-0f962ad0308e" }, { "sourceId": "f03526f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fafabaf9-fe58-4a9a-b555-026521aeb2fe" }, { "sourceId": "f03acc41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/6c9fed2c-558a-4db3-8360-659b5e8c46e4" }, { "sourceId": "f0463df1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e9fb83d2-5f14-4442-92b5-67e613f2e35f" }, { "sourceId": "f04fb3d0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e7a0f82f-be8d-4ada-a4b1-13e8165e08be" }, { "sourceId": "f05272f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9aba488a-22b3-4932-85a7-52c461203541" }, { "sourceId": "f0581841-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/457415f6-6d0c-4304-8533-0d5b43fac564" }, { "sourceId": "generated-8fefb48c-6fde-4fd6-8f33-a1f3f3b62105", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-30c61aa5-f5bd-4077-8c32-336b87acbe96", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-d5da37db-d486-46d4-8f7d-1e0710a77eb5", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-77af26fe-9e22-48af-99e3-f63f10fbe6de", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-2e807016-3d11-4b60-bec7-c380a608b67d", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-615d02e9-62c2-43ab-9df7-753b6b8e2c22", "url": "file/sample/location/519f6c80-96ef-440f-9d37-ccf36c7d1e5d" }, { "sourceId": "generated-3e1600fd-a626-4ee6-972b-5f0187e96c38", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-1dcb208c-6a58-4334-a60c-6fb54c8a2af5", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f024ac30-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0af2107b-4231-4d23-bef3-4e417ac6c5d3" }, { "sourceId": "f0282ea1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0f592681-fd23-4194-ae43-42f61c664485" }, { "sourceId": "f02c4d50-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ec46b9a3-99af-410a-af7d-726f8854909f" }, { "sourceId": "f02b8a00-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aed7e5da-b524-4d41-b264-28ce615ec826" }, { "sourceId": "f02b14d1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/b88c9055-ab0d-4d27-a405-265ba2a15f0c" }, { "sourceId": "f03044f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb8c4df9-d59e-4ac3-880e-4ea94cd880a4" }, { "sourceId": "f034ffe1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/59f3fbe8-b300-4861-9b2f-dac7b15aea7d" }, { "sourceId": "f03c2bd0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/19a06d54-41ed-419d-9947-f10cd5f0d85c" }, { "sourceId": "f03fae41-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a9a48a62-7d62-4f67-b281-cc6fdc1e722c" }, { "sourceId": "f0455390-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/0aeffc0a-a5ad-46ff-abab-1b3bc6a5840a" }, { "sourceId": "f04b1ff1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/9a08aaed-c125-48f7-9d1d-fd11266c2b12" }, { "sourceId": "f04cf4b1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/17a6e0f9-aa64-411f-9af7-837c84f7443f" }, { "sourceId": "f0511360-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/fb633c73-cb33-4806-bc08-049024644856" }, { "sourceId": "f0538460-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/a7012248-6769-42da-a6c8-d4b831f6efce" }, { "sourceId": "f058db91-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/bcf71522-6168-48c4-86c9-995bca60ae51" }, { "sourceId": "generated-adf005c4-95c1-4904-9968-09cc19a26bfe", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-c4d367a4-4cdc-412e-af79-09b227f2e3ba", "url": "file/sample/location/3d927190-1c4d-4af2-91cf-2968d3ccfe70" }, { "sourceId": "generated-48dba018-f884-49db-b87e-67274e244c8f", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "generated-26700b83-4892-420e-8b46-1ee21eba75fb", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "generated-632f3198-c0dc-4348-974f-51684d4e443e", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "generated-86e2dd1d-1aa4-4dbe-b37b-b488f5dd1c70", "url": "file/sample/location/e87da4d1-72da-47a3-801d-43e01c050c89" }, { "sourceId": "f04134e0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/ff8f59bf-7757-4d51-a7e4-619f3e8ffaf2" }, { "sourceId": "f04f65b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/d66467d1-3ac6-4041-8d15-e722ee07231f" }, { "sourceId": "1413900_15255", "url": "file/location/a9e20260-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-e953493b-cbe3-4319-885e-00c82089c76c", "url": "file/sample/location/ec16facd-86e3-4c3f-8dfb-7a2ad3a4e18c" }, { "sourceId": "generated-65c54676-3adb-4ef0-b65e-8e2a49533cbf", "url": "file/sample/location/07ec28a1-189e-4f2a-9dd5-f3ca68ce977d" }, { "sourceId": "f02ac6b0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/21568877-07a5-411f-9715-5e92806c4448" }, { "sourceId": "f02fcfc1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/f3b1f1a2-48d3-475d-a607-2e5a1fe532e7" }, { "sourceId": "f03526f0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/84a40c66-d925-4a4a-ba62-8491d26e29e9" }, { "sourceId": "f03e75c1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/e84c00e8-a148-46cf-9a0b-431c4c2aeb08" }, { "sourceId": "f0429471-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/178de9fa-7cc8-457a-8fb6-5c080e6163ea" }, { "sourceId": "f047eba0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/18d153aa-e13b-4264-ae03-f3da75eb425b" }, { "sourceId": "f04fdae0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/7c843e53-8d87-47cf-bca5-1a02e7f5e33f" }, { "sourceId": "f0553210-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/26bacd65-9082-4d83-9506-90e5f1ccd16a" }, { "sourceId": "1413900_84904", "url": "file/location/d8f7b090-5315-11e8-bf88-0efd527701fc" }, { "sourceId": "generated-84adc784-8d7d-4088-ba51-16fde57fbc21", "url": "file/sample/location/3881aea9-a731-4e22-9ead-2d6eccc51140" }, { "sourceId": "generated-9e49c58b-0b33-4daf-a39a-8fc91e302328", "url": "file/sample/location/4bce4154-fb4b-4f0a-887d-a0cd12d4d214" }, { "sourceId": "f02dd3f1-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/8937b328-8f0d-4762-8d1f-7d7bc80c3d2e" }, { "sourceId": "f03240c0-86e8-11e8-af77-0a2ba4eae3ec", "url": "file/test/location/aab6e386-4d59-4b40-b257-9aed12a45446" } ] } } } } ================================================ FILE: test-harness/src/test/resources/switch_and_fork_join_integration_test.json ================================================ { "name": "ForkConditionalTest", "description": "ForkConditionalTest", "version": 1, "tasks": [ { "name": "forkTask", "taskReferenceName": "forkTask", "inputParameters": {}, "type": "FORK_JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [ [ { "name": "switchTask", "taskReferenceName": "switchTask", "inputParameters": { "case": "${workflow.input.case}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "c": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "integration_task_5", "taskReferenceName": "t5", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_20", "taskReferenceName": "t20", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], [ { "name": "integration_task_10", "taskReferenceName": "t10", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] ], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "joinTask", "taskReferenceName": "joinTask", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [ "t20", "t10" ], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/switch_and_terminate_integration_test.json ================================================ { "name": "ConditionalTerminateWorkflow", "description": "ConditionalTerminateWorkflow", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "tp11": "${workflow.input.param1}", "tp12": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "switch", "taskReferenceName": "switch", "inputParameters": { "case": "${workflow.input.case}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "one": [ { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "tp21": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "FAILED", "workflowOutput": "${t1.output.op}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_3", "taskReferenceName": "t3", "inputParameters": { "tp31": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [ "param1", "param2" ], "outputParameters": { "o2": "${t3.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/switch_with_no_default_case_integration_test.json ================================================ { "name": "SwitchWithNoDefaultCaseWF", "description": "switch_with_no_default_case", "version": 1, "tasks": [ { "name": "switchTask", "taskReferenceName": "switchTask", "inputParameters": { "case": "${workflow.input.case}" }, "type": "SWITCH", "evaluatorType": "value-param", "expression": "case", "decisionCases": { "c": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/terminate_task_completed_workflow_integration_test.json ================================================ { "name": "test_terminate_task_wf", "version": 1, "tasks": [ { "name": "lambda", "taskReferenceName": "lambda0", "inputParameters": { "input": "${workflow.input}", "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false}}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": "${lambda0.output}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": { "o1": "${lambda0.output}", "o2": "${t2.output}" }, "failureWorkflow": "failure_workflow", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/terminate_task_failed_workflow_integration.json ================================================ { "name": "test_terminate_task_failed_wf", "version": 1, "tasks": [ { "name": "lambda", "taskReferenceName": "lambda0", "inputParameters": { "input": "${workflow.input}", "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false}}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "FAILED", "terminationReason": "Early exit in terminate", "workflowOutput": "${lambda0.output}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "failureWorkflow": "failure_workflow", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/terminate_task_parent_workflow.json ================================================ { "name": "test_terminate_task_parent_wf", "version": 1, "tasks": [ { "name": "test_forkjoin", "taskReferenceName": "forkx", "type": "FORK_JOIN", "forkTasks": [ [ { "name": "test_lambda_task1", "taskReferenceName": "lambdaTask1", "inputParameters": { "lambdaValue": "${workflow.input.lambdaValue}", "scriptExpression": "var i = 10; if ($.lambdaValue == 1){ return {testvalue: 'Lambda value was 1', iValue: i} } else { return {testvalue: 'Lambda value was NOT 1', iValue: i + 3} }" }, "type": "LAMBDA" }, { "name": "test_terminate_subworkflow", "taskReferenceName": "test_terminate_subworkflow", "inputParameters": { }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "test_terminate_task_sub_wf" } } ], [ { "name": "test_lambda_task2", "taskReferenceName": "lambdaTask2", "inputParameters": { "lambdaValue": "${workflow.input.lambdaValue}", "scriptExpression": "var i = 10; if ($.lambdaValue == 1){ return {testvalue: 'Lambda value was 1', iValue: i} } else { return {testvalue: 'Lambda value was NOT 1', iValue: i + 3} }" }, "type": "LAMBDA" }, { "name": "test_wait_task", "taskReferenceName": "basicJavaA", "type": "WAIT" }, { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": "some output" }, "type": "TERMINATE", "startDelay": 0, "optional": false }, { "name": "test_second_wait_task", "taskReferenceName": "basicJavaB", "type": "WAIT" } ] ] }, { "name": "join", "taskReferenceName": "thejoin", "type": "JOIN", "joinOn": [ "test_terminate_subworkflow", "basicJavaB" ] } ], "schemaVersion": 2, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/terminate_task_sub_workflow.json ================================================ { "name": "test_terminate_task_sub_wf", "version": 1, "tasks": [ { "name": "integration_task_3", "taskReferenceName": "t3", "type": "SIMPLE" } ], "schemaVersion": 2, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/test_task_failed_parent_workflow.json ================================================ { "name": "test_task_failed_parent_wf", "version": 1, "tasks": [ { "name": "test_lambda_task1", "taskReferenceName": "lambdaTask1", "inputParameters": { "lambdaValue": "${workflow.input.lambdaValue}", "scriptExpression": "var i = 10; if ($.lambdaValue == 1){ return {testvalue: 'Lambda value was 1', iValue: i} } else { return {testvalue: 'Lambda value was NOT 1', iValue: i + 3} }" }, "type": "LAMBDA" }, { "name": "test_task_failed_sub_wf", "taskReferenceName": "test_task_failed_sub_wf", "inputParameters": { }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "test_task_failed_sub_wf" } }, { "name": "test_lambda_task2", "taskReferenceName": "lambdaTask2", "inputParameters": { "lambdaValue": "${workflow.input.lambdaValue}", "scriptExpression": "var i = 10; if ($.lambdaValue == 1){ return {testvalue: 'Lambda value was 1', iValue: i} } else { return {testvalue: 'Lambda value was NOT 1', iValue: i + 3} }" }, "type": "LAMBDA" } ], "schemaVersion": 2, "ownerEmail": "test@harness.com", "failureWorkflow": "failure_workflow" } ================================================ FILE: test-harness/src/test/resources/test_task_failed_sub_workflow.json ================================================ { "name": "test_task_failed_sub_wf", "version": 1, "tasks": [ { "name": "lambda", "taskReferenceName": "lambda0", "inputParameters": { "input": "${workflow.input}", "scriptExpression": "if ($.input.a==1){return {testvalue: true}} else{return {testvalue: false}}" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "FAILED", "workflowOutput": "${lambda0.output}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_2", "taskReferenceName": "t2", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/wait_workflow_integration_test.json ================================================ { "name": "test_wait_workflow", "version": 1, "tasks": [ { "name": "wait", "taskReferenceName": "wait0", "inputParameters": {}, "type": "WAIT", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/workflow_that_starts_another_workflow.json ================================================ { "name": "workflow_that_starts_another_workflow", "description": "A workflow that uses START_WORKFLOW task to start another workflow", "version": 1, "tasks": [ { "name": "start_workflow", "taskReferenceName": "st", "inputParameters": { "startWorkflow": "${workflow.input.startWorkflow}" }, "type": "START_WORKFLOW" } ], "inputParameters": ["start_workflow"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/workflow_with_sub_workflow_1_integration_test.json ================================================ { "name": "integration_test_wf_with_sub_wf", "description": "integration_test_wf_with_sub_wf", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "inputParameters": { "p1": "${workflow.input.param1}", "p2": "${workflow.input.param2}", "someNullKey": null }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sub_workflow_task", "taskReferenceName": "t2", "inputParameters": { "param1": "${workflow.input.param1}", "param2": "${workflow.input.param2}", "subwf": "${workflow.input.nextSubwf}" }, "type": "SUB_WORKFLOW", "subWorkflowParam": { "name": "${workflow.input.subwf}", "version": 1 }, "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "retryCount": 0 } ], "inputParameters": [ "param1", "param2" ], "failureWorkflow": "$workflow.input.failureWfName", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "ownerEmail": "test@harness.com" } ================================================ FILE: test-harness/src/test/resources/workflow_with_synchronous_system_task.json ================================================ { "name": "workflow_with_synchronous_system_task", "description": "A workflow with a simple task followed a synchronous task", "version": 1, "tasks": [ { "name": "integration_task_1", "taskReferenceName": "t1", "type": "SIMPLE" }, { "name": "jsonjq", "taskReferenceName": "jsonjq", "inputParameters": { "queryExpression": ".tp2.TEST_SAMPLE | length", "tp1": "${workflow.input.param1}", "tp2": "${t1.output.op}" }, "type": "JSON_JQ_TRANSFORM" } ], "inputParameters": [], "outputParameters": { "data": "${jsonjq.output.resources}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "example@email.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ================================================ FILE: ui/.eslintrc ================================================ { "extends": ["react-app", "plugin:cypress/recommended"], "rules": { "import/no-anonymous-default-export": 0 } } ================================================ FILE: ui/.gitignore ================================================ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. # dependencies /node_modules /.pnp .pnp.js # testing /coverage /cypress/screenshots /cypress/videos # production /build # misc .DS_Store .env.local .env.development.local .env.test.local .env.production.local npm-debug.log* yarn-debug.log* yarn-error.log* ================================================ FILE: ui/.prettierignore ================================================ build ================================================ FILE: ui/.prettierrc.json ================================================ {} ================================================ FILE: ui/README.md ================================================ ## Conductor UI The UI is a standard `create-react-app` React Single Page Application (SPA). To get started, with Node 14 and `yarn` installed, first run `yarn install` from within the `/ui` directory to retrieve package dependencies. For more information regarding CRA configuration and usage, see the official [doc site](https://create-react-app.dev/). > ### For upgrading users > > The UI is designed to operate directly with the Conductor Server API. A Node `express` backend is no longer required. ### Development Server To run the UI on the bundled development server, run `yarn run start`. Navigate your browser to `http://localhost:5000`. #### Reverse Proxy configuration The default setup expects that the Conductor Server API will be available at `localhost:8080/api`. You may select an alternate port and hostname, or rewrite the API path by editing `setupProxy.js`. Note that `setupProxy.js` is used ONLY by the development server. ### Hosting for Production There is no need to "build" the project unless you require compiled assets to host on a production web server. In this case, the project can be built with the command `yarn build`. The assets will be produced to `/build`. Your hosting environment should make the Conductor Server API available on the same domain. This avoids complexities regarding cross-origin data fetching. The default path prefix is `/api`. If a different prefix is desired, `plugins/fetch.js` can be modified to customize the API fetch behavior. See `docker/serverAndUI` for an `nginx` based example. #### Different host path The static UI would work when rendered from any host route. The default is '/'. You can customize this by setting the 'homepage' field in package.json Refer - https://docs.npmjs.com/cli/v9/configuring-npm/package-json#homepage - https://create-react-app.dev/docs/deployment/#building-for-relative-paths ### Customization Hooks For ease of maintenance, a number of touch points for customization have been removed to `/plugins`. - `AppBarModules.jsx` - `AppLogo.jsx` - `env.js` - `fetch.js` ### Authentication We recommend that authentication & authorization be de-coupled from the UI and handled at the web server/access gateway. #### Examples (WIP) - Basic Auth (username/password) with `nginx` - Commercial IAM Vendor - Node `express` server with `passport.js` ================================================ FILE: ui/cypress/e2e/spec.cy.js ================================================ describe("Landing Page", () => { beforeEach(() => { cy.intercept("/api/workflow/search?**", { fixture: "workflowSearch.json" }); cy.intercept("/api/tasks/search?**", { fixture: "taskSearch.json" }); cy.intercept("/api/metadata/workflow", { fixture: "metadataWorkflow.json", }); cy.intercept("/api/metadata/taskdefs", { fixture: "metadataTasks.json" }); }); it("Homepage preloads with default query", () => { cy.visit("/"); cy.contains("Search Execution"); cy.contains("Page 1 of 5"); cy.get(".rdt_TableCell").contains("feature_value_compute_workflow"); }); it("Workflow name dropdown", () => { cy.get(".MuiAutocomplete-inputRoot input").first().click(); cy.get("li.MuiAutocomplete-option") .contains("Do_While_Workflow_Iteration_Fix") .click(); cy.get(".MuiAutocomplete-tag").contains("Do_While_Workflow_Iteration_Fix"); }); it("Switch to Task Tab - No results", () => { cy.get("a.MuiTab-root").contains("Tasks").click(); cy.contains("Task Name"); cy.contains("There are no records to display"); }); it("Task Name Dropdown", () => { cy.get(".MuiAutocomplete-inputRoot input").first().click(); cy.get("li.MuiAutocomplete-option").contains("example_task_2").click(); cy.get(".MuiAutocomplete-tag").contains("example_task_2"); }); it("Execute Task Search", () => { cy.get("button").contains("Search").click(); cy.contains("Page 1 of 1"); cy.get(".rdt_TableCell").contains("36d24c5c-9c26-46cf-9709-e1bc6963b8a5"); }); }); ================================================ FILE: ui/cypress/fixtures/doWhile/doWhileSwitch.json ================================================ { "ownerApp": "nq_mwi_conductor_ui_server", "createTime": 1660252744369, "status": "COMPLETED", "endTime": 1660252745449, "workflowId": "9aaf69a6-9c61-4460-93b5-0a657a084ba4", "tasks": [ { "taskType": "INLINE", "status": "COMPLETED", "inputData": { "evaluatorType": "javascript", "expression": "1", "value": null }, "referenceTaskName": "inline_task_outside", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "inline_task_outside", "scheduledTime": 1660252744439, "startTime": 1660252744437, "endTime": 1660252744504, "updateTime": 1660252744446, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "9aaf69a6-9c61-4460-93b5-0a657a084ba4", "workflowType": "LoopTestWithSwitch", "taskId": "07ae873e-5316-4e89-9c1e-a9cab711f1a2", "callbackAfterSeconds": 0, "outputData": { "result": 1 }, "workflowTask": { "name": "inline_task_outside", "taskReferenceName": "inline_task_outside", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "1" }, "type": "INLINE", "startDelay": 0, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -2, "loopOverTask": false }, { "taskType": "DO_WHILE", "status": "COMPLETED", "inputData": { "value": null }, "referenceTaskName": "LoopTask", "retryCount": 0, "seq": 2, "pollCount": 0, "taskDefName": "Loop Task", "scheduledTime": 1660252744620, "startTime": 1660252744618, "endTime": 1660252745337, "updateTime": 1660252744808, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "9aaf69a6-9c61-4460-93b5-0a657a084ba4", "workflowType": "LoopTestWithSwitch", "taskId": "790126b0-81e8-4286-ac65-d1f4c8eca271", "callbackAfterSeconds": 0, "outputData": { "1": { "inline_task": { "result": { "result": "NODE_2" } }, "switch_task": { "evaluationResult": ["null"] } }, "iteration": 1 }, "workflowTask": { "name": "Loop Task", "taskReferenceName": "LoopTask", "inputParameters": { "value": "${workflow.input.value}" }, "type": "DO_WHILE", "startDelay": 0, "optional": false, "asyncComplete": false, "loopCondition": "false", "loopOver": [ { "name": "inline_task", "taskReferenceName": "inline_task", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "function e() { if ($.value == 1){return {\"result\": 'NODE_1'}} else { return {\"result\": 'NODE_2'}}} e();" }, "type": "INLINE", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${inline_task_1.output.result.result}" }, "type": "SWITCH", "decisionCases": { "NODE_1": [ { "name": "Set_NODE_1", "taskReferenceName": "Set_NODE_1", "inputParameters": { "node": "NODE_1" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ], "NODE_2": [ { "name": "Set_NODE_2", "taskReferenceName": "Set_NODE_2", "inputParameters": { "node": "NODE_2" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ] }, "startDelay": 0, "optional": false, "asyncComplete": false, "evaluatorType": "value-param", "expression": "switchCaseValue" } ] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "workflowPriority": 0, "iteration": 1, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -2, "loopOverTask": true }, { "taskType": "INLINE", "status": "COMPLETED", "inputData": { "evaluatorType": "javascript", "expression": "function e() { if ($.value == 1){return {\"result\": 'NODE_1'}} else { return {\"result\": 'NODE_2'}}} e();", "value": null }, "referenceTaskName": "inline_task__1", "retryCount": 0, "seq": 3, "pollCount": 0, "taskDefName": "inline_task", "scheduledTime": 1660252744696, "startTime": 1660252744693, "endTime": 1660252744931, "updateTime": 1660252744702, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "9aaf69a6-9c61-4460-93b5-0a657a084ba4", "workflowType": "LoopTestWithSwitch", "taskId": "27f7fbc4-325b-43c4-872f-37dc64c9dab0", "callbackAfterSeconds": 0, "outputData": { "result": { "result": "NODE_2" } }, "workflowTask": { "name": "inline_task", "taskReferenceName": "inline_task", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "function e() { if ($.value == 1){return {\"result\": 'NODE_1'}} else { return {\"result\": 'NODE_2'}}} e();" }, "type": "INLINE", "startDelay": 0, "optional": false, "asyncComplete": false }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 1, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -3, "loopOverTask": true }, { "taskType": "SWITCH", "status": "COMPLETED", "inputData": { "case": "null" }, "referenceTaskName": "switch_task__1", "retryCount": 0, "seq": 4, "pollCount": 0, "taskDefName": "SWITCH", "scheduledTime": 1660252745049, "startTime": 1660252745047, "endTime": 1660252745163, "updateTime": 1660252745056, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "9aaf69a6-9c61-4460-93b5-0a657a084ba4", "workflowType": "LoopTestWithSwitch", "taskId": "2e2a0836-a2e6-4902-9e41-9bbc2c75e0ed", "callbackAfterSeconds": 0, "outputData": { "evaluationResult": ["null"] }, "workflowTask": { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${inline_task_1.output.result.result}" }, "type": "SWITCH", "decisionCases": { "NODE_1": [ { "name": "Set_NODE_1", "taskReferenceName": "Set_NODE_1", "inputParameters": { "node": "NODE_1" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ], "NODE_2": [ { "name": "Set_NODE_2", "taskReferenceName": "Set_NODE_2", "inputParameters": { "node": "NODE_2" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ] }, "startDelay": 0, "optional": false, "asyncComplete": false, "evaluatorType": "value-param", "expression": "switchCaseValue" }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 1, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -2, "loopOverTask": true } ], "input": {}, "output": { "evaluationResult": ["null"] }, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "createTime": 1660244498873, "updateTime": 1660252731854, "name": "LoopTestWithSwitch", "description": "Loop Test With Switch WF", "version": 3, "tasks": [ { "name": "inline_task_outside", "taskReferenceName": "inline_task_outside", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "1" }, "type": "INLINE", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "Loop Task", "taskReferenceName": "LoopTask", "inputParameters": { "value": "${workflow.input.value}" }, "type": "DO_WHILE", "startDelay": 0, "optional": false, "asyncComplete": false, "loopCondition": "false", "loopOver": [ { "name": "inline_task", "taskReferenceName": "inline_task", "inputParameters": { "value": "${workflow.input.value}", "evaluatorType": "javascript", "expression": "function e() { if ($.value == 1){return {\"result\": 'NODE_1'}} else { return {\"result\": 'NODE_2'}}} e();" }, "type": "INLINE", "startDelay": 0, "optional": false, "asyncComplete": false }, { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${inline_task_1.output.result.result}" }, "type": "SWITCH", "decisionCases": { "NODE_1": [ { "name": "Set_NODE_1", "taskReferenceName": "Set_NODE_1", "inputParameters": { "node": "NODE_1" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ], "NODE_2": [ { "name": "Set_NODE_2", "taskReferenceName": "Set_NODE_2", "inputParameters": { "node": "NODE_2" }, "type": "SET_VARIABLE", "startDelay": 0, "optional": false, "asyncComplete": false } ] }, "startDelay": 0, "optional": false, "asyncComplete": false, "evaluatorType": "value-param", "expression": "switchCaseValue" } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "abc@example.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1660252744369, "workflowName": "LoopTestWithSwitch", "workflowVersion": 3 } ================================================ FILE: ui/cypress/fixtures/dynamicFork/externalizedInput.json ================================================ { "ownerApp": "nq_mwi_conductor_ui_server", "createTime": 1656008300448, "status": "COMPLETED", "endTime": 1656008301210, "workflowId": "e66254b6-388d-43a6-b890-c518df832e51", "tasks": [ { "taskType": "FORK", "status": "COMPLETED", "externalInputPayloadStoragePath": "task/input/c8569b00-62d9-4a4b-b918-93a4bf4e6004.json", "referenceTaskName": "dynamic_tasks", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "FORK", "scheduledTime": 1656008300534, "startTime": 1656008300525, "endTime": 1656008300525, "updateTime": 1656008300549, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "b49dc1be-66eb-4816-8ee1-6aaea25f14ba", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -9, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 46, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "first_task", "retryCount": 0, "seq": 2, "pollCount": 0, "taskDefName": "first_task", "scheduledTime": 1656008300535, "startTime": 1656008300527, "endTime": 1656008300922, "updateTime": 1656008300628, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "6ce064a7-7ef3-413c-b6af-318cb7e6751e", "callbackAfterSeconds": 0, "outputData": { "result": 45 }, "workflowTask": { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -8, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 234, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "second_task", "retryCount": 0, "seq": 3, "pollCount": 0, "taskDefName": "second_task", "scheduledTime": 1656008300537, "startTime": 1656008300529, "endTime": 1656008300977, "updateTime": 1656008300683, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "b936ea24-9c3e-4651-8702-2ff5aa4dd579", "callbackAfterSeconds": 0, "outputData": { "result": 233 }, "workflowTask": { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -8, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 12, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "third_task", "retryCount": 0, "seq": 4, "pollCount": 0, "taskDefName": "third_task", "scheduledTime": 1656008300540, "startTime": 1656008300531, "endTime": 1656008301031, "updateTime": 1656008300760, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "bf2963cd-e545-4a26-b533-2ae760e77634", "callbackAfterSeconds": 0, "outputData": { "result": 11 }, "workflowTask": { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -9, "loopOverTask": false }, { "taskType": "JOIN", "status": "COMPLETED", "inputData": { "joinOn": ["first_task", "second_task", "third_task"] }, "referenceTaskName": "join_dynamic", "retryCount": 0, "seq": 5, "pollCount": 0, "taskDefName": "JOIN", "scheduledTime": 1656008300542, "startTime": 1656008300531, "endTime": 1656008301085, "updateTime": 1656008300831, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "25ddfe4d-eaf0-4171-964f-9b53ad06002b", "callbackAfterSeconds": 0, "outputData": { "second_task": { "result": 233 }, "third_task": { "result": 11 }, "first_task": { "result": 45 } }, "workflowTask": { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -11, "loopOverTask": false } ], "input": { "tasksJSON": [ { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "tasksInputJSON": { "first_task": { "number": 46 }, "second_task": { "number": 234 }, "third_task": { "number": 12 } } }, "output": { "second_task": { "result": 233 }, "third_task": { "result": 11 }, "first_task": { "result": 45 } }, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "createTime": 1656005417724, "updateTime": 1656005671608, "name": "example_dynamic_tasks", "description": "A workflow that allows dynamic execution of tasks", "version": 2, "tasks": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": ["tasksJSON", "tasksInputJSON"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "mwi-workflow-dev@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1656008300448, "workflowName": "example_dynamic_tasks", "workflowVersion": 2 } ================================================ FILE: ui/cypress/fixtures/dynamicFork/noneSpawned.json ================================================ { "ownerApp": "peterl@netflix.com", "createTime": 1656096815470, "status": "COMPLETED", "endTime": 1656096815832, "workflowId": "fe4efd7b-73ea-4c48-8147-840fa4e1e63b", "tasks": [ { "taskType": "FORK", "status": "COMPLETED", "inputData": { "forkedTaskDefs": [], "forkedTasks": [] }, "referenceTaskName": "dynamic_tasks", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "FORK", "scheduledTime": 1656096815568, "startTime": 1656096815566, "endTime": 1656096815566, "updateTime": 1656096815577, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "fe4efd7b-73ea-4c48-8147-840fa4e1e63b", "workflowType": "example_dynamic_tasks", "taskId": "01a706f7-c28d-4287-a179-8075d16ff201", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -2, "loopOverTask": false }, { "taskType": "JOIN", "status": "COMPLETED", "inputData": { "joinOn": [] }, "referenceTaskName": "join_dynamic", "retryCount": 0, "seq": 2, "pollCount": 0, "taskDefName": "JOIN", "scheduledTime": 1656096815570, "startTime": 1656096815566, "endTime": 1656096815708, "updateTime": 1656096815631, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "fe4efd7b-73ea-4c48-8147-840fa4e1e63b", "workflowType": "example_dynamic_tasks", "taskId": "00781ceb-1931-4537-a4f5-ab38b04015f1", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -4, "loopOverTask": false } ], "input": { "tasksJSON": [], "tasksInputJSON": {} }, "output": {}, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "createTime": 1656005417724, "updateTime": 1656005671608, "name": "example_dynamic_tasks", "description": "A workflow that allows dynamic execution of tasks", "version": 2, "tasks": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": ["tasksJSON", "tasksInputJSON"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "mwi-workflow-dev@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1656096815470, "workflowName": "example_dynamic_tasks", "workflowVersion": 2 } ================================================ FILE: ui/cypress/fixtures/dynamicFork/notExecuted.json ================================================ { "ownerApp": "nq_mwi_conductor_ui_server", "createTime": 1656017015654, "status": "COMPLETED", "endTime": 1656017016239, "workflowId": "5daaf83f-e1f4-454f-9293-4d0443c6c729", "tasks": [ { "taskType": "SWITCH", "status": "COMPLETED", "inputData": { "case": "false" }, "referenceTaskName": "switch_task", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "SWITCH", "scheduledTime": 1656017015966, "startTime": 1656017015955, "endTime": 1656017016105, "updateTime": 1656017015987, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "5daaf83f-e1f4-454f-9293-4d0443c6c729", "workflowType": "example_dynamic_tasks_switch", "taskId": "d0d6ab7b-ac8f-4754-9020-3ea13429d92b", "callbackAfterSeconds": 0, "outputData": { "evaluationResult": ["false"] }, "workflowTask": { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${workflow.input.runFork}" }, "type": "SWITCH", "decisionCases": { "true": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "evaluatorType": "value-param", "expression": "switchCaseValue" }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -11, "loopOverTask": false } ], "input": { "runFork": false }, "output": { "evaluationResult": ["false"] }, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "createTime": 1656015554295, "updateTime": 1656015597435, "name": "example_dynamic_tasks_switch", "description": "A workflow that allows dynamic execution of tasks", "version": 2, "tasks": [ { "name": "switch_task", "taskReferenceName": "switch_task", "inputParameters": { "switchCaseValue": "${workflow.input.runFork}" }, "type": "SWITCH", "decisionCases": { "true": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [], "evaluatorType": "value-param", "expression": "switchCaseValue" } ], "inputParameters": ["tasksJSON", "tasksInputJSON"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "peterl@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1656017015654, "workflowName": "example_dynamic_tasks_switch", "workflowVersion": 2 } ================================================ FILE: ui/cypress/fixtures/dynamicFork/oneFailed.json ================================================ { "ownerApp": "nq_mwi_conductor_ui_server", "createTime": 1656008463986, "status": "FAILED", "endTime": 1656008464720, "workflowId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "tasks": [ { "taskType": "FORK", "status": "COMPLETED", "inputData": { "forkedTaskDefs": [ { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": null }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkedTasks": ["first_task", "second_task", "third_task"] }, "referenceTaskName": "dynamic_tasks", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "FORK", "scheduledTime": 1656008464075, "startTime": 1656008464065, "endTime": 1656008464065, "updateTime": 1656008464094, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "workflowType": "example_dynamic_tasks", "taskId": "743d552b-a683-473d-831e-bb7fae622e08", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -10, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 46, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "first_task", "retryCount": 0, "seq": 2, "pollCount": 0, "taskDefName": "first_task", "scheduledTime": 1656008464077, "startTime": 1656008464069, "endTime": 1656008464374, "updateTime": 1656008464148, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "workflowType": "example_dynamic_tasks", "taskId": "4ccaa5a8-59d0-40d7-b5f8-918f22c2536f", "callbackAfterSeconds": 0, "outputData": { "result": 45 }, "workflowTask": { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -8, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "FAILED", "inputData": { "number": 234, "scriptExpression": null }, "referenceTaskName": "second_task", "retryCount": 0, "seq": 3, "pollCount": 0, "taskDefName": "second_task", "scheduledTime": 1656008464081, "startTime": 1656008464072, "endTime": 1656008464428, "updateTime": 1656008464202, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "workflowType": "example_dynamic_tasks", "taskId": "33e96a6e-5096-4004-ac29-87e0732232f5", "reasonForIncompletion": "Empty 'scriptExpression' in Lambda task's input parameters. A non-empty String value must be provided.", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": null }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -9, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 12, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "third_task", "retryCount": 0, "seq": 4, "pollCount": 0, "taskDefName": "third_task", "scheduledTime": 1656008464083, "startTime": 1656008464073, "endTime": 1656008464482, "updateTime": 1656008464257, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "workflowType": "example_dynamic_tasks", "taskId": "951ef8b2-fa61-4896-bdf9-e781fded8e82", "callbackAfterSeconds": 0, "outputData": { "result": 11 }, "workflowTask": { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -10, "loopOverTask": false }, { "taskType": "JOIN", "status": "FAILED", "inputData": { "joinOn": ["first_task", "second_task", "third_task"] }, "referenceTaskName": "join_dynamic", "retryCount": 0, "seq": 5, "pollCount": 0, "taskDefName": "JOIN", "scheduledTime": 1656008464086, "startTime": 1656008464073, "endTime": 1656008464537, "updateTime": 1656008464312, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "d4b14434-73a7-4be9-b085-d4b40b30856e", "workflowType": "example_dynamic_tasks", "taskId": "6471a9e8-3049-40f2-9ab8-c75876c9c1a3", "reasonForIncompletion": "Empty 'scriptExpression' in Lambda task's input parameters. A non-empty String value must be provided. ", "callbackAfterSeconds": 0, "outputData": { "first_task": { "result": 45 } }, "workflowTask": { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -13, "loopOverTask": false } ], "input": { "tasksJSON": [ { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": null }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "tasksInputJSON": { "first_task": { "number": 46 }, "second_task": { "number": 234 }, "third_task": { "number": 12 } } }, "output": { "first_task": { "result": 45 } }, "reasonForIncompletion": "Empty 'scriptExpression' in Lambda task's input parameters. A non-empty String value must be provided.", "taskToDomain": {}, "failedReferenceTaskNames": ["second_task", "join_dynamic"], "workflowDefinition": { "createTime": 1656005417724, "updateTime": 1656005671608, "name": "example_dynamic_tasks", "description": "A workflow that allows dynamic execution of tasks", "version": 2, "tasks": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": ["tasksJSON", "tasksInputJSON"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "mwi-workflow-dev@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1656008463986, "workflowName": "example_dynamic_tasks", "workflowVersion": 2 } ================================================ FILE: ui/cypress/fixtures/dynamicFork/success.json ================================================ { "ownerApp": "nq_mwi_conductor_ui_server", "createTime": 1656008300448, "status": "COMPLETED", "endTime": 1656008301210, "workflowId": "e66254b6-388d-43a6-b890-c518df832e51", "tasks": [ { "taskType": "FORK", "status": "COMPLETED", "inputData": { "forkedTaskDefs": [ { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkedTasks": ["first_task", "second_task", "third_task"] }, "referenceTaskName": "dynamic_tasks", "retryCount": 0, "seq": 1, "pollCount": 0, "taskDefName": "FORK", "scheduledTime": 1656008300534, "startTime": 1656008300525, "endTime": 1656008300525, "updateTime": 1656008300549, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "b49dc1be-66eb-4816-8ee1-6aaea25f14ba", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -9, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 46, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "first_task", "retryCount": 0, "seq": 2, "pollCount": 0, "taskDefName": "first_task", "scheduledTime": 1656008300535, "startTime": 1656008300527, "endTime": 1656008300922, "updateTime": 1656008300628, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "6ce064a7-7ef3-413c-b6af-318cb7e6751e", "callbackAfterSeconds": 0, "outputData": { "result": 45 }, "workflowTask": { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -8, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 234, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "second_task", "retryCount": 0, "seq": 3, "pollCount": 0, "taskDefName": "second_task", "scheduledTime": 1656008300537, "startTime": 1656008300529, "endTime": 1656008300977, "updateTime": 1656008300683, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "b936ea24-9c3e-4651-8702-2ff5aa4dd579", "callbackAfterSeconds": 0, "outputData": { "result": 233 }, "workflowTask": { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -8, "loopOverTask": false }, { "taskType": "LAMBDA", "status": "COMPLETED", "inputData": { "number": 12, "scriptExpression": "return $.number - 1;" }, "referenceTaskName": "third_task", "retryCount": 0, "seq": 4, "pollCount": 0, "taskDefName": "third_task", "scheduledTime": 1656008300540, "startTime": 1656008300531, "endTime": 1656008301031, "updateTime": 1656008300760, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "bf2963cd-e545-4a26-b533-2ae760e77634", "callbackAfterSeconds": 0, "outputData": { "result": 11 }, "workflowTask": { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -9, "loopOverTask": false }, { "taskType": "JOIN", "status": "COMPLETED", "inputData": { "joinOn": ["first_task", "second_task", "third_task"] }, "referenceTaskName": "join_dynamic", "retryCount": 0, "seq": 5, "pollCount": 0, "taskDefName": "JOIN", "scheduledTime": 1656008300542, "startTime": 1656008300531, "endTime": 1656008301085, "updateTime": 1656008300831, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "e66254b6-388d-43a6-b890-c518df832e51", "workflowType": "example_dynamic_tasks", "taskId": "25ddfe4d-eaf0-4171-964f-9b53ad06002b", "callbackAfterSeconds": 0, "outputData": { "second_task": { "result": 233 }, "third_task": { "result": 11 }, "first_task": { "result": 45 } }, "workflowTask": { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": -11, "loopOverTask": false } ], "input": { "tasksJSON": [ { "name": "first_task", "taskReferenceName": "first_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "second_task", "taskReferenceName": "second_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "third_task", "taskReferenceName": "third_task", "inputParameters": { "number": "${number}", "scriptExpression": "return $.number - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "tasksInputJSON": { "first_task": { "number": 46 }, "second_task": { "number": 234 }, "third_task": { "number": 12 } } }, "output": { "second_task": { "result": 233 }, "third_task": { "result": 11 }, "first_task": { "result": 45 } }, "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "createTime": 1656005417724, "updateTime": 1656005671608, "name": "example_dynamic_tasks", "description": "A workflow that allows dynamic execution of tasks", "version": 2, "tasks": [ { "name": "dynamic_tasks", "taskReferenceName": "dynamic_tasks", "inputParameters": { "dynamicTasks": "${workflow.input.tasksJSON}", "dynamicTasksInput": "${workflow.input.tasksInputJSON}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "dynamicTasks", "dynamicForkTasksInputParamName": "dynamicTasksInput", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "join", "taskReferenceName": "join_dynamic", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": ["tasksJSON", "tasksInputJSON"], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "mwi-workflow-dev@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1656008300448, "workflowName": "example_dynamic_tasks", "workflowVersion": 2 } ================================================ FILE: ui/cypress/fixtures/dynamicFork.json ================================================ { "ownerApp": "", "createTime": 1608153919527, "status": "TERMINATED", "endTime": 1608173713271, "workflowId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "parentWorkflowId": "9f9057d0-86c4-464f-b4d0-1606e66798fd", "parentWorkflowTaskId": "1f908fd3-b02f-47f1-b223-7625bc2da1a3", "tasks": [ { "taskType": "FORK", "status": "COMPLETED", "inputData": { "forkedTaskDefs": [ { "name": "processshot", "taskReferenceName": "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "processshot", "taskReferenceName": "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkedTasks": [ "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2", "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2", "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2" ] }, "referenceTaskName": "shot_processing", "retryCount": 0, "seq": 52, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 0, "taskDefName": "FORK", "scheduledTime": 1608154318377, "startTime": 0, "endTime": 1608154318334, "updateTime": 1608154318402, "startDelayInSeconds": 0, "retried": false, "executed": true, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "taskId": "1a859277-c27e-4d28-bb57-271ea5a16f96", "callbackAfterSeconds": 0, "outputData": {}, "workflowTask": { "name": "asset_processing", "taskReferenceName": "shot_processing", "inputParameters": { "taskDefs": "${prepareShotProcessingTasks.output.result.taskDefs}", "taskInputs": "${prepareShotProcessingTasks.output.result.taskInputs}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "taskDefs", "dynamicForkTasksInputParamName": "taskInputs", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 0, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "4cf51bb0-3fe3-11eb-8740-12f4b5a75f47" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "4cf45860-3fe3-11eb-8740-12f4b5a75f47" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "4cf45860-3fe3-11eb-8740-12f4b5a75f47:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2", "retryCount": 0, "seq": 53, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318379, "startTime": 1608154318538, "endTime": 1608173718867, "updateTime": 1608154318720, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "30f507f2-fc34-4123-9ffb-07af344a56b0", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "5f8285f0-72de-4233-a201-1599b558e645" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "5f8285f0-72de-4233-a201-1599b558e645", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211706, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "4f943090-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "4f936d40-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "4f936d40-3fe3-11eb-9af1-12a7f1c641e3:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2", "retryCount": 0, "seq": 54, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318382, "startTime": 1608154318569, "endTime": 1608173719056, "updateTime": 1608154318774, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "fa90e44a-d68c-40b8-84a7-ebbcd21b94e3", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "ef25bcc5-f5e6-4172-99f5-7fb76b1f11f8" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "ef25bcc5-f5e6-4172-99f5-7fb76b1f11f8", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211652, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "52576f40-3fe3-11eb-8a3e-12ffdb69dc47" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "5256abf0-3fe3-11eb-8a3e-12ffdb69dc47" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "5256abf0-3fe3-11eb-8a3e-12ffdb69dc47:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "retryCount": 0, "seq": 55, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318384, "startTime": 1608154318582, "endTime": 1608173719208, "updateTime": 1608154318774, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "0cde3f5c-3fb0-4c4e-b74c-3a7bcabf892c", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "28564fa1-c0d3-45fa-948d-969c09f49af5" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "28564fa1-c0d3-45fa-948d-969c09f49af5", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211652, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "54ec9910-3fe3-11eb-bd3b-1230be2091b7" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "54ebd5c0-3fe3-11eb-bd3b-1230be2091b7" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "54ebd5c0-3fe3-11eb-bd3b-1230be2091b7:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2", "retryCount": 0, "seq": 56, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318386, "startTime": 1608154318637, "endTime": 1608173719368, "updateTime": 1608154318882, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "88595b13-34e2-4442-994d-2875f21f44f7", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "ebce2351-2ce2-4920-899d-fffbf66be4f4" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "ebce2351-2ce2-4920-899d-fffbf66be4f4", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211544, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "57784d02-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "57784d00-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "57784d00-3fe3-11eb-9af1-12a7f1c641e3:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2", "retryCount": 0, "seq": 57, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318387, "startTime": 1608154318657, "endTime": 1608173719530, "updateTime": 1608154318987, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "f2e835e5-ce1b-4d9b-91f7-5ffa5b43a40e", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "7e326487-f315-43b6-aab4-279c82155132" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "7e326487-f315-43b6-aab4-279c82155132", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211439, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "59f3ad42-3fe3-11eb-8a3e-12ffdb69dc47" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "retryCount": 0, "seq": 58, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318390, "startTime": 1608154318739, "endTime": 1608173719684, "updateTime": 1608154318987, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "74fbb03c-5a2c-46e9-b429-19053cd1a3ec", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "2be7d404-3732-4e90-88e5-b0b221aeeda1" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "2be7d404-3732-4e90-88e5-b0b221aeeda1", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211439, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n��� *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "5c52abe1-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "5c51e890-3fe3-11eb-9af1-12a7f1c641e3" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "5c51e890-3fe3-11eb-9af1-12a7f1c641e3:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2", "retryCount": 0, "seq": 59, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318393, "startTime": 1608154318748, "endTime": 1608173719850, "updateTime": 1608154319021, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "abb3eebf-8430-446a-a5fa-dd26cf367a95", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "8f3e3098-72c7-40b7-8547-4cc2293a6def" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "8f3e3098-72c7-40b7-8547-4cc2293a6def", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211405, "loopOverTask": false }, { "taskType": "SUB_WORKFLOW", "status": "CANCELED", "inputData": { "playlistId": 375594, "metadata": { "submission_note": null, "version_name": null, "scope_of_work": null, "reason_for_review": "• *SUBMITTING FOR* - null\n• *SCOPE OF WORK* - null\n• *NOTES* - null", "vendor": null, "link": null, "submitting_for": null, "sort_order": null }, "subWorkflowTaskToDomain": null, "subWorkflowName": "vfxmediareview.shotprocessing_v2", "reviewAssetId": { "versionId": "1.4", "id": "5ec02971-3fe3-11eb-bd3b-1230be2091b7" }, "vendorId": null, "shotname": null, "topLevelAssetId": { "versionId": "1.2", "id": "5ec00260-3fe3-11eb-bd3b-1230be2091b7" }, "vendorName": "Netflix", "reviewProjectId": "222", "subWorkflowVersion": 1, "playlistName": "HUB-3087_20201216_01", "subWorkflowDefinition": null, "workflowInput": {}, "sgAmpRefId": "5ec00260-3fe3-11eb-bd3b-1230be2091b7:1.2", "reviewServer": "netflix-review-staging.shotgunstudio.com" }, "referenceTaskName": "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2", "retryCount": 0, "seq": 60, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 1, "taskDefName": "processshot", "scheduledTime": 1608154318394, "startTime": 1608154318871, "endTime": 1608173719989, "updateTime": 1608154319182, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "69630514-33df-42ba-805e-af17760ba111", "callbackAfterSeconds": 30, "outputData": { "subWorkflowId": "b53c817d-19a5-436e-b26c-0b57d334b122" }, "workflowTask": { "name": "processshot", "taskReferenceName": "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2", "inputParameters": {}, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotprocessing_v2" }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subWorkflowId": "b53c817d-19a5-436e-b26c-0b57d334b122", "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 47847211244, "loopOverTask": false }, { "taskType": "JOIN", "status": "CANCELED", "inputData": { "joinOn": [ "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2", "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2", "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2", "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2", "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2" ] }, "referenceTaskName": "shot_processing_join", "retryCount": 0, "seq": 61, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "pollCount": 0, "taskDefName": "JOIN", "scheduledTime": 1608154318395, "startTime": 1608154318407, "endTime": 1608173719994, "updateTime": 1608154318407, "startDelayInSeconds": 0, "retried": false, "executed": false, "callbackFromWorker": true, "responseTimeoutSeconds": 0, "workflowInstanceId": "637364c4-31bf-4c50-8c81-c04d1dafe27f", "workflowType": "pipelines.vfxmediareview", "taskId": "80a2e505-2f54-4d40-bcee-17f8c6a39b71", "callbackAfterSeconds": 0, "outputData": { "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2": {}, "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2": {}, "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2": {}, "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2": {}, "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2": {}, "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2": {}, "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2": {}, "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2": {} }, "workflowTask": { "name": "shot_processing_join", "taskReferenceName": "shot_processing_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 0, "workflowPriority": 0, "iteration": 0, "subworkflowChanged": false, "taskDefinition": null, "queueWaitTime": 12, "loopOverTask": false } ], "input": { "pipelineInput": { "pipelineConfig": { "requestNamespace": "pipelines", "requestType": "vfxmediareview", "type": "vfxmediareview" }, "primaryRequestNamespace": "pipelines", "primaryRequestId": "20fa83b7-9b91-406a-9644-4fd62aea8b4b", "user": "jcronk@netflix.com", "inputParameters": { "submissionId": "HUB-3087_20201216_01", "ownerUser": "jcronk@netflix.com", "submissionNodeId": "229590a0-3fe5-11eb-9910-12d0bc41bfa1", "vendorId": null, "reviewType": "PRODUCTION", "movieId": 81112280, "projectId": "92311fe0-5bd7-11e9-b8ed-0e4d3942d506" }, "pipelineId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "primaryRequestType": "vfxmediareview" } }, "output": { "processshot_59f3ad40-3fe3-11eb-8a3e-12ffdb69dc47_1.2": {}, "processshot_5c51e890-3fe3-11eb-9af1-12a7f1c641e3_1.2": {}, "processshot_54ebd5c0-3fe3-11eb-bd3b-1230be2091b7_1.2": {}, "processshot_4cf45860-3fe3-11eb-8740-12f4b5a75f47_1.2": {}, "processshot_5ec00260-3fe3-11eb-bd3b-1230be2091b7_1.2": {}, "processshot_4f936d40-3fe3-11eb-9af1-12a7f1c641e3_1.2": {}, "processshot_5256abf0-3fe3-11eb-8a3e-12ffdb69dc47_1.2": {}, "processshot_57784d00-3fe3-11eb-9af1-12a7f1c641e3_1.2": {} }, "correlationId": "6980f3f8-9077-45ca-9f0f-5d9545799a61", "reasonForIncompletion": "Parent workflow has been terminated with status TIMED_OUT", "taskToDomain": {}, "failedReferenceTaskNames": [], "workflowDefinition": { "updateTime": 1608073180721, "name": "pipelines.vfxmediareview", "version": 1, "tasks": [ { "name": "stl.pipeline.init", "taskReferenceName": "initializePipeline", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385855, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.init", "description": "Initial task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": ["pipeline"], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.get", "taskReferenceName": "pipelineData", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385634, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.get", "description": "Read pipeline given pipeline id", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": ["pipeline"], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "reviewType", "inputParameters": { "inputData": "${pipelineData.output.request.request.data.reviewType}", "expression": ". | ascii_downcase" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.getProjectSchema", "taskReferenceName": "reviewServerSchema", "inputParameters": { "schemaGroup": "PIPELINE", "schemaType": "${reviewType.output.result}review", "projectId": "${pipelineData.output.request.request.data.projectId}", "user": "contenthub-system-user@netflix.com" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1588011760479, "createdBy": "CPEWORKFLOW", "name": "stl.common.getProjectSchema", "description": "Get Project Schema", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["requestId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "cpe-che-backend@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "reviewServerConfig", "inputParameters": { "inputData": "${reviewServerSchema.output.output}", "expression": ".[0].schema as $s | if $s.server == null or $s.projectId == null then error(\"Configuration cannot be null\") else $s end" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "vendorDetails", "taskReferenceName": "vendorDetails", "inputParameters": { "vendorId": "${pipelineData.output.request.request.data.vendorId}" }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.vendordetails", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "processSubmission", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "process", "details": { "skipPostProcess": true }, "skipIfInState": ["IN_PROGRESS"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.map", "taskReferenceName": "details", "inputParameters": { "mappings": { "contenthubBaseUrl": "@environment.getProperty('contenthub.url')", "contenthubProjectUrl": "@environment.getProperty('contenthub.url').concat(\"/projects/${pipelineData.output.request.request.data.projectId}\")" } }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082386616, "createdBy": "CPEWORKFLOW", "name": "stl.common.map", "description": "General purpose task to apply expression language (SpEL) transforms", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["mappings"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decide", "taskReferenceName": "assetDiscoveryFlow", "inputParameters": { "status": "${pipelineData.output.request.request.data.skipAssetDiscovery}" }, "type": "DECISION", "caseValueParam": "status", "decisionCases": { "true": [ { "name": "stl.common.noop", "taskReferenceName": "proceed", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082386204, "createdBy": "CPEWORKFLOW", "name": "stl.common.noop", "description": "Do nothing", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "stl.common.jq", "taskReferenceName": "getPipelineResourceIds", "inputParameters": { "inputData": { "pipeline": "${pipelineData.output}" }, "expression": "[.pipeline.pipelineId] + (.pipeline.pipelineResources // [] | map(.id))" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.manageResourcesToPipelineTask", "taskReferenceName": "detachPipelineResources", "inputParameters": { "pipelineResourceManagementInput": { "detachById": "${getPipelineResourceIds.output.result}" }, "pipelineId": "${pipelineData.output.pipelineId}", "contextUser": "pipelineapi" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1575590191136, "createdBy": "CPEWORKFLOW", "name": "stl.common.manageResourcesToPipelineTask", "description": "Attach / detach resources on a pipeline", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "pipelineId", "contextUser", "pipelineResourceManagementInput" ], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "assetdiscovery", "taskReferenceName": "assetdiscovery", "inputParameters": { "pipelineId": "${pipelineData.output.pipelineId}", "folderId": "${pipelineData.output.request.request.data.submissionNodeId}" }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.assetdiscovery", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decide", "taskReferenceName": "checkDiscoveryOutcome", "inputParameters": { "outcome": "${assetdiscovery.output.outcome}" }, "type": "DECISION", "caseValueParam": "outcome", "decisionCases": { "MANIFEST_NOT_FOUND": [ { "name": "stl.common.jq", "taskReferenceName": "prepareRecipientDomainsForManifestNotFound", "inputParameters": { "inputData": { "reviewServerConfig": "${reviewServerConfig.output.result}", "domains": [ { "domains": ["netflix.amp.domain.studio_vfx"] }, { "domains": ["netflix.amp.domain.production_vfx"], "partnerId": "${pipelineData.output.request.request.data.vendorId}", "ignoreIfPartnerNull": true }, { "profiles": ["PRODUCTION_VFX_ADMIN"], "includeUsersWithProfile": true } ], "fallbackDomains": [ { "domains": ["netflix.amp.domain.studio_vfx"] } ] }, "expression": ". as $in | $in.domains" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sendManifestErrorsEmail", "taskReferenceName": "sendManifestNotFoundEmail", "inputParameters": { "emailType": "SINGLE", "domains": "${prepareRecipientDomainsForManifestNotFound.output.result}", "additionalRecipients": [ "vfx-media-review@netflix.com", "e1u9g2p6u1b8e5x7@netflix.slack.com" ], "additionalUsers": [ "${pipelineData.output.request.request.data.ownerUser}" ], "eventName": "EVENT_MESSAGE_VFX_REVIEW_SUBMISSION", "eventType": "MESSAGE_VFX_REVIEW_SUBMISSION", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "emailPayload": { "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionId": "${pipelineData.output.request.request.data.submissionId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "pipelineId": "${pipelineData.output.pipelineId}", "subject": "Processing of submission ${pipelineData.output.request.request.data.submissionId} failed!", "manifestName": "${assetDiscovery.output.manifestName}", "manifestIncluded": false, "status": "FAILED", "oldCsv": "${oldCsv.output.result}", "chProjectId": "${pipelineData.output.request.request.data.projectId}" } }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.notification", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "requestManifestDelivery", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "redeliver", "skipIfInState": ["REDELIVERY"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_SubmissionRejected1", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "SubmissionRejected_1", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": { "outcome": "SUBMISSION_REJECTED", "code": "MANIFEST_NOT_FOUND" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "DISCOVERY_ERROR": [ { "name": "stl.common.jq", "taskReferenceName": "prepareRecipientDomainsForManifestErrors", "inputParameters": { "inputData": { "reviewServerConfig": "${reviewServerConfig.output.result}", "domains": [ { "domains": ["netflix.amp.domain.studio_vfx"] }, { "domains": ["netflix.amp.domain.production_vfx"], "partnerId": "${pipelineData.output.request.request.data.vendorId}", "ignoreIfPartnerNull": true }, { "profiles": ["PRODUCTION_VFX_ADMIN"], "includeUsersWithProfile": true } ], "fallbackDomains": [ { "domains": ["netflix.amp.domain.studio_vfx"] } ] }, "expression": ". as $in | $in.domains" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sendManifestErrorsEmail", "taskReferenceName": "sendManifestErrorsEmail", "inputParameters": { "emailType": "SINGLE", "domains": "${prepareRecipientDomainsForManifestErrors.output.result}", "additionalRecipients": [ "vfx-media-review@netflix.com", "e1u9g2p6u1b8e5x7@netflix.slack.com" ], "additionalUsers": [ "${pipelineData.output.request.request.data.ownerUser}" ], "eventName": "EVENT_MESSAGE_VFX_REVIEW_SUBMISSION", "eventType": "MESSAGE_VFX_REVIEW_SUBMISSION", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "emailPayload": { "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionId": "${pipelineData.output.request.request.data.submissionId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "manifestErrors": "${assetDiscovery.output.displayErrors}", "pipelineId": "${pipelineData.output.pipelineId}", "subject": "Processing of submission ${pipelineData.output.request.request.data.submissionId} failed!", "manifestName": "${assetDiscovery.output.manifestName}", "manifestIncluded": true, "manifestLink": "${details.output.submissionUri}&nodeIds=${assetDiscovery.output.manifestNodeId}", "filesMissingInManifest": "${assetDiscovery.output.filesMissingInManifest}", "filesMissingInSubmission": "${assetDiscovery.output.filesMissingInSubmission}", "manifestWarnings": "${assetDiscovery.output.displayWarnings}", "status": "FAILED", "oldCsv": "${oldCsv.output.result}", "chProjectId": "${pipelineData.output.request.request.data.projectId}" } }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.notification", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "requestManifestRedelivery", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "redeliver", "skipIfInState": ["REDELIVERY"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_SubmissionRejected2", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "SubmissionRejected_2", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": { "outcome": "SUBMISSION_REJECTED", "code": "MANIFEST_WITH_ERRORS" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "MULTIPLE_MANIFESTS_FOUND": [ { "name": "stl.common.jq", "taskReferenceName": "prepareRecipientDomainsForEncodingErrors1", "inputParameters": { "inputData": { "reviewServerConfig": "${reviewServerConfig.output.result}", "domains": [ { "domains": ["netflix.amp.domain.studio_vfx"] }, { "domains": ["netflix.amp.domain.production_vfx"], "partnerId": "${pipelineData.output.request.request.data.vendorId}", "ignoreIfPartnerNull": true }, { "profiles": ["PRODUCTION_VFX_ADMIN"], "includeUsersWithProfile": true } ], "fallbackDomains": [ { "domains": ["netflix.amp.domain.studio_vfx"] } ] }, "expression": ". as $in | $in.domains" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sendMultipleManifestsEmail", "taskReferenceName": "sendMultipleManifestsEmail", "inputParameters": { "emailType": "SINGLE", "domains": "${prepareRecipientDomainsForEncodingErrors1.output.result}", "additionalRecipients": [ "vfx-media-review@netflix.com", "e1u9g2p6u1b8e5x7@netflix.slack.com" ], "additionalUsers": [ "${pipelineData.output.request.request.data.ownerUser}" ], "eventName": "EVENT_MESSAGE_VFX_REVIEW_SUBMISSION", "eventType": "MESSAGE_VFX_REVIEW_SUBMISSION", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "emailPayload": { "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionId": "${pipelineData.output.request.request.data.submissionId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "pipelineId": "${pipelineData.output.pipelineId}", "subject": "Processing of submission ${pipelineData.output.request.request.data.submissionId} failed!", "chProjectId": "${pipelineData.output.request.request.data.projectId}", "multipleManifests": true, "status": "FAILED", "oldCsv": "${oldCsv.output.result}" } }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.notification", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_SubmissionRejected3", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "SubmissionRejected_3", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": { "outcome": "SUBMISSION_REJECTED", "code": "MULTIPLE_MANIFESTS_FOUND" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "addCdriveNodeIdAsExternalStatus", "inputParameters": { "inputData": "${assetdiscovery.output.assets}", "expression": "[.[] | . as $in | { derivations:.derivations, movieId:.movieId, assetTree: {derivatives: .assetTree.derivatives, assets:.assetTree.assets | (to_entries | map( if(.value.payload.file != null) then .value.payload.externalStatuses |= .+{\"originalCDriveNodeId\": {value: $in.derivations.snapshot | to_entries | .[].value.nodeId}} else . end) | from_entries ), rootRefId:.assetTree.rootRefId, relations:.assetTree.relations}}]" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareImportAssetsFork", "inputParameters": { "inputData": { "assetIngestRequests": "${addCdriveNodeIdAsExternalStatus.output.result}", "user": "${workflow.input.pipelineInput.user}", "limit": 20, "partnerId": "${pipelineData.output.request.request.data.vendorId}" }, "expression": ". as $in | .assetIngestRequests | {taskDefs: map( {subWorkflowParam:{name:\"vfxmediareview.createandshareassets\"},name:\"createandshareassets\", taskReferenceName :\"createandshareassets_\\(.derivations.snapshot[]|.nodeId)\",\"type\": \"SUB_WORKFLOW\"}), \"taskInputs\":map({key:\"createandshareassets_\\(.derivations.snapshot[]|.nodeId)\", value:{assetIngestRequests:[.], user:$in.user, partnerId:$in.partnerId}})| from_entries}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "importAsset", "taskReferenceName": "createandshareassets", "inputParameters": { "taskDefs": "${prepareImportAssetsFork.output.result.taskDefs}", "taskInputs": "${prepareImportAssetsFork.output.result.taskInputs}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "taskDefs", "dynamicForkTasksInputParamName": "taskInputs", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "createandshareassets_join", "taskReferenceName": "createandshareassets_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.manageResourcesToPipelineTask", "taskReferenceName": "managePipelineResources", "inputParameters": { "pipelineResourceManagementInput": "${assetdiscovery.output.pipelineResourceManagementInput}", "pipelineId": "${workflow.input.pipelineInput.pipelineId}", "contextUser": "pipelineapi" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 60, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1575590191136, "createdBy": "CPEWORKFLOW", "name": "stl.common.manageResourcesToPipelineTask", "description": "Attach / detach resources on a pipeline", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "pipelineId", "contextUser", "pipelineResourceManagementInput" ], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.get", "taskReferenceName": "pipelineDataAfterParsing", "inputParameters": { "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385634, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.get", "description": "Read pipeline given pipeline id", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": ["pipeline"], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.AwaitStatusMatchPipelineResource", "taskReferenceName": "awaitPipelineResourceState", "inputParameters": { "pipelineId": "${pipelineData.output.pipelineId}", "selector": { "resourceTypes": ["AMP_ASSET"] }, "allowedStatuses": ["COMPLETED", "FAILED"], "retryAfterSeconds": "60", "user": "pipelineapi" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1599196237488, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.AwaitStatusMatchPipelineResource", "description": "Wait for Resources to MatchStates", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "pipelineId", "selector", "allowedStatuses", "retryAfterSeconds", "user" ], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "cpe-che-backend@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.get", "taskReferenceName": "pipelineDataAndResources", "inputParameters": { "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385634, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.get", "description": "Read pipeline given pipeline id", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": ["pipeline"], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "manifestResource", "inputParameters": { "inputData": "${pipelineDataAndResources.output.pipelineResources}", "expression": "map(select(.attachmentType == \"CSV\"))[0]" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "oldCsv", "inputParameters": { "inputData": "${manifestResource.output.result.resourceContext.entries}", "expression": ". // [] | any(has(\"Primary Shot Type\") or has(\"Shot Types\") or has(\"Shot Type\"))" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "manifestNodeDetailsTask", "inputParameters": { "inputData": "${pipelineDataAndResources.output.pipelineResources}", "expression": "map(select(.attachmentType == \"CSV\") | .resourceIdentity.resourceId)[0] as $rid | $rid | if . == null then (\"stl.common.noop\") else (\"stl.cdrive.downloadManifest\") end | { task: ., nodeId: $rid }" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.cdrive.downloadManifest", "taskReferenceName": "manifestNodeDetails", "inputParameters": { "task": "${manifestNodeDetailsTask.output.result.task}", "nodeId": "${manifestNodeDetailsTask.output.result.nodeId}", "userId": "${pipelineData.output.request.request.data.ownerUser}", "useAppAuth": true }, "type": "DYNAMIC", "dynamicTaskNameParam": "task", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082388180, "createdBy": "CPEWORKFLOW", "name": "stl.cdrive.downloadManifest", "description": "get the cdrive manifest of a node.", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["nodeId", "userId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "manifestDetails", "inputParameters": { "inputData": "${manifestNodeDetails.output.output}", "expression": "if .downloadManifest != null then .assets[0] | { manifestName: .fileName[(.fileName|index(\"/\"))+1:], manifestBagginsUrl: .bagginsUrl, manifestNodeId: .nodeId, manifestIncluded: true } else {} end" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "getCreatedAssets", "inputParameters": { "inputData": "${pipelineDataAndResources.output.pipelineResources}", "expression": "map(select(.resourceIdentity.resourceType == \"AMP_ASSET\") | .resourceIdentity | { assetId: .resourceId, versionId: .versionId })" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareGetAssetsTasks", "inputParameters": { "inputData": "${getCreatedAssets.output.result}", "expressions": ["map({id: .assetId, version: .versionId})"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.amp.getAssetsTree", "taskReferenceName": "assets_tree", "inputParameters": { "assetIds": "${prepareGetAssetsTasks.output.result}", "derivatives": ["PROXY"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1600451428431, "createdBy": "CPEWORKFLOW", "name": "stl.amp.getAssetsTree", "description": "Get the full asset tree", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["assetIds", "derivatives", "relations"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "cpe-che-backend@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "assetsByType", "inputParameters": { "inputData": "${assets_tree.output.output}", "expressions": [ "map( .assets | to_entries | map(.value)) | {proxies: map(select(any(.payload.type == \"PMR_REVIEW_PROXY\")) | map({assetId: .assetId, type: .payload.type})), images: map(select(any(.payload.type == \"IMAGE\")) | map({assetId: .assetId, type: .payload.type}))} " ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareAssetProcessingTasks", "inputParameters": { "inputData": { "assets": "${assetsByType.output.result}", "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "expressions": [ ". as $in | [$in.assets.proxies, $in.assets.images] | add | if . | length == 0 then [] else . | add end | map(select(.type == \"PMR_REVIEW_VERSION\") | {assetId: .assetId.id, versionId: .assetId.version}) | {assets: . , pipelineId: $in.pipelineId} as $in |", " $in.assets | { taskDefs: map( {type: \"SUB_WORKFLOW\", name: \"process_asset\", taskReferenceName: \"process_asset_\\(.assetId)_\\(.versionId)\", subWorkflowParam: {name: \"vfxmediareview.assetprocessing\"}}),", " taskInputs: map( { key: \"process_asset_\\(.assetId)_\\(.versionId)\", value: {assetId, versionId, pipelineId: $in.pipelineId}}) | from_entries }" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "asset_processing", "taskReferenceName": "asset_processing", "inputParameters": { "taskDefs": "${prepareAssetProcessingTasks.output.result.taskDefs}", "taskInputs": "${prepareAssetProcessingTasks.output.result.taskInputs}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "taskDefs", "dynamicForkTasksInputParamName": "taskInputs", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "asset_processing_join", "taskReferenceName": "asset_processing_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "evaluateAssetProcessingResults", "inputParameters": { "inputData": "${asset_processing_join.output}", "expression": "to_entries | map(.value | { file, message: .description, outcome }) | { encodedFiles: map({ file, message }), outcome: (map(select(.outcome == \"FAILED\")) | if length > 0 then \"REPORT_ERRORS\" else \"PROCESS_SUBMISSION\" end) }" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decide", "taskReferenceName": "checkDeliveryStatus", "inputParameters": { "outcome": "${evaluateAssetProcessingResults.output.result.outcome}" }, "type": "DECISION", "caseValueParam": "outcome", "decisionCases": { "REPORT_ERRORS": [ { "name": "stl.common.jq", "taskReferenceName": "assetProcessingResults", "inputParameters": { "inputData": "${asset_processing_join.output}", "expression": "to_entries | map(.value | select(.outcome != \"SUCCESS\") | { file, message: .description, outcome }) | { encodedFiles: map({ file, message }), outcome: (map(select(.outcome == \"FAILED\")) | if length > 0 then \"REPORT_ERRORS\" else \"PROCESS_SUBMISSION\" end) }" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareRecipientDomainsForEncodingErrors", "inputParameters": { "inputData": { "reviewServerConfig": "${reviewServerConfig.output.result}", "domains": [ { "domains": ["netflix.amp.domain.studio_vfx"] }, { "domains": ["netflix.amp.domain.production_vfx"], "partnerId": "${pipelineData.output.request.request.data.vendorId}", "ignoreIfPartnerNull": true }, { "profiles": ["PRODUCTION_VFX_ADMIN"], "includeUsersWithProfile": true } ], "fallbackDomains": [ { "domains": ["netflix.amp.domain.studio_vfx"] } ] }, "expression": ". as $in | $in.domains" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "notify_sendEncodingErrorsEmail", "taskReferenceName": "sendEncodingErrorsEmail", "inputParameters": { "emailType": "SINGLE", "domains": "${prepareRecipientDomainsForEncodingErrors.output.result}", "additionalRecipients": [ "vfx-media-review@netflix.com", "e1u9g2p6u1b8e5x7@netflix.slack.com" ], "additionalUsers": [ "${pipelineData.output.request.request.data.ownerUser}" ], "eventName": "EVENT_MESSAGE_VFX_REVIEW_SUBMISSION", "eventType": "MESSAGE_VFX_REVIEW_SUBMISSION", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "emailPayload": { "status": "FAILED", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionId": "${pipelineData.output.request.request.data.submissionId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "filesWithEncodingErrors": "${assetProcessingResults.output.result.encodedFiles}", "pipelineId": "${pipelineData.output.pipelineId}", "subject": "Processing of submission ${pipelineData.output.request.request.data.submissionId} failed!", "chProjectId": "${pipelineData.output.request.request.data.projectId}", "oldCsv": "${oldCsv.output.result}", "manifestWarnings": "${assetdiscovery.output.displayWarnings}" } }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.notification", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "requestRedelivery", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "redeliver", "skipIfInState": ["REDELIVERY"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_SubmissionRejected", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "SubmissionRejected", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": { "outcome": "SUBMISSION_REJECTED", "code": "SOURCE_ERRORS" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "PROCESS_SUBMISSION": [ { "name": "stl.common.noop", "taskReferenceName": "proceedWithShotgunProcessing", "inputParameters": {}, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082386204, "createdBy": "CPEWORKFLOW", "name": "stl.common.noop", "description": "Do nothing", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [ { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_UnknownError1", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "UnknownError1", "inputParameters": { "terminationStatus": "FAILED", "workflowOutput": { "outcome": "UNKNOWN_ERROR", "code": "INTERNAL_ERRORS" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "submissionFolderProjection", "taskReferenceName": "submissionFolderProjection", "inputParameters": { "pipelineId": "${pipelineData.output.request.request.data.pipelineId}", "manifest": { "name": "${manifestDetails.output.result.manifestName}", "bagginsUrl": "${manifestDetails.output.result.manifestBagginsUrl}" }, "assets": "${getCreatedAssets.output.result}", "vendorName": "${vendorDetails.output.vendorName}" }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.projectsubmission", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.map", "taskReferenceName": "pipelineDetails", "inputParameters": { "submissionUri": "workspace?workspaceRoot=SHARED&layout=TREE&categoryType=workspace&workspaceFolderId=${submissionFolderProjection.output.sharedSubmissionFolderId}", "mappings": { "submissionUri": "['submissionUri']" } }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082386616, "createdBy": "CPEWORKFLOW", "name": "stl.common.map", "description": "General purpose task to apply expression language (SpEL) transforms", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["mappings"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.get", "taskReferenceName": "getPipelineData", "inputParameters": { "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385634, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.get", "description": "Read pipeline given pipeline id", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": ["pipeline"], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "updateShotgunProcessingStart", "inputParameters": { "inputData": { "resources": "${getPipelineData.output.pipelineResources}" }, "expression": ".resources | map((select(.attachmentType==\"EXPECTED_ASSET\") | . as $r | $r | .resourceContext.progress | map( if .name == \"SHOTGUN_UPLOAD\" then { name, status: \"IN_PROGRESS\" } else . end ) as $p | $r * { resourceContext: ($r.resourceContext * {progress: $p}) }) )" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.updateResource", "taskReferenceName": "updateResourceShotgunStarted", "inputParameters": { "resources": "${updateShotgunProcessingStart.output.result}", "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1576822186221, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.updateResource", "description": "Update a pipeline resource", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId", "resource"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.titus", "taskReferenceName": "getVendors", "inputParameters": { "applicationName": "che/vfx-review-cli", "version": "${NETFLIX_ENVIRONMENT}.latest", "entryPoint": "python cli.py vendors -s \"https://${reviewServerConfig.output.result.server}\" -p ${reviewServerConfig.output.result.projectId} --rc-conductor \"${CPEWF_TASK_ID}\" " }, "type": "TITUS", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1580327637347, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.titus", "retryCount": 3, "timeoutSeconds": 3600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 90, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": true, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "playlistVendorParameter", "inputParameters": { "inputData": "${getVendors.output.result}", "expressions": [ ".vendors | map(select(.sg_global_vendor.sg_vendor_id == \"${pipelineData.output.request.request.data.vendorId}\"))[0] // null", "| if . then \"--vendor-id \\(.id)\" else \"\" end" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sgPlaylist", "taskReferenceName": "playlistInfo", "inputParameters": { "pipelineId": "${pipelineData.output.request.request.data.pipelineId}", "playlistName": "${pipelineData.output.request.request.data.submissionId}", "contenthubProjectUrl": "${details.output.contenthubProjectUrl}", "reviewServerConfig": { "server": "${reviewServerConfig.output.result.server}", "projectId": "${reviewServerConfig.output.result.projectId}" }, "playlistVendorParameter": "${playlistVendorParameter.output.result}" }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.shotgunplaylist", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "assetAndComputedMetadata", "inputParameters": { "inputData": { "assets": "${assets_tree.output.output}", "entries": "${manifestResource.output.result.resourceContext.entries}" }, "expressions": [ "def sf(e): .submitting_for; ", "def desc(e): .submission_note; ", "def sow(e): .scope_of_work; ", ". as $in | ", "$in.entries | to_entries as $entries | ", "$in.assets | map(.assets | to_entries | map(.value)) | add | map(. as $a | $a | .payload.metadata as $meta | ", "$meta | { metadata: { reason_for_review: ([ \"• *SUBMITTING FOR* - \\(sf(.))\", \"• *SCOPE OF WORK* - \\(sow(.))\", \"• *NOTES* - \\(desc(.))\" ] | join(\"\\n\")), sort_order: $entries | map(select($meta.version_name == .value.version_name))[0].key }, assetId: $a.assetId.id, assetVersion: $a.assetId.version })" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.amp.updateMetadataAndAddTypes", "taskReferenceName": "updateAssetComputedMetadata", "inputParameters": { "assets": "${assetAndComputedMetadata.output.result}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1579029014610, "createdBy": "CPEWORKFLOW", "name": "stl.amp.updateMetadataAndAddTypes", "description": "This will update the metadata and add Type. The needed metadata and type should be the input", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["assets"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.amp.getAssetsTree", "taskReferenceName": "assets_tree_2", "inputParameters": { "assetIds": "${prepareGetAssetsTasks.output.result}", "derivatives": ["PROXY"] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1600451428431, "createdBy": "CPEWORKFLOW", "name": "stl.amp.getAssetsTree", "description": "Get the full asset tree", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["assetIds", "derivatives", "relations"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "cpe-che-backend@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "assetAndManifest", "inputParameters": { "inputData": { "assets": "${assets_tree_2.output.output}", "entries": "${manifestResource.output.result.resourceContext.entries}" }, "expressions": [ ". as $in | .assets | ", "map(.assets | to_entries | ", " { topLevelAssetId: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\") | .value)[0].assetId, ", " sgAmpRefId: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\") | .value)[0] | \"\\(.assetId.id):\\(.assetId.version)\", ", " reviewAssetId: map(select(.value.payload.type == \"IMAGE\" or .value.payload.type == \"PMR_REVIEW_PROXY\") | .value)[0].assetId, ", " shotname: map(select(.value.payload.type == \"IMAGE\" or .value.payload.type == \"PMR_REVIEW_PROXY\") | .value)[0].payload.file.name, ", " metadata: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\") | .value.payload.metadata)[0] | { submission_note, version_name, scope_of_work, vendor, link, submitting_for, reason_for_review, sort_order}", " }", ")" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "updateShotgunProcessingInProgress", "inputParameters": { "inputData": { "resources": "${updateShotgunProcessingStart.output.result}" }, "expression": ".resources | map(. as $r | $r | .resourceContext.progress | map(if .name == \"SHOTGUN_UPLOAD\" then { name, status: \"IN_PROGRESS\" } else . end) as $p | $r * { resourceContext: ($r.resourceContext * {progress: $p}) })" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.updateResource", "taskReferenceName": "updateResourceShotgunInProgress", "inputParameters": { "resources": "${updateShotgunProcessingInProgress.output.result}", "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1576822186221, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.updateResource", "description": "Update a pipeline resource", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId", "resource"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareShotProcessingAssets", "inputParameters": { "inputData": { "assets": "${assets_tree_2.output.output}" }, "expressions": [ ".assets | to_entries | ", "map(.value.assets | to_entries | ", " { topLevelAssetId: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\"))[0].value.assetId | {id, versionId: .version}, ", " sgAmpRefId: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\"))[0].value.assetId | \"\\(.id):\\(.version)\", ", " reviewAssetId: map(select(.value.payload.type == \"PMR_REVIEW_PROXY\" or .value.payload.type == \"IMAGE\"))[0].value.assetId | {id, versionId: .version}, ", " shotname: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\"))[0].value.payload.metadata.version_name, ", " metadata: map(select(.value.payload.type == \"PMR_REVIEW_VERSION\"))[0].value.payload.metadata | {submission_note, version_name, scope_of_work, reason_for_review, vendor, link, submitting_for, sort_order}", " }", ")" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareShotProcessingTasks", "inputParameters": { "inputData": { "reviewServer": "${reviewServerConfig.output.result.server}", "reviewProjectId": "${reviewServerConfig.output.result.projectId}", "vendorId": "${workflow.input.sgVendorId}", "playlistId": "${playlistInfo.output.result.id}", "playlistName": "${playlistInfo.output.result.code}", "assetAndManifest": "${prepareShotProcessingAssets.output.result}" }, "expressions": [ ". as $in | $in.assetAndManifest | map({topLevelAssetId, reviewAssetId, sgAmpRefId, shotname, reviewServer: $in.reviewServer, reviewProjectId: $in.reviewProjectId, vendorId: $in.vendorId, playlistId: $in.playlistId, playlistName: $in.playlistName, vendorName: \"${vendorDetails.output.vendorName}\", metadata: .metadata }) | ", "{ taskDefs: map({type: \"SUB_WORKFLOW\", name: \"processshot\", taskReferenceName: \"processshot_\\(.topLevelAssetId.id)_\\(.topLevelAssetId.versionId)\", subWorkflowParam: { name: \"vfxmediareview.shotprocessing_v2\" }}), ", " taskInputs: map({ key: \"processshot_\\(.topLevelAssetId.id)_\\(.topLevelAssetId.versionId)\", value: . }) | from_entries}" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "asset_processing", "taskReferenceName": "shot_processing", "inputParameters": { "taskDefs": "${prepareShotProcessingTasks.output.result.taskDefs}", "taskInputs": "${prepareShotProcessingTasks.output.result.taskInputs}" }, "type": "FORK_JOIN_DYNAMIC", "decisionCases": {}, "dynamicForkTasksParam": "taskDefs", "dynamicForkTasksInputParamName": "taskInputs", "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "shot_processing_join", "taskReferenceName": "shot_processing_join", "inputParameters": {}, "type": "JOIN", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "processedFiles", "inputParameters": { "inputData": { "encodingResult": "${asset_processing_join.output}", "uploadResult": "${shot_processing_join.output}" }, "expressions": [ ". as $in | .uploadResult | to_entries | map(.value.result.upload_shot | { file: .filename, size: .file_size, labels: (if .skipped == true then [\"UPLOAD_SKIPPED\"] else [] end) }) as $ur | $in | .encodingResult | to_entries | map(.value | { file, message: .description, labels: (if .code == \"ENCODE_SUCCESS\" or .code == \"SUCCESS\" then (.file as $f | $ur | map(select($f == .file) | .labels)[0]) else [.code] + (.file as $f | $ur | map(select($f == .file) | .labels)[0]) end) })" ] }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.titus", "taskReferenceName": "updatePlaylist", "inputParameters": { "applicationName": "che/vfx-review-cli", "version": "${NETFLIX_ENVIRONMENT}.latest", "entryPoint": "python cli.py playlist -s \"https://${reviewServerConfig.output.result.server}\" -p ${reviewServerConfig.output.result.projectId} --action \"update\" --playlist-id ${playlistInfo.output.result.id} --playlist-status \"rev\" --contenthub-workspace-link \"${details.output.contenthubProjectUrl}/${pipelineDetails.output.submissionUri}\" --rc-conductor \"${CPEWF_TASK_ID}\"" }, "type": "TITUS", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1580327637347, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.titus", "retryCount": 3, "timeoutSeconds": 3600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 90, "responseTimeoutSeconds": 1200, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": true, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "updateShotgunProcessingCompleted", "inputParameters": { "inputData": { "resources": "${updateShotgunProcessingStart.output.result}" }, "expression": ".resources | map( . as $r | $r | .resourceContext.progress | map( if .name == \"SHOTGUN_UPLOAD\" then { name, status: \"COMPLETED\" } else . end ) as $p | $r * {\"status\":\"COMPLETED\", resourceContext: ($r.resourceContext * {progress: $p}) } )" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.updateResource", "taskReferenceName": "updateResourceShotgunCompleted", "inputParameters": { "resources": "${updateShotgunProcessingCompleted.output.result}", "pipelineId": "${pipelineData.output.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1576822186221, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.updateResource", "description": "Update a pipeline resource", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId", "resource"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "submissionProcessed", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "processed" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.common.jq", "taskReferenceName": "prepareRecipientDomainsForProcessingComplete", "inputParameters": { "inputData": { "reviewServerConfig": "${reviewServerConfig.output.result}", "domains": [ { "domains": ["netflix.amp.domain.studio_vfx"] }, { "domains": ["netflix.amp.domain.production_vfx"], "partnerId": "${pipelineData.output.request.request.data.vendorId}", "ignoreIfPartnerNull": true }, { "profiles": ["PRODUCTION_VFX_ADMIN"], "includeUsersWithProfile": true } ], "fallbackDomains": [ { "domains": ["netflix.amp.domain.studio_vfx"] } ] }, "expression": ". as $in | $in.domains" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082387383, "createdBy": "CPEWORKFLOW", "name": "stl.common.jq", "description": "Run JQ expression", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["expression", "inputData"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "sendProcessingCompleteEmail", "taskReferenceName": "sendProcessingCompleteEmail", "inputParameters": { "emailType": "INTERNAL_EXTERNAL", "domains": "${prepareRecipientDomainsForProcessingComplete.output.result}", "additionalRecipients": [ "vfx-media-review@netflix.com", "e1u9g2p6u1b8e5x7@netflix.slack.com" ], "additionalUsers": [ "${pipelineData.output.request.request.data.ownerUser}" ], "eventName": "EVENT_MESSAGE_VFX_REVIEW_SUBMISSION", "eventType": "MESSAGE_VFX_REVIEW_SUBMISSION", "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "emailPayload": { "movieId": "${pipelineData.output.request.request.data.movieId}", "submissionId": "${pipelineData.output.request.request.data.submissionId}", "submissionNodeId": "${pipelineData.output.request.request.data.submissionNodeId}", "shotgunBaseUrl": "https://${reviewServerConfig.output.result.server}", "shotgunProjectId": "${reviewServerConfig.output.result.projectId}", "pipelineId": "${pipelineData.output.pipelineId}", "subject": "Playlist ${pipelineData.output.request.request.data.submissionId} is ready for review!", "playlistId": "${playlistInfo.output.result.id}", "status": "SUCCESS", "playlistReadyForReview": true, "filesInSubmission": "${processedFiles.output.result}", "manifestName": "${manifestDetails.output.result.manifestName}", "manifestLink": "${pipelineDetails.output.submissionUri}&nodeIds=${manifestDetails.output.result.manifestNodeId}", "manifestIncluded": "${manifestDetails.output.result.manifestIncluded}", "vendorName": "${vendorDetails.output.name}", "chProjectId": "${pipelineData.output.request.request.data.projectId}", "oldCsv": "${oldCsv.output.result}", "manifestWarnings": "${manifestDetails.output.result.displayWarnings}" } }, "type": "SUB_WORKFLOW", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "subWorkflowParam": { "name": "vfxmediareview.notification", "version": 1 }, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pegasus.processPipelineEvent", "taskReferenceName": "processPipelineEvent", "inputParameters": { "assetType": "PMR_REVIEW_VERSION", "downloadDescription": "Download of submission ${pipelineData.output.request.request.data.submissionId}", "movieId": "${pipelineData.output.request.request.data.movieId}", "pipelineType": "${pipelineData.output.request.request.type}", "nodeIds": ["${submissionFolderProjection.output.downloadFolderId}"], "relativePath": "vfxmediareview/${pipelineData.output.request.request.data.submissionId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1574880092269, "createdBy": "CPEWORKFLOW", "name": "stl.pegasus.processPipelineEvent", "description": "Tell Pegasus Stargate that we have a node that is ready for download", "retryCount": 3, "timeoutSeconds": 300, "inputKeys": [ "assetType", "downloadDescription", "movieId", "pipelineType", "nodeId", "relativePath" ], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "cperequest_transition", "taskReferenceName": "completeRequest", "inputParameters": { "namespace": "${pipelineData.output.request.request.namespace}", "type": "${pipelineData.output.request.request.type}", "requestId": "${pipelineData.output.request.request.id}", "transitionName": "complete" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "updateTime": 1604373979513, "updatedBy": "cperequest", "name": "cperequest_transition", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": [ "namespace", "type", "requestId", "transitionName", "currentState", "currentVersion", "assignee", "clearAssignee", "dueDate", "clearDueDate", "skipIfInState", "transitionDetails" ], "outputKeys": ["request"], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 60, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "mce-workflow-infra@netflix.com", "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "stl.pipeline.complete", "taskReferenceName": "completePipeline_SubmissionAccepted", "inputParameters": { "pipelineId": "${workflow.input.pipelineInput.pipelineId}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "createTime": 1556082385940, "createdBy": "CPEWORKFLOW", "name": "stl.pipeline.complete", "description": "Final task for all pipeline workflows", "retryCount": 3, "timeoutSeconds": 0, "inputKeys": ["pipelineId"], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 60, "responseTimeoutSeconds": 300, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "terminate", "taskReferenceName": "SubmissionAccepted", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": { "outcome": "SUBMISSION_ACCEPTED", "code": "SUBMISSION_ACCEPTED" } }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "failureWorkflow": "pipelines.vfxmediareview.failure", "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "cpe-che-backend@netflix.com", "timeoutPolicy": "TIME_OUT_WF", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, "priority": 0, "variables": {}, "lastRetriedTime": 0, "startTime": 1608153919527, "workflowName": "pipelines.vfxmediareview", "workflowVersion": 1 } ================================================ FILE: ui/cypress/fixtures/metadataTasks.json ================================================ [ { "updateTime": 1629995112563, "updatedBy": "user1@example.com", "name": "example_task_1", "retryCount": 4, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 120, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "user1@example.com", "backoffScaleFactor": 1 }, { "createTime": 1562373417179, "createdBy": "user2@example.com", "name": "example_task_2", "retryCount": 2, "timeoutSeconds": 0, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "RETRY", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 30, "responseTimeoutSeconds": 120, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, { "createTime": 1627367321969, "createdBy": "user3@example.com", "name": "example_task_3", "retryCount": 3, "timeoutSeconds": 1800, "inputKeys": ["projectId"], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "EXPONENTIAL_BACKOFF", "retryDelaySeconds": 3, "responseTimeoutSeconds": 1800, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "ownerEmail": "user3@example.com", "backoffScaleFactor": 1 } ] ================================================ FILE: ui/cypress/fixtures/metadataWorkflow.json ================================================ [ { "createTime": 1638226947603, "name": "19test009", "description": "test workflow", "version": 1, "tasks": [ { "name": "fetch_data", "taskReferenceName": "fetch_data", "inputParameters": { "http_request": { "connectionTimeOut": "3600", "readTimeOut": "3600", "uri": "${workflow.input.uri}", "method": "GET", "accept": "application/json", "content-Type": "application/json", "headers": {} } }, "type": "HTTP", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "taskDefinition": { "name": "fetch_data", "retryCount": 0, "timeoutSeconds": 3600, "inputKeys": [], "outputKeys": [], "timeoutPolicy": "TIME_OUT_WF", "retryLogic": "FIXED", "retryDelaySeconds": 0, "responseTimeoutSeconds": 3000, "inputTemplate": {}, "rateLimitPerFrequency": 0, "rateLimitFrequencyInSeconds": 1, "backoffScaleFactor": 1 }, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": true, "ownerEmail": "test@163.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, { "createTime": 1610653237179, "name": "ConditionalTerminateWorkflow", "description": "ConditionalTerminateWorkflow", "version": 1, "tasks": [ { "name": "perf_task_1", "taskReferenceName": "t1", "inputParameters": { "tp11": "${workflow.input.param1}", "tp12": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "decision", "taskReferenceName": "decision", "inputParameters": { "case": "${t1.output.case}" }, "type": "DECISION", "caseValueParam": "case", "decisionCases": { "one": [ { "name": "perf_task_2", "taskReferenceName": "t2", "inputParameters": { "tp21": "${workflow.input.param1}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "two": [ { "name": "terminate", "taskReferenceName": "terminate0", "inputParameters": { "terminationStatus": "COMPLETED", "workflowOutput": "${t1.output.op}" }, "type": "TERMINATE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] }, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] }, { "name": "perf_task_3", "taskReferenceName": "t3", "inputParameters": { "tp31": "${workflow.input.param2}" }, "type": "SIMPLE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ], "inputParameters": ["param1", "param2"], "outputParameters": { "o2": "${t1.output.op}" }, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "test@harness.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} }, { "createTime": 1654202968736, "name": "Do_While_Workflow_Iteration_Fix", "description": "Do_While_Workflow_Iteration_Fix", "version": 1, "tasks": [ { "name": "loopTask", "taskReferenceName": "loopTask", "inputParameters": { "value": "${workflow.input.loop}" }, "type": "DO_WHILE", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", "loopOver": [ { "name": "form_uri", "taskReferenceName": "form_uri", "inputParameters": { "index": "${loopTask['iteration']}", "scriptExpression": "return $.index - 1;" }, "type": "LAMBDA", "decisionCases": {}, "defaultCase": [], "forkTasks": [], "startDelay": 0, "joinOn": [], "optional": false, "defaultExclusiveJoinTask": [], "asyncComplete": false, "loopOver": [] } ] } ], "inputParameters": [], "outputParameters": {}, "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, "ownerEmail": "peterl@netflix.com", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} } ] ================================================ FILE: ui/cypress/fixtures/taskSearch.json ================================================ { "totalHits": 1, "results": [ { "workflowId": "e577cf0c-4cc0-4224-b729-79c5a2609b30", "workflowType": "JXU_PROMO_MEDIA_PUBLISH_TO_PAL_WORKFLOW", "scheduledTime": "2022-05-17T22:52:46.628Z", "startTime": "2022-05-17T22:52:47.212Z", "updateTime": "2022-05-17T22:52:47.212Z", "endTime": "2022-05-17T22:52:47.602Z", "status": "COMPLETED", "executionTime": 390, "queueWaitTime": 584, "taskDefName": "JXU_PROMO_MEDIA_PUBLISH_BUNDLE_TO_PAL", "taskType": "JXU_PROMO_MEDIA_PUBLISH_BUNDLE_TO_PAL", "input": "{bundleId=workflow.input.bundleId}", "output": "{singleAssetPublishTasks=[{taskReferenceName=fork_0, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_1, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_2, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_3, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_4, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_5, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_6, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_7, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_8, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_9, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_10, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_11, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_12, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_13, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_14, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_15, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_16, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_17, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_18, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_19, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_20, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_21, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_22, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_23, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_24, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_25, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_26, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_27, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_28, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_29, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_30, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_31, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_32, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_33, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_34, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_35, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_36, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_37, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_38, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_39, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_40, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_41, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_42, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_43, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_44, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_45, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_46, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_47, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_48, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_49, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_50, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_51, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_52, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_53, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_54, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_55, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_56, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_57, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_58, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_59, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_60, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_61, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_62, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_63, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_64, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_65, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_66, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_67, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_68, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_69, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_70, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_71, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_72, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_73, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_74, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_75, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_76, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_77, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_78, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_79, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_80, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_81, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_82, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_83, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_84, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_85, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_86, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_87, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_88, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_89, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_90, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_91, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_92, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_93, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_94, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_95, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_96, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_97, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_98, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}, {taskReferenceName=fork_99, name=JXU_PROMO_MEDIA_PUBLISH_SINGLE_ASSET_TO_PAL, type=SIMPLE}], singleAssetPublishTaskInput={fork_89={assetId=89}, fork_87={assetId=87}, fork_88={assetId=88}, fork_85={assetId=85}, fork_86={assetId=86}, fork_83={assetId=83}, fork_84={assetId=84}, fork_81={assetId=81}, fork_82={assetId=82}, fork_80={assetId=80}, fork_12={assetId=12}, fork_13={assetId=13}, fork_10={assetId=10}, fork_98={assetId=98}, fork_11={assetId=11}, fork_99={assetId=99}, fork_96={assetId=96}, fork_97={assetId=97}, fork_94={assetId=94}, fork_95={assetId=95}, fork_92={assetId=92}, fork_93={assetId=93}, fork_90={assetId=90}, fork_91={assetId=91}, fork_23={assetId=23}, fork_24={assetId=24}, fork_21={assetId=21}, fork_22={assetId=22}, fork_20={assetId=20}, fork_18={assetId=18}, fork_19={assetId=19}, fork_16={assetId=16}, fork_17={assetId=17}, fork_14={assetId=14}, fork_15={assetId=15}, fork_34={assetId=34}, fork_35={assetId=35}, fork_32={assetId=32}, fork_33={assetId=33}, fork_30={assetId=30}, fork_31={assetId=31}, fork_29={assetId=29}, fork_27={assetId=27}, fork_28={assetId=28}, fork_25={assetId=25}, fork_26={assetId=26}, fork_45={assetId=45}, fork_46={assetId=46}, fork_43={assetId=43}, fork_44={assetId=44}, fork_41={assetId=41}, fork_42={assetId=42}, fork_40={assetId=40}, fork_38={assetId=38}, fork_39={assetId=39}, fork_36={assetId=36}, fork_37={assetId=37}, fork_56={assetId=56}, fork_57={assetId=57}, fork_54={assetId=54}, fork_9={assetId=9}, fork_55={assetId=55}, fork_8={assetId=8}, fork_52={assetId=52}, fork_7={assetId=7}, fork_53={assetId=53}, fork_6={assetId=6}, fork_50={assetId=50}, fork_5={assetId=5}, fork_51={assetId=51}, fork_4={assetId=4}, fork_3={assetId=3}, fork_2={assetId=2}, fork_1={assetId=1}, fork_0={assetId=0}, fork_49={assetId=49}, fork_47={assetId=47}, fork_48={assetId=48}, fork_67={assetId=67}, fork_68={assetId=68}, fork_65={assetId=65}, fork_66={assetId=66}, fork_63={assetId=63}, fork_64={assetId=64}, fork_61={assetId=61}, fork_62={assetId=62}, fork_60={assetId=60}, fork_58={assetId=58}, fork_59={assetId=59}, fork_78={assetId=78}, fork_79={assetId=79}, fork_76={assetId=76}, fork_77={assetId=77}, fork_74={assetId=74}, fork_75={assetId=75}, fork_72={assetId=72}, fork_73={assetId=73}, fork_70={assetId=70}, fork_71={assetId=71}, fork_69={assetId=69}}}", "taskId": "36d24c5c-9c26-46cf-9709-e1bc6963b8a5", "workflowPriority": 0 } ] } ================================================ FILE: ui/cypress/fixtures/workflowSearch.json ================================================ { "totalHits": 5, "results": [ { "workflowType": "feature_value_compute_workflow", "version": 1, "workflowId": "d11255ed-4708-4ce5-992d-92803f0f19fc", "startTime": "2022-06-09T16:32:56.851Z", "status": "RUNNING", "input": "{clientContext={}, featureDefId={namespace={name=gemstone-dev}, featureDefName=gcarmo-orchestration-test-3, featureDefVersion=2}, computeInfo={metaflowCompute={endpoint=https://httpbin.org/pos}}, triggerDagobahAttemptId=2c3c3444-dbb5-3bcc-aa7d-a3405c686c5c, gemIds=[8b132cd5-bde9-30ad-88b3-46f4ad720c73], featureDefTriggerId={namespace={name=some_trigger_id}, featureDefName=some_feature_def_name}}", "output": "{}", "executionTime": 0, "failedReferenceTaskNames": "", "priority": 0, "inputSize": 398, "outputSize": 2 }, { "workflowType": "feature_value_compute_workflow", "version": 1, "workflowId": "7ff5c1d5-da27-4b27-9e60-0404eb4a1d23", "startTime": "2022-06-09T16:31:54.904Z", "endTime": "2022-06-09T16:32:31.901Z", "status": "TERMINATED", "input": "{clientContext={}, featureDefId={namespace={name=gemstone-dev}, featureDefName=gcarmo-orchestration-test-3, featureDefVersion=2}, computeInfo={metaflowCompute={endpoint=https://httpbin.org/pos}}, triggerDagobahAttemptId=2c3c3444-dbb5-3bcc-aa7d-a3405c686c5c, gemIds=[8b132cd5-bde9-30ad-88b3-46f4ad720c73], featureDefTriggerId={namespace={name=some_trigger_id}, featureDefName=some_feature_def_name}}", "output": "{}", "reasonForIncompletion": "Some reason!!!", "executionTime": 36997, "failedReferenceTaskNames": "feature_value_compute_task", "priority": 0, "inputSize": 398, "outputSize": 2 }, { "workflowType": "feature_value_compute_workflow", "version": 1, "workflowId": "ede49264-407d-4879-a708-e01526cee2ba", "startTime": "2022-06-09T16:29:07.349Z", "endTime": "2022-06-09T16:30:22.945Z", "status": "FAILED", "input": "{clientContext={}, featureDefId={namespace={name=gemstone-dev}, featureDefName=gcarmo-orchestration-test-3, featureDefVersion=2}, computeInfo={metaflowCompute={endpoint=https://httpbin.org/pos}}, triggerDagobahAttemptId=2c3c3444-dbb5-3bcc-aa7d-a3405c686c5c, gemIds=[8b132cd5-bde9-30ad-88b3-46f4ad720c73], featureDefTriggerId={namespace={name=some_trigger_id}, featureDefName=some_feature_def_name}}", "output": "{}", "reasonForIncompletion": "Request to https://httpbin.org/pos failed with status code 404\n\n404 Not Found\n

    Not Found

    \n

    The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

    \n", "executionTime": 75596, "failedReferenceTaskNames": "feature_value_compute_task", "priority": 0, "inputSize": 398, "outputSize": 2 }, { "workflowType": "feature_value_compute_workflow", "version": 1, "workflowId": "3950353b-9225-4729-a9e4-c8b4e244e041", "startTime": "2022-06-09T16:27:48.666Z", "endTime": "2022-06-09T16:27:50.560Z", "status": "COMPLETED", "input": "{clientContext={}, featureDefId={namespace={name=gemstone-dev}, featureDefName=gcarmo-orchestration-test-3, featureDefVersion=1}, computeInfo={metaflowCompute={endpoint=https://httpbin.org/post}}, triggerDagobahAttemptId=2c3c3444-dbb5-3bcc-aa7d-a3405c686c5c, gemIds=[8b132cd5-bde9-30ad-88b3-46f4ad720c73], featureDefTriggerId={namespace={name=some_trigger_id}, featureDefName=some_feature_def_name}}", "output": "{}", "executionTime": 1894, "failedReferenceTaskNames": "", "priority": 0, "inputSize": 399, "outputSize": 2 }, { "workflowType": "feature_value_compute_workflow", "version": 1, "workflowId": "9a6438c5-60a4-4af6-b530-f2bf3a2dd859", "startTime": "2022-06-09T16:20:28.188Z", "endTime": "2022-06-09T16:20:29.935Z", "status": "COMPLETED", "input": "{clientContext={}, featureDefId={namespace={name=gemstone-dev}, featureDefName=gcarmo-orchestration-test-3, featureDefVersion=1}, computeInfo={metaflowCompute={endpoint=https://httpbin.org/post}}, triggerDagobahAttemptId=2c3c3444-dbb5-3bcc-aa7d-a3405c686c5c, gemIds=[8b132cd5-bde9-30ad-88b3-46f4ad720c73], featureDefTriggerId={namespace={name=some_trigger_id}, featureDefName=some_feature_def_name}}", "output": "{}", "executionTime": 1747, "failedReferenceTaskNames": "", "priority": 0, "inputSize": 399, "outputSize": 2 } ] } ================================================ FILE: ui/cypress/support/commands.ts ================================================ /// // *********************************************** // This example commands.ts shows you how to // create various custom commands and overwrite // existing commands. // // For more comprehensive examples of custom // commands please read more here: // https://on.cypress.io/custom-commands // *********************************************** // // // -- This is a parent command -- // Cypress.Commands.add('login', (email, password) => { ... }) // // // -- This is a child command -- // Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... }) // // // -- This is a dual command -- // Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... }) // // // -- This will overwrite an existing command -- // Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... }) // // declare global { // namespace Cypress { // interface Chainable { // login(email: string, password: string): Chainable // drag(subject: string, options?: Partial): Chainable // dismiss(subject: string, options?: Partial): Chainable // visit(originalFn: CommandOriginalFn, url: string, options: Partial): Chainable // } // } // } ================================================ FILE: ui/cypress/support/component-index.html ================================================ Components App
    ================================================ FILE: ui/cypress/support/component.ts ================================================ // *********************************************************** // This example support/component.ts is processed and // loaded automatically before your test files. // // This is a great place to put global configuration and // behavior that modifies Cypress. // // You can change the location of this file or turn off // automatically serving support files with the // 'supportFile' configuration option. // // You can read more here: // https://on.cypress.io/configuration // *********************************************************** // Import commands.js using ES2015 syntax: import "./commands"; // Alternatively you can use CommonJS syntax: // require('./commands') import { mount } from "cypress/react"; // Augment the Cypress namespace to include type definitions for // your custom command. // Alternatively, can be defined in cypress/support/component.d.ts // with a at the top of your spec. declare global { namespace Cypress { interface Chainable { mount: typeof mount; } } } Cypress.Commands.add("mount", mount); // Example use: // cy.mount() ================================================ FILE: ui/cypress/support/e2e.ts ================================================ // *********************************************************** // This example support/e2e.ts is processed and // loaded automatically before your test files. // // This is a great place to put global configuration and // behavior that modifies Cypress. // // You can change the location of this file or turn off // automatically serving support files with the // 'supportFile' configuration option. // // You can read more here: // https://on.cypress.io/configuration // *********************************************************** // Import commands.js using ES2015 syntax: import "./commands"; // Alternatively you can use CommonJS syntax: // require('./commands') ================================================ FILE: ui/cypress.config.ts ================================================ import { defineConfig } from "cypress"; export default defineConfig({ e2e: { baseUrl: "http://localhost:5000", }, component: { devServer: { framework: "create-react-app", bundler: "webpack", }, }, }); ================================================ FILE: ui/package.json ================================================ { "name": "client", "version": "3.8.0", "dependencies": { "@material-ui/core": "^4.12.3", "@material-ui/icons": "^4.11.2", "@material-ui/lab": "^4.0.0-alpha.60", "@material-ui/styles": "^4.11.4", "@monaco-editor/react": "^4.4.0", "clsx": "^1.1.1", "cronstrue": "^1.72.0", "d3": "^6.2.0", "dagre-d3": "^0.6.4", "date-fns": "^2.16.1", "formik": "^2.2.9", "http-proxy-middleware": "^2.0.1", "immutability-helper": "^3.1.1", "json-bigint-string": "^1.0.0", "lodash": "^4.17.20", "moment": "^2.29.2", "monaco-editor": "^0.44.0", "node-forge": "^1.3.0", "parse-svg-path": "^0.1.2", "prop-types": "^15.7.2", "react": "^16.8.0", "react-cron-generator": "^1.3.5", "react-data-table-component": "^6.11.8", "react-dom": "^16.8.0", "react-helmet": "^6.1.0", "react-is": "^17.0.2", "react-query": "^3.19.4", "react-resize-detector": "^5.2.0", "react-router": "^5.2.0", "react-router-dom": "^5.2.0", "react-router-use-location-state": "^2.5.0", "react-scripts": "^5.0.1", "react-vis-timeline-2": "^2.1.6", "rison": "^0.1.1", "styled-components": "^5.3.0", "url-parse": "^1.5.1", "use-local-storage-state": "^10.0.0", "xss": "^1.0.8", "yup": "^0.32.11" }, "scripts": { "start": "react-scripts start", "build": "react-scripts build", "test": "react-scripts test", "eject": "react-scripts eject", "prettier": "prettier --write .", "serve-build": "http-server ./build --port 5000 --proxy http://localhost:8080", "cypress:open": "cypress open", "cypress:test": "BROWSER=none start-server-and-test serve-build http-get://localhost:5000 'cypress run'" }, "browserslist": { "production": [ ">0.2%", "not dead", "not op_mini all" ], "development": [ "last 1 chrome version", "last 1 firefox version", "last 1 safari version" ] }, "resolutions": { "validator": "^13.7.0", "nth-check": "^2.0.1", "async": "^3.2.2", "ejs": "^3.1.7" }, "devDependencies": { "@babel/core": "^7.18.2", "@babel/preset-env": "^7.18.2", "@babel/register": "^7.17.7", "@cypress/react": "^5.12.5", "@cypress/webpack-dev-server": "^1.8.4", "cypress": "^10.0.3", "eslint-plugin-cypress": "^2.12.1", "http-server": "^14.1.1", "js-yaml": "4.1.0", "prettier": "^2.2.1", "sass": "^1.49.9", "start-server-and-test": "^1.14.0", "typescript": "^4.6.3" }, "engines": { "node": ">=14.17.0" }, "license": "Apache-2.0" } ================================================ FILE: ui/public/index.html ================================================ Conductor UI
    ================================================ FILE: ui/public/robots.txt ================================================ # https://www.robotstxt.org/robotstxt.html User-agent: * Disallow: ================================================ FILE: ui/src/App.jsx ================================================ import React from "react"; import { Route, Switch } from "react-router-dom"; import { makeStyles } from "@material-ui/styles"; import { loader } from '@monaco-editor/react'; import { Button, AppBar, Toolbar } from "@material-ui/core"; import AppLogo from "./plugins/AppLogo"; import NavLink from "./components/NavLink"; import WorkflowSearch from "./pages/executions/WorkflowSearch"; import TaskSearch from "./pages/executions/TaskSearch"; import Execution from "./pages/execution/Execution"; import WorkflowDefinitions from "./pages/definitions/Workflow"; import WorkflowDefinition from "./pages/definition/WorkflowDefinition"; import TaskDefinitions from "./pages/definitions/Task"; import TaskDefinition from "./pages/definition/TaskDefinition"; import EventHandlerDefinitions from "./pages/definitions/EventHandler"; import EventHandlerDefinition from "./pages/definition/EventHandler"; import TaskQueue from "./pages/misc/TaskQueue"; import KitchenSink from "./pages/kitchensink/KitchenSink"; import DiagramTest from "./pages/kitchensink/DiagramTest"; import Examples from "./pages/kitchensink/Examples"; import Gantt from "./pages/kitchensink/Gantt"; import CustomRoutes from "./plugins/CustomRoutes"; import AppBarModules from "./plugins/AppBarModules"; import CustomAppBarButtons from "./plugins/CustomAppBarButtons"; import Workbench from "./pages/workbench/Workbench"; const useStyles = makeStyles((theme) => ({ root: { backgroundColor: "#efefef", // TODO: Use theme var display: "flex", }, body: { width: "100vw", height: "100vh", paddingTop: theme.overrides.MuiAppBar.root.height, }, toolbarRight: { marginLeft: "auto", display: "flex", flexDirection: "row", }, toolbarRegular: { minHeight: 80, }, })); export default function App() { const classes = useStyles(); return ( // Provide context for backward compatibility with class components
    ); } if (process.env.REACT_APP_MONACO_EDITOR_USING_CDN === "false") { // Change the source of the monaco files, see https://github.com/suren-atoyan/monaco-react/issues/168#issuecomment-762336713 loader.config({ paths: { vs: '/monaco-editor/min/vs' } }); } ================================================ FILE: ui/src/components/Banner.jsx ================================================ import React from "react"; import { Paper } from "@material-ui/core"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ root: { padding: 15, backgroundColor: "rgba(73, 105, 228, 0.1)", color: "rgba(0, 0, 0, 0.9)", borderLeft: "solid rgba(73, 105, 228, 0.1) 4px", }, }); export default function Banner({ children, ...rest }) { const classes = useStyles(); return ( {children} ); } ================================================ FILE: ui/src/components/Button.jsx ================================================ import { Button as MuiButton } from "@material-ui/core"; export default function Button({ variant = "primary", ...props }) { if (variant === "secondary") { return ; } else { // primary or invalid return ; } } ================================================ FILE: ui/src/components/ButtonGroup.jsx ================================================ import React from "react"; import { FormControl, InputLabel, ButtonGroup, Button, } from "@material-ui/core"; export default function ({ options, label, style, classes, ...props }) { return ( {label && {label}} {options.map((option, idx) => ( ))} ); } ================================================ FILE: ui/src/components/ConfirmChoiceDialog.jsx ================================================ import React from "react"; import { Dialog, DialogActions, DialogContent, DialogTitle, } from "@material-ui/core"; import Text from "./Text"; import Button from "./Button"; export default function ({ header = "Confirmation", message = "Please confirm", handleConfirmationValue, open, }) { return ( handleConfirmationValue(false)} > {header} {message} ); } ================================================ FILE: ui/src/components/CustomButtons.jsx ================================================ import Button from "@material-ui/core/Button"; import { styled } from "@material-ui/core"; export const fontFamilyList = [ "-apple-system", "BlinkMacSystemFont", '"Segoe UI"', "Roboto", '"Helvetica Neue"', "Arial", "sans-serif", '"Apple Color Emoji"', '"Segoe UI Emoji"', '"Segoe UI Symbol"', ].join(","); const hoverCss = { backgroundColor: "#857aff", borderColor: "#857aff", boxShadow: "none", "&> .MuiButton-label": { color: "white", }, }; const buttonBaseStyle = { boxShadow: "none", textTransform: "none", fontSize: 16, padding: "6px 12px", border: "1px solid", lineHeight: 1.3, color: "#ffffff", backgroundColor: "#6558F5", borderColor: "#6558F5", fontFamily: fontFamilyList, "&:hover": hoverCss, "&:active": hoverCss, "&:focus": { boxShadow: "0 0 0 0.2rem rgba(0,123,255,.5)", }, "&> .MuiButton-label": { color: "#ffffff", }, }; export const BootstrapButton = styled(Button)(buttonBaseStyle); const outlineHoverCss = { ...hoverCss, "&> .MuiButton-label": { color: "ghostwhite", }, }; const actionHoverCss = { ...hoverCss, backgroundColor: "#30499f", borderColor: "#30499f", }; export const BootstrapOutlineButton = styled(Button)({ ...buttonBaseStyle, color: "#ffffff", backgroundColor: "ghostwhite", borderColor: "#6558F5", "&> .MuiButton-label": { color: "#6558F5", }, "&:hover": outlineHoverCss, "&:active": outlineHoverCss, }); export const BootstrapOutlineActionButton = styled(Button)({ ...buttonBaseStyle, color: "#ffffff", backgroundColor: "ghostwhite", borderColor: "#30499f", "&> .MuiButton-label": { color: "#30499f", }, "&:hover": actionHoverCss, "&:active": actionHoverCss, }); export const BootstrapTextButton = styled(Button)({ ...buttonBaseStyle, color: "#ffffff", backgroundColor: "ghostwhite", borderColor: "transparent", "&> .MuiButton-label": { color: "#6558F5", }, "&:hover": outlineHoverCss, "&:active": outlineHoverCss, }); export const BootstrapActionButton = styled(Button)({ ...buttonBaseStyle, fontSize: 14, lineHeight: 1.5, backgroundColor: "#4969e4", borderColor: "#4969e4", "&> .MuiButton-label": { color: "#ffffff", }, "&:hover": actionHoverCss, "&:active": actionHoverCss, }); ================================================ FILE: ui/src/components/DataTable.jsx ================================================ import React, { useMemo, useState } from "react"; import RawDataTable from "react-data-table-component"; import { Checkbox, MenuItem, ListItemText, IconButton, Menu, Tooltip, Popover, } from "@material-ui/core"; import ViewColumnIcon from "@material-ui/icons/ViewColumn"; import SearchIcon from "@material-ui/icons/Search"; import { Heading, Select, Input } from "./"; import { timestampRenderer, timestampMsRenderer } from "../utils/helpers"; import { useLocalStorage } from "../utils/localstorage"; import _ from "lodash"; export const DEFAULT_ROWS_PER_PAGE = 15; export default function DataTable(props) { const { localStorageKey, columns, data, options, defaultShowColumns, paginationPerPage = 15, showFilter = true, showColumnSelector = true, paginationServer = false, title, onFilterChange, initialFilterObj, ...rest } = props; const DEFAULT_FILTER_OBJ = { columnName: columns.find((col) => col.searchable !== false).name, substring: "", }; // If no defaultColumns passed - use all columns const defaultColumns = useMemo( () => props.defaultShowColumns || props.columns.map((col) => getColumnId(col)), [props.defaultShowColumns, props.columns] ); const [tableState, setTableState] = useLocalStorage( localStorageKey, defaultColumns ); const [filterObj, setFilterObj] = useState( initialFilterObj || DEFAULT_FILTER_OBJ ); const handleFilterChange = (val) => { setFilterObj(val); if (onFilterChange) { if (!_.isEmpty(val.substring)) { onFilterChange(val); } else { onFilterChange(undefined); } } }; // Append bodyRenderer for date fields; const dataTableColumns = useMemo(() => { let viewColumns = []; if (tableState) { for (let col of columns) { if (tableState.includes(getColumnId(col))) { viewColumns.push(col); } } } else { viewColumns = columns; } return viewColumns.map((column) => { let { id, name, label, type, renderer, wrap = true, sortable = true, ...rest } = column; const internalOptions = {}; if (type === "date") { internalOptions.format = (row) => timestampRenderer(_.get(row, name)); } else if (type === "date-ms") { internalOptions.format = (row) => timestampMsRenderer(_.get(row, name)); } else if (type === "json") { internalOptions.format = (row) => JSON.stringify(_.get(row, name)); } if (renderer) { internalOptions.format = (row) => renderer(_.get(row, name), row); } return { id: getColumnId(column), selector: name, name: getColumnLabel(column), sortable: sortable, wrap: wrap, type, ...internalOptions, ...rest, }; }); }, [tableState, columns]); const filteredItems = useMemo(() => { const column = dataTableColumns.find( (col) => col.id === filterObj.columnName ); if (!filterObj.substring || !filterObj.columnName) { return data; } else { try { const regexp = new RegExp(filterObj.substring, "i"); return data.filter((row) => { let target; if ( column.type === "json" || column.type === "date" || column.type === "date-ms" || column.searchable === "calculated" ) { target = column.format(row); if (!_.isString(target)) { target = JSON.stringify(target); } } else { target = _.get(row, column.selector); } return _.isString(target) && regexp.test(target); }); } catch (e) { // Bad or incomplete Regexp console.log(e); return []; } } }, [data, dataTableColumns, filterObj]); return ( {title}} columns={dataTableColumns} data={filteredItems} pagination paginationServer={paginationServer} paginationPerPage={paginationPerPage} paginationRowsPerPageOptions={[15, 30, 100, 1000]} actions={ <> {!paginationServer && showFilter && ( )} {showColumnSelector && ( )} } {...rest} /> ); } function Filter({ columns, filterObj, setFilterObj }) { const [anchorEl, setAnchorEl] = React.useState(null); const handleClick = (event) => { setAnchorEl(event.currentTarget); }; const handleClose = () => { setAnchorEl(null); }; const handleValueChange = (v) => { setFilterObj({ columnName: filterObj.columnName, substring: v, }); }; const handleColumnChange = (c) => { setFilterObj({ columnName: c, substring: "", }); }; return ( <> ); } function getColumnLabelById(columnId, columns) { const col = columns.find((c) => c.id === columnId || c.name === columnId); return col.label || col.name; } function getColumnLabel(col) { return col.label || col.name; } function getColumnId(col) { return col.id || col.name; } function ColumnsSelector({ columns, selected, setSelected, defaultColumns }) { const [anchorEl, setAnchorEl] = React.useState(null); const handleClick = (event) => { setAnchorEl(event.currentTarget); }; const handleClose = () => { setAnchorEl(null); }; const handleChange = (columnId, checked) => { if (!checked && selected.includes(columnId)) { setSelected(selected.filter((v) => v !== columnId)); } else { setSelected([...selected, columnId]); } }; const reset = () => { setSelected(defaultColumns); }; return ( <> {[ ...columns.map((column) => ( handleChange(getColumnId(column), e.target.checked) } /> )), Reset to default , ]} ); } ================================================ FILE: ui/src/components/DateRangePicker.jsx ================================================ import React from "react"; import { Input } from "./"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ wrapper: { display: "flex", }, input: { marginRight: 5, flex: "0 1 50%", }, quick: { flex: "0 0 auto", }, }); export default function DateRangePicker({ onFromChange, from, onToChange, to, label, disabled, }) { const classes = useStyles(); return (
    ); } ================================================ FILE: ui/src/components/Dropdown.jsx ================================================ import React from "react"; import { Input } from "./"; import Autocomplete from "@material-ui/lab/Autocomplete"; import FormControl from "@material-ui/core/FormControl"; import InputLabel from "@material-ui/core/InputLabel"; import CloseIcon from "@material-ui/icons/Close"; import { InputAdornment, CircularProgress } from "@material-ui/core"; export default function ({ label, className, style, error, helperText, name, value, placeholder, loading, disabled, ...props }) { return ( {label && {label}} } renderInput={({ InputProps, ...params }) => ( ), }), }} placeholder={loading ? "Loading Options" : placeholder} name={name} error={!!error} helperText={helperText} /> )} value={value === undefined ? null : value} // convert undefined to null /> ); } ================================================ FILE: ui/src/components/DropdownButton.jsx ================================================ import React from "react"; import Button from "@material-ui/core/Button"; import ArrowDropDownIcon from "@material-ui/icons/ArrowDropDown"; import { ClickAwayListener, Popper, MenuItem, MenuList, } from "@material-ui/core"; import { Paper } from "./"; export default function DropdownButton({ children, options }) { const [open, setOpen] = React.useState(false); const anchorRef = React.useRef(null); const handleToggle = () => { setOpen((prevOpen) => !prevOpen); }; const handleClose = (event) => { if (anchorRef.current && anchorRef.current.contains(event.target)) { return; } setOpen(false); }; return ( {options.map(({ label, handler }, index) => ( { handler(event, index); setOpen(false); }} > {label} ))} ); } ================================================ FILE: ui/src/components/Heading.jsx ================================================ import React from "react"; import Typography from "@material-ui/core/Typography"; const levelMap = ["h6", "h5", "h4", "h3", "h2", "h1"]; export default function ({ level = 3, ...props }) { return ; } ================================================ FILE: ui/src/components/Input.jsx ================================================ import React, { useRef } from "react"; import { TextField, InputAdornment, IconButton } from "@material-ui/core"; import ClearIcon from "@material-ui/icons/Clear"; export default function (props) { const { label, clearable, onBlur, onChange, InputProps, ...rest } = props; const inputRef = useRef(); function handleClear() { inputRef.current.value = ""; if (onBlur) return onBlur(""); if (onChange) return onChange(""); } function handleBlur(e) { if (onBlur) onBlur(e.target.value); } function handleChange(e) { if (onChange) onChange(e.target.value); } return ( ), } } onBlur={handleBlur} onChange={handleChange} {...rest} /> ); } ================================================ FILE: ui/src/components/KeyValueTable.jsx ================================================ import React from "react"; import { makeStyles } from "@material-ui/styles"; import { List, ListItem, ListItemText, Tooltip } from "@material-ui/core"; import _ from "lodash"; import { useEnv } from "../plugins/env"; import { timestampRenderer, timestampMsRenderer, durationRenderer, } from "../utils/helpers"; import { customTypeRenderers } from "../plugins/customTypeRenderers"; const useStyles = makeStyles((theme) => ({ value: { flex: 0.7, }, label: { flex: 0.3, minWidth: "100px", }, labelText: { fontWeight: "bold !important", }, })); export default function KeyValueTable({ data }) { const classes = useStyles(); const env = useEnv(); return ( {data.map((item, index) => { let tooltipText = ""; let displayValue; const renderer = item.type ? customTypeRenderers[item.type] : null; if (renderer) { displayValue = renderer(item.value, data, env); } else { switch (item.type) { case "date": displayValue = !isNaN(item.value) && item.value > 0 ? timestampRenderer(item.value) : "N/A"; tooltipText = new Date(item.value).toISOString(); break; case "date-ms": displayValue = !isNaN(item.value) && item.value > 0 ? timestampMsRenderer(item.value) : "N/A"; tooltipText = new Date(item.value).toISOString(); break; case "duration": displayValue = !isNaN(item.value) && item.value > 0 ? durationRenderer(item.value) : "N/A"; break; default: displayValue = !_.isNil(item.value) ? item.value : "N/A"; } } return ( {displayValue} } /> ); })} ); } ================================================ FILE: ui/src/components/LinearProgress.jsx ================================================ import React from "react"; import { makeStyles } from "@material-ui/styles"; import clsx from "clsx"; import LinearProgress from "@material-ui/core/LinearProgress"; const useStyles = makeStyles({ progress: { marginBottom: -4, zIndex: 999, }, }); export default function ({ className, ...props }) { const classes = useStyles(); return ( ); } ================================================ FILE: ui/src/components/NavLink.jsx ================================================ import React from "react"; import { Link as RouterLink, useHistory } from "react-router-dom"; import { Link } from "@material-ui/core"; import LaunchIcon from "@material-ui/icons/Launch"; import Url from "url-parse"; import { useEnv } from "../plugins/env"; import { getBasename } from "../utils/helpers"; import { cleanDuplicateSlash } from "../plugins/fetch"; // 1. Strip `navigate` from props to prevent error // 2. Preserve stack param export default React.forwardRef((props, ref) => { const { navigate, path, newTab, absolutePath = false, ...rest } = props; const { stack, defaultStack } = useEnv(); const url = new Url(path, {}, true); if (stack !== defaultStack) { url.query.stack = stack; } if (!newTab) { return ( {rest.children} ); } else { const href = absolutePath ? url.toString() : cleanDuplicateSlash(getBasename() + url.toString()); return ( {rest.children}   ); } }); export function usePushHistory() { const history = useHistory(); const { stack, defaultStack } = useEnv(); return (path) => { const url = new Url(path, {}, true); if (stack !== defaultStack) { url.query.stack = stack; } history.push(url.toString()); }; } ================================================ FILE: ui/src/components/Paper.jsx ================================================ import React from "react"; import { makeStyles } from "@material-ui/styles"; import clsx from "clsx"; import Paper from "@material-ui/core/Paper"; const useStyles = makeStyles({ padded: { padding: 15, }, }); export default React.forwardRef(function ( { elevation, className, padded, ...props }, ref ) { const classes = useStyles(); const internalClassName = []; if (padded) internalClassName.push(classes.padded); return ( ); }); ================================================ FILE: ui/src/components/Pill.jsx ================================================ import { makeStyles } from "@material-ui/styles"; import Chip from "@material-ui/core/Chip"; const COLORS = { red: "rgb(229, 9, 20)", yellow: "rgb(251, 164, 4)", green: "rgb(65, 185, 87)", }; const useStyles = makeStyles({ pill: { borderColor: (props) => COLORS[props.color], color: (props) => COLORS[props.color], }, }); export default function Pill({ color, ...props }) { const classes = useStyles({ color }); return ( ); } ================================================ FILE: ui/src/components/PrimaryButton.jsx ================================================ import React from "react"; import Button from "@material-ui/core/Button"; export default function (props) { return dispatch({ type: actions.TOGGLE_GRAPH_PANEL })} > {workflowDefState.toggleGraphPanel && ( )} {!workflowDefState.toggleGraphPanel && ( )} handleMouseDown(e)} />
    {dag && }
    ); } function versionTime(versionObj) { return ( versionObj && timestampRenderer(versionObj.updateTime || versionObj.createTime) ); } ================================================ FILE: ui/src/pages/definitions/EventHandler.jsx ================================================ import React from "react"; import { NavLink, DataTable } from "../../components"; import { makeStyles } from "@material-ui/styles"; import Header from "./Header"; import sharedStyles from "../styles"; import { Helmet } from "react-helmet"; import { useEventHandlers } from "../../data/misc"; const useStyles = makeStyles(sharedStyles); const columns = [ { name: "name", renderer: (name) => ( {name} ), }, { name: "event" }, { name: "createTime", type: "date" }, { name: "actions", renderer: (val) => JSON.stringify(val.map((action) => action.action)), }, ]; export default function EventHandlers() { const classes = useStyles(); const { data: eventHandlers, isFetching } = useEventHandlers(); return (
    Conductor UI - Event Handler Definitions
    {eventHandlers && ( )}
    ); } ================================================ FILE: ui/src/pages/definitions/Header.jsx ================================================ import React from "react"; import { Tab, Tabs, NavLink, LinearProgress, Heading } from "../../components"; import { makeStyles } from "@material-ui/styles"; import sharedStyles from "../styles"; const useStyles = makeStyles(sharedStyles); export default function Header({ tabIndex, loading }) { const classes = useStyles(); return (
    {loading && }
    Definitions
    ); } ================================================ FILE: ui/src/pages/definitions/Task.jsx ================================================ import React from "react"; import { NavLink, DataTable, Button } from "../../components"; import { makeStyles } from "@material-ui/styles"; import Header from "./Header"; import sharedStyles from "../styles"; import { Helmet } from "react-helmet"; import AddIcon from "@material-ui/icons/Add"; import { useTaskDefs } from "../../data/task"; const useStyles = makeStyles(sharedStyles); const columns = [ { name: "name", renderer: (name) => {name}, }, { name: "description", grow: 2 }, { name: "createTime", type: "date" }, { name: "ownerEmail" }, { name: "inputKeys", type: "json", sortable: false }, { name: "outputKeys", type: "json", sortable: false }, { name: "timeoutPolicy", grow: 0.5 }, { name: "timeoutSeconds", grow: 0.5 }, { name: "retryCount", grow: 0.5 }, { name: "retryLogic" }, { name: "retryDelaySeconds", grow: 0.5 }, { name: "responseTimeoutSeconds", grow: 0.5 }, { name: "inputTemplate", type: "json", sortable: false }, { name: "rateLimitPerFrequency", grow: 0.5 }, { name: "rateLimitFrequencyInSeconds", grow: 0.5 }, { name: "name", label: "Executions", id: "executions_link", grow: 0.5, renderer: (name) => ( Query ), sortable: false, searchable: false, }, { name: "concurrentExecLimit" }, { name: "pollTimeoutSeconds" }, ]; export default function TaskDefinitions() { const classes = useStyles(); const { data: tasks, isFetching } = useTaskDefs(); return (
    Conductor UI - Task Definitions
    {tasks && ( )}
    ); } ================================================ FILE: ui/src/pages/definitions/Workflow.jsx ================================================ import React, { useMemo } from "react"; import { NavLink, DataTable, Button } from "../../components"; import { makeStyles } from "@material-ui/styles"; import _ from "lodash"; import { useQueryState } from "react-router-use-location-state"; import { useLatestWorkflowDefs } from "../../data/workflow"; import Header from "./Header"; import sharedStyles from "../styles"; import { Helmet } from "react-helmet"; import AddIcon from "@material-ui/icons/Add"; const useStyles = makeStyles(sharedStyles); const columns = [ { name: "name", renderer: (val) => ( {val.trim()} ), }, { name: "description", grow: 2 }, { name: "createTime", type: "date" }, { name: "version", label: "Latest Version", grow: 0.5 }, { name: "schemaVersion", grow: 0.5 }, { name: "restartable", grow: 0.5 }, { name: "workflowStatusListenerEnabled", grow: 0.5 }, { name: "ownerEmail" }, { name: "inputParameters", type: "json", sortable: false }, { name: "outputParameters", type: "json", sortable: false }, { name: "timeoutPolicy", grow: 0.5 }, { name: "timeoutSeconds", grow: 0.5 }, { id: "task_types", name: "tasks", label: "Task Types", searchable: "calculated", sortable: false, renderer: (val) => { const taskTypeSet = new Set(); for (let task of val) { taskTypeSet.add(task.type); } return Array.from(taskTypeSet).join(", "); }, }, { id: "task_count", name: "tasks", label: "Tasks", searchable: "calculated", sortable: false, grow: 0.5, renderer: (val) => (_.isArray(val) ? val.length : 0), }, { id: "executions_link", name: "name", label: "Executions", sortable: false, searchable: false, grow: 0.5, renderer: (name) => ( Query ), }, ]; export default function WorkflowDefinitions() { const classes = useStyles(); const { data, isFetching } = useLatestWorkflowDefs(); const [filterParam, setFilterParam] = useQueryState("filter", ""); const filterObj = filterParam === "" ? undefined : JSON.parse(filterParam); const handleFilterChange = (obj) => { if (obj) { setFilterParam(JSON.stringify(obj)); } else { setFilterParam(""); } }; const workflows = useMemo(() => { // Extract latest versions only if (data) { const unique = new Map(); const types = new Set(); for (let workflowDef of data) { if (!unique.has(workflowDef.name)) { unique.set(workflowDef.name, workflowDef); } else if (unique.get(workflowDef.name).version < workflowDef.version) { unique.set(workflowDef.name, workflowDef); } for (let task of workflowDef.tasks) { types.add(task.type); } } return Array.from(unique.values()); } }, [data]); return (
    Conductor UI - Workflow Definitions
    {workflows && ( )}
    ); } ================================================ FILE: ui/src/pages/execution/ActionModule.jsx ================================================ import { makeStyles } from "@material-ui/styles"; import { isFailedTask } from "../../utils/helpers"; import { DropdownButton } from "../../components"; import { ListItemIcon, ListItemText } from "@material-ui/core"; import StopIcon from "@material-ui/icons/Stop"; import PauseIcon from "@material-ui/icons/Pause"; import ReplayIcon from "@material-ui/icons/Replay"; import ResumeIcon from "@material-ui/icons/PlayArrow"; import RedoIcon from "@material-ui/icons/Redo"; import FlareIcon from "@material-ui/icons/Flare"; import { useRestartAction, useRestartLatestAction, useResumeAction, useRetryResumeSubworkflowTasksAction, useRetryAction, useTerminateAction, usePauseAction, } from "../../data/actions"; const useStyles = makeStyles({ terminate: { color: "red", }, }); export default function ActionModule({ execution, triggerReload }) { const classes = useStyles(); const { workflowId, workflowDefinition } = execution; const restartAction = useRestartAction({ workflowId, onSuccess }); const restartLatestAction = useRestartLatestAction({ workflowId, onSuccess }); const retryAction = useRetryAction({ workflowId, onSuccess }); const retryResumeSubworkflowTasksAction = useRetryResumeSubworkflowTasksAction({ workflowId, onSuccess }); const terminateAction = useTerminateAction({ workflowId, onSuccess }); const resumeAction = useResumeAction({ workflowId, onSuccess }); const pauseAction = usePauseAction({ workflowId, onSuccess }); const { restartable } = workflowDefinition; function onSuccess() { triggerReload(); } const options = []; // RESTART buttons if ( ["COMPLETED", "FAILED", "TIMED_OUT", "TERMINATED"].includes( execution.status ) && restartable ) { options.push({ label: ( <> Restart with Current Definitions ), handler: () => restartAction.mutate(), }); options.push({ label: ( <> Restart with Latest Definitions ), handler: () => restartLatestAction.mutate(), }); } // PAUSE button if (execution.status === "RUNNING") { options.push({ label: ( <> Pause ), handler: () => pauseAction.mutate(), }); } // RESUME button if (execution.status === "PAUSED") { options.push({ label: ( <> Resume ), handler: () => resumeAction.mutate(), }); } // RETRY (from task) button if (["FAILED", "TIMED_OUT", "TERMINATED"].includes(execution.status)) { options.push({ label: ( <> Retry - From failed task ), handler: () => retryAction.mutate(), }); } // RETRY (failed subworkflow) button if ( ["FAILED", "TIMED_OUT", "TERMINATED"].includes(execution.status) && execution.tasks.find( (task) => task.workflowTask.type === "SUB_WORKFLOW" && isFailedTask(task.status) ) ) { options.push({ label: ( <> Retry - Resume failed subworkflow ), handler: () => retryResumeSubworkflowTasksAction.mutate(), }); } // RERUN button // TERMINATE button if (["RUNNING", "FAILED", "TIMED_OUT", "PAUSED"].includes(execution.status)) { options.push({ label: ( <> Terminate ), handler: () => terminateAction.mutate(), }); options.push({ label: ( <> Terminate with Reason ), handler: () => { const reason = window.prompt("Termination Reason", ""); if (reason) terminateAction.mutate({ reason }); }, }); } return Actions; } ================================================ FILE: ui/src/pages/execution/Execution.jsx ================================================ import React, { useMemo, useState, useEffect, useCallback } from "react"; import { useQueryState } from "react-router-use-location-state"; import Alert from "@material-ui/lab/Alert"; import { Tabs, Tab, NavLink, SecondaryButton, LinearProgress, Heading, } from "../../components"; import { Tooltip } from "@material-ui/core"; import { makeStyles } from "@material-ui/styles"; import { useRouteMatch } from "react-router-dom"; import TaskDetails from "./TaskDetails"; import ExecutionSummary from "./ExecutionSummary"; import ExecutionJson from "./ExecutionJson"; import InputOutput from "./ExecutionInputOutput"; import clsx from "clsx"; import ActionModule from "./ActionModule"; import IconButton from "@material-ui/core/IconButton"; import CloseIcon from "@material-ui/icons/Close"; import FullscreenIcon from "@material-ui/icons/Fullscreen"; import FullscreenExitIcon from "@material-ui/icons/FullscreenExit"; import RightPanel from "./RightPanel"; import WorkflowDAG from "../../components/diagram/WorkflowDAG"; import StatusBadge from "../../components/StatusBadge"; import { Helmet } from "react-helmet"; import sharedStyles from "../styles"; import rison from "rison"; import { useWorkflow } from "../../data/workflow"; const maxWindowWidth = window.innerWidth; const INIT_DRAWER_WIDTH = 650; const useStyles = makeStyles({ header: sharedStyles.header, drawer: { zIndex: 999, position: "absolute", top: 0, right: 0, bottom: 0, width: (state) => (state.isFullWidth ? "100%" : state.drawerWidth), }, drawerHeader: { display: "flex", alignItems: "center", padding: 10, justifyContent: "flex-end", height: 80, flexShrink: 0, boxShadow: "0 4px 8px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%)", zIndex: 1, backgroundColor: "#fff", }, dragger: { display: (state) => (state.isFullWidth ? "none" : "block"), width: "5px", cursor: "ew-resize", padding: "4px 0 0", position: "absolute", height: "100%", zIndex: "100", backgroundColor: "#f4f7f9", }, drawerMain: { paddingLeft: (state) => (state.isFullWidth ? 0 : 4), height: "100%", display: "flex", flexDirection: "column", }, drawerContent: { flex: 1, backgroundColor: "#fff", display: "flex", flexDirection: "column", overflow: "hidden", }, content: { height: "100%", display: "flex", flexDirection: "column", }, contentShift: { marginRight: (state) => state.drawerWidth, }, tabContent: { flex: 1, overflow: "hidden", display: "flex", flexDirection: "column", }, headerSubtitle: { marginBottom: 20, }, fr: { display: "flex", position: "relative", float: "right", marginRight: 50, marginTop: 10, zIndex: 1, }, frItem: { display: "flex", alignItems: "center", marginRight: 15, }, rightPanel: { height: "100%", display: "flex", flexDirection: "column", }, }); export default function Execution() { const match = useRouteMatch(); const { data: execution, isFetching, refetch: refresh, } = useWorkflow(match.params.id); const [isFullWidth, setIsFullWidth] = useState(false); const [isResizing, setIsResizing] = useState(false); const [drawerWidth, setDrawerWidth] = useState(INIT_DRAWER_WIDTH); const [tabIndex, setTabIndex] = useQueryState("tabIndex", 0); const [selectedTaskRison, setSelectedTaskRison] = useQueryState("task", ""); const dag = useMemo( () => (execution ? new WorkflowDAG(execution) : null), [execution] ); const selectedTask = useMemo( () => selectedTaskRison && rison.decode(selectedTaskRison), [selectedTaskRison] ); const setSelectedTask = (taskPointer) => { setSelectedTaskRison(rison.encode(taskPointer)); }; const classes = useStyles({ isFullWidth, drawerWidth, }); const handleMousemove = useCallback( (e) => { // we don't want to do anything if we aren't resizing. if (!isResizing) { return; } // Stop highlighting e.preventDefault(); const offsetRight = document.body.offsetWidth - (e.clientX - document.body.offsetLeft); const minWidth = 0; const maxWidth = maxWindowWidth - 100; if (offsetRight > minWidth && offsetRight < maxWidth) { setDrawerWidth(offsetRight); } }, [isResizing] ); const handleMousedown = (e) => setIsResizing(true); const handleClose = () => { setSelectedTaskRison(null); }; const handleFullScreen = () => { setIsFullWidth(true); }; const handleFullScreenExit = () => { setIsFullWidth(false); }; // On load and destroy only useEffect(() => { const mouseUp = (e) => setIsResizing(false); document.addEventListener("mousemove", handleMousemove); document.addEventListener("mouseup", mouseUp); return () => { document.removeEventListener("mousemove", handleMousemove); document.removeEventListener("mouseup", mouseUp); }; }, [handleMousemove]); return ( <> Conductor UI - Execution - {match.params.id}
    {isFetching && } {execution && ( <>
    {execution.parentWorkflowId && (
    Parent Workflow
    )}
    Definition
    Refresh
    {execution.workflowType || execution.workflowName}{" "} {execution.workflowId} {execution.reasonForIncompletion && ( {execution.reasonForIncompletion} )} setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> setTabIndex(3)} />
    {tabIndex === 0 && ( )} {tabIndex === 1 && } {tabIndex === 2 && } {tabIndex === 3 && }
    )}
    {selectedTask && (
    handleMousedown(event)} className={classes.dragger} />
    {isFullWidth ? ( handleFullScreenExit()}> ) : ( handleFullScreen()}> )} handleClose()}>
    )} ); } ================================================ FILE: ui/src/pages/execution/ExecutionInputOutput.jsx ================================================ import React from "react"; import { Paper, ReactJson } from "../../components"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ wrapper: { margin: 30, height: "100%", display: "flex", flexDirection: "column", overflow: "hidden", }, column: { display: "flex", flexDirection: "row", gap: 15, flex: 2, marginBottom: 15, overflow: "hidden", }, paper: { flex: 1, overflow: "hidden", }, }); export default function InputOutput({ execution }) { const classes = useStyles(); return (
    ); } ================================================ FILE: ui/src/pages/execution/ExecutionJson.jsx ================================================ import React from "react"; import { Paper } from "../../components"; import ReactJson from "../../components/ReactJson"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ paper: { margin: 30, flex: 1, }, wrapper: { flex: 1, display: "flex", flexDirection: "column", }, }); export default function ExecutionJson({ execution }) { const classes = useStyles(); return (
    ); } ================================================ FILE: ui/src/pages/execution/ExecutionSummary.jsx ================================================ import React from "react"; import { Paper, NavLink, KeyValueTable } from "../../components"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ paper: { margin: 30, }, wrapper: { overflowY: "auto", }, }); export default function ExecutionSummary({ execution }) { const classes = useStyles(); // To accommodate unexecuted tasks, read type & name out of workflowTask const data = [ { label: "Workflow ID", value: execution.workflowId }, { label: "Status", value: execution.status }, { label: "Version", value: execution.workflowVersion }, { label: "Start Time", value: execution.startTime, type: "date" }, { label: "End Time", value: execution.endTime, type: "date" }, { label: "Duration", value: execution.endTime - execution.startTime, type: "duration", }, ]; if (execution.parentWorkflowId) { data.push({ label: "Parent Workflow ID", value: ( {execution.parentWorkflowId} ), }); } if (execution.parentWorkflowTaskId) { data.push({ label: "Parent Task ID", value: execution.parentWorkflowTaskId, }); } if (execution.reasonForIncompletion) { data.push({ label: "Reason for Incompletion", value: execution.reasonForIncompletion, }); } return (
    ); } ================================================ FILE: ui/src/pages/execution/Legend.jsx ================================================ import React, { Component } from "react"; import WorkflowDAG from "../../components/diagram/WorkflowDAG"; import WorkflowGraph from "../../components/diagram/WorkflowGraph"; const workflowDef = { tasks: [ { name: "fork_join", taskReferenceName: "fork", type: "FORK_JOIN", forkTasks: [ [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkChild_grp1a", }, { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkChild_grp1b", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp2", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp3", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp4", }, ], ], }, { name: "join", taskReferenceName: "join", type: "JOIN", joinOn: ["forkChild_par1", "forkChild_par2", "forkChild_ser1"], }, { name: "decision", taskReferenceName: "decision", type: "DECISION", decisionCases: [ [ { name: "simple_task", type: "SIMPLE", taskReferenceName: "completed", }, ], [ { name: "simple_task", type: "SIMPLE", taskReferenceName: "failed", }, ], ], }, { name: "exclusive_join", taskReferenceName: "exclusiveJoin", type: "EXCLUSIVE_JOIN", joinOn: ["completed", "failed"], defaultExclusiveJoinTask: ["completed"], }, { name: "subworkflow", taskReferenceName: "subworkflow", type: "SUB_WORKFLOW", subworkflowParam: { name: "foo" }, }, { name: "dynamic_fork", taskReferenceName: "dynamic_fork", type: "FORK_JOIN_DYNAMIC", dynamicForkTasksParam: "dynamicTasks", dynamicForkTasksInputParamName: "dynamicTasksInput", }, { name: "join", taskReferenceName: "dynamic_join", type: "JOIN", }, ], }; class Legend extends Component { constructor() { super(); this.state = { dag: new WorkflowDAG(null, workflowDef), }; } render() { const { dag } = this.state; return (
    ); } } export default Legend; ================================================ FILE: ui/src/pages/execution/RightPanel.jsx ================================================ import { useState, useEffect, useMemo } from "react"; import { Tabs, Tab, ReactJson, Dropdown, Banner } from "../../components"; import { TabPanel, TabContext } from "@material-ui/lab"; import TaskSummary from "./TaskSummary"; import TaskLogs from "./TaskLogs"; import { makeStyles } from "@material-ui/styles"; import _ from "lodash"; import TaskPollData from "./TaskPollData"; const useStyles = makeStyles({ banner: { margin: 15, }, dfSelect: { padding: 15, backgroundColor: "#efefef", }, tabPanel: { padding: 0, flex: 1, overflowY: "auto", }, }); export default function RightPanel({ selectedTask, dag, onTaskChange }) { const [tabIndex, setTabIndex] = useState("summary"); const classes = useStyles(); useEffect(() => { setTabIndex("summary"); // Reset to Status Tab on ref change }, [selectedTask]); const taskResult = useMemo( () => dag && dag.resolveTaskResult(selectedTask), [dag, selectedTask] ); const dfOptions = useMemo( () => dag && dag.getSiblings(selectedTask), [dag, selectedTask] ); const retryOptions = useMemo( () => dag && dag.getRetries(selectedTask), [dag, selectedTask] ); if (!taskResult) { return null; } else return ( {dfOptions && (
    { onTaskChange({ ref: v.ref }); }} options={dfOptions} disableClearable value={dfOptions.find( (opt) => opt.ref === taskResult.referenceTaskName )} getOptionLabel={(x) => `${dropdownIcon(x.status)} ${x.ref}`} style={{ marginBottom: 20, width: 500 }} />
    )} {_.size(retryOptions) > 1 && (
    { onTaskChange({ id: v.taskId, }); }} options={retryOptions} value={retryOptions.find( (opt) => opt.taskId === taskResult.taskId )} getOptionLabel={(t) => `${dropdownIcon(t.status)} Attempt ${t.retryCount} - ${ t.taskId }` } style={{ marginBottom: 20, width: 500 }} />
    )} setTabIndex(v)}> {[ , , , , , , ...(_.get(taskResult, "workflowTask.type") === "SIMPLE" ? [ , ] : []), ]} <> {taskResult.externalInputPayloadStoragePath ? ( This task has externalized input. Please reference{" "} externalInputPayloadStoragePath for the storage location. ) : ( )} {taskResult.externalOutputPayloadStoragePath ? ( This task has externalized output. Please reference{" "} externalOutputPayloadStoragePath for the storage location. ) : ( )}     ); } function dropdownIcon(status) { let icon; switch (status) { case "COMPLETED": icon = "\u2705"; break; // Green-checkmark case "COMPLETED_WITH_ERRORS": icon = "\u2757"; break; // Exclamation case "CANCELED": icon = "\uD83D\uDED1"; break; // stopsign case "IN_PROGRESS": case "SCHEDULED": icon = "\u231B"; break; // hourglass default: icon = "\u274C"; // red-X } return icon + "\u2003"; } ================================================ FILE: ui/src/pages/execution/TaskDetails.jsx ================================================ import React, { useState } from "react"; import { Tabs, Tab, Paper } from "../../components"; import Timeline from "./Timeline"; import TaskList from "./TaskList"; import WorkflowGraph from "../../components/diagram/WorkflowGraph"; import { makeStyles } from "@material-ui/styles"; const useStyles = makeStyles({ taskWrapper: { overflowY: "auto", padding: 30, height: "100%", }, }); export default function TaskDetails({ execution, dag, selectedTask, setSelectedTask, }) { const [tabIndex, setTabIndex] = useState(0); const classes = useStyles(); return (
    setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> {tabIndex === 0 && ( )} {tabIndex === 1 && ( )} {tabIndex === 2 && ( )}
    ); } ================================================ FILE: ui/src/pages/execution/TaskList.jsx ================================================ import { DataTable, TaskLink } from "../../components"; export default function TaskList({ selectedTask, tasks, workflowId }) { const taskDetailFields = [ { name: "seq", grow: 0.2 }, { name: "taskId", renderer: (taskId) => ( ), grow: 2, }, { name: "workflowTask.name", id: "taskName", label: "Task Name" }, { name: "referenceTaskName", label: "Ref" }, { name: "workflowTask.type", id: "taskType", label: "Type", grow: 0.5 }, { name: "scheduledTime", type: "date-ms" }, { name: "startTime", type: "date-ms" }, { name: "endTime", type: "date-ms" }, { name: "status", grow: 0.8 }, { name: "updateTime", type: "date-ms" }, { name: "callbackAfterSeconds" }, { name: "pollCount", grow: 0.5 }, ]; return ( ); } ================================================ FILE: ui/src/pages/execution/TaskLogs.jsx ================================================ import React from "react"; import { useLogs } from "../../data/misc"; import { DataTable, Text, LinearProgress } from "../../components"; export default function TaskLogs({ task }) { const { taskId } = task; const { data: log, isFetching } = useLogs({ taskId }); if (isFetching) { return ; } return log && log.length > 0 ? ( ) : ( No logs available ); } ================================================ FILE: ui/src/pages/execution/TaskPollData.jsx ================================================ import React from "react"; import { KeyValueTable, LinearProgress } from "../../components"; import { usePollData, useQueueSize } from "../../data/task"; import _ from "lodash"; import { timestampRenderer } from "../../utils/helpers"; export default function TaskPollData({ task }) { const { data: pollData, isLoading } = usePollData(task.workflowTask.name); const { data: queueSize, isLoadingQueueSize } = useQueueSize( task.workflowTask.name, task.domain ); if (isLoading || isLoadingQueueSize) { return ; } const pollDataRow = pollData.find((row) => { if (task.domain) { return row.domain === task.domain; } else { return _.isUndefined(row.domain); } }); const data = [ { label: "Task Name", value: task.workflowTask.name }, { label: "Domain", value: _.defaultTo(task.domain, "(No Domain Set)") }, ]; if (pollDataRow) { data.push({ label: "Last Polled By Worker", value: pollDataRow.workerId, }); data.push({ label: "Last Poll Time", value: timestampRenderer(pollDataRow.lastPollTime), }); } if (queueSize !== undefined) { data.push({ label: "Current Queue Size", value: queueSize, }); } return ; } ================================================ FILE: ui/src/pages/execution/TaskSummary.jsx ================================================ import React from "react"; import _ from "lodash"; import { NavLink, KeyValueTable } from "../../components"; import { useTime } from "../../hooks/useTime"; export default function TaskSummary({ taskResult }) { const now = useTime(); // To accommodate unexecuted tasks, read type & name & ref out of workflow const data = [ { label: "Task Type", value: taskResult.workflowTask.type }, { label: "Status", value: taskResult.status || "Not executed" }, { label: "Task Name", value: taskResult.workflowTask.name }, { label: "Task Reference", value: taskResult.referenceTaskName || taskResult.workflowTask.aliasForRef || taskResult.workflowTask.taskReferenceName, }, ]; if (taskResult.domain) { data.push({ label: "Domain", value: taskResult.domain }); } if (taskResult.taskId) { data.push({ label: "Task Execution ID", value: taskResult.taskId }); } if (_.isFinite(taskResult.retryCount)) { data.push({ label: "Retry Count", value: taskResult.retryCount }); } if (taskResult.scheduledTime) { data.push({ label: "Scheduled Time", value: taskResult.scheduledTime > 0 && taskResult.scheduledTime, type: "date-ms", }); } if (taskResult.startTime) { data.push({ label: "Start Time", value: taskResult.startTime > 0 && taskResult.startTime, type: "date-ms", }); } if (taskResult.endTime) { data.push({ label: "End Time", value: taskResult.endTime, type: "date-ms", }); } if (taskResult.startTime && taskResult.endTime) { data.push({ label: "Duration", value: taskResult.startTime > 0 && taskResult.endTime - taskResult.startTime, type: "duration", }); } if (taskResult.startTime && taskResult.status === "IN_PROGRESS") { data.push({ label: "Current Elapsed Time", value: taskResult.startTime > 0 && now - taskResult.startTime, type: "duration", }); } if (!_.isNil(taskResult.retrycount)) { data.push({ label: "Retry Count", value: taskResult.retryCount }); } if (taskResult.reasonForIncompletion) { data.push({ label: "Reason for Incompletion", value: taskResult.reasonForIncompletion, }); } if (taskResult.workerId) { data.push({ label: "Worker", value: taskResult.workerId, type: "workerId", }); } if (taskResult.taskType === "DECISION") { data.push({ label: "Evaluated Case", value: _.has(taskResult, "outputData.caseOutput[0]") && taskResult.outputData.caseOutput[0], }); } if (taskResult.workflowTask.type === "SUB_WORKFLOW") { data.push({ label: "Subworkflow Definition", value: ( {taskResult.workflowTask.subWorkflowParam.name}{" "} ), }); if (_.has(taskResult, "subWorkflowId")) { data.push({ label: "Subworkflow ID", value: ( {taskResult.subWorkflowId} ), }); } } if (taskResult.externalInputPayloadStoragePath) { data.push({ label: "Externalized Input", value: taskResult.externalInputPayloadStoragePath, }); } if (taskResult.externalOutputPayloadStoragePath) { data.push({ label: "Externalized Output", value: taskResult.externalOutputPayloadStoragePath, }); } return ; } ================================================ FILE: ui/src/pages/execution/Timeline.jsx ================================================ import React, { useMemo } from "react"; import Timeline from "react-vis-timeline-2"; import { timestampRenderer, durationRenderer } from "../../utils/helpers"; import _ from "lodash"; import "./timeline.scss"; import ZoomOutMapIcon from "@material-ui/icons/ZoomOutMap"; import { IconButton, Tooltip } from "@material-ui/core"; export default function TimelineComponent({ dag, tasks, onClick, selectedTask, }) { const timelineRef = React.useRef(); /* const selectedId = useMemo(() => { if(selectedTask){ const taskResult = dag.resolveTaskResult(selectedTask); return _.get(taskResult, "taskId") } }, [dag, selectedTask]); */ const selectedId = null; const { items, groups } = useMemo(() => { const groupMap = new Map(); for (const task of tasks) { groupMap.set(task.referenceTaskName, { id: task.referenceTaskName, content: `${task.referenceTaskName} (${task.workflowTask.name})`, }); } const items = tasks .filter((t) => t.startTime > 0 || t.endTime > 0) .map((task) => { const dfParent = dag.graph .predecessors(task.referenceTaskName) .map((t) => dag.graph.node(t)) .find((t) => t.type === "FORK_JOIN_DYNAMIC"); const startTime = task.startTime > 0 ? new Date(task.startTime) : new Date(task.endTime); const endTime = task.endTime > 0 ? new Date(task.endTime) : new Date(task.startTime); const duration = durationRenderer( endTime.getTime() - startTime.getTime() ); const retval = { id: task.taskId, group: task.referenceTaskName, content: `${duration}`, start: startTime, end: endTime, title: `${task.referenceTaskName} (${ task.status })
    ${timestampRenderer(startTime)} - ${timestampRenderer( endTime )}`, className: `status_${task.status}`, }; if (dfParent || task.type === "FORK_JOIN_DYNAMIC") { //retval.subgroup=task.referenceTaskName const gp = groupMap.get(dfParent.ref); if (!gp.nestedGroups) { gp.nestedGroups = []; } groupMap.get(task.referenceTaskName).treeLevel = 2; gp.nestedGroups.push(task.referenceTaskName); } return retval; }); return { items: items, groups: Array.from(groupMap.values()), }; }, [tasks, dag]); const onFit = () => { timelineRef.current.timeline.fit(); }; const handleClick = (e) => { const { group, item, what } = e; if (group && what !== "background") { if (_.size(dag.graph.node(group).taskResults) > 1) { onClick({ ref: group, taskId: item, }); } else { onClick({ ref: group }); } } }; return (
    Ctrl-scroll to zoom.{" "}

    ); } ================================================ FILE: ui/src/pages/execution/timeline.scss ================================================ @mixin barColor($colorfg, $colorbg: #fff) { background-color: $colorbg; border-color: $colorfg; color: $colorfg; } .vis-timeline { border: none; } .vis-panel { &.vis-top, &.vis-center { border-left: none; } } .vis-label { .vis-inner { margin-left: 5px; } &.vis-nested-group.vis-group-level-2 { background: white; } } .vis-item { &.status_COMPLETED { @include barColor(#163e1d, #aee1b8); } &.status_COMPLETED_WITH_ERRORS { @include barColor(#8b5b02, #feeac5); } &.status_IN_PROGRESS, &.status_SCHEDULED { @include barColor(#11497a, #cbe2f7); } //&.status_CANCELED { @include barColor(#26194b, #ded5f8); } &.status_FAILED, &.status_FAILED_WITH_TERMINAL_ERROR, &.status_TIMED_OUT, &.status_DF_PARTIAL, &.status_CANCELED { @include barColor(#7f050b, #f9c6c9); } &.status_SKIPPED { @include barColor(gray); } &.vis-selected { filter: brightness(70%); } .vis-item-content { font-size: 10px; padding: 0px 3px 0px 3px; } } ================================================ FILE: ui/src/pages/executions/BulkActionModule.jsx ================================================ import React, { useState } from "react"; import { Dialog, DialogContent, DialogActions, DialogTitle, } from "@material-ui/core"; import { makeStyles } from "@material-ui/styles"; import { DataTable, DropdownButton, LinearProgress, PrimaryButton, Heading, } from "../../components"; import { useBulkRestartAction, useBulkRestartLatestAction, useBulkResumeAction, useBulkTerminateAction, useBulkPauseAction, useBulkRetryAction, useBulkTerminateWithReasonAction, } from "../../data/bulkactions"; const useStyles = makeStyles({ actionBar: { display: "flex", alignItems: "center", paddingRight: 10, "&>div, &>p": { marginRight: 10, }, width: "100%", justifyContent: "space-between", }, }); export default function BulkActionModule({ selectedRows }) { const selectedIds = selectedRows.map((row) => row.workflowId); const [results, setResults] = useState(); const classes = useStyles(); const { mutate: pauseAction, isLoading: pauseLoading } = useBulkPauseAction({ onSuccess, }); const { mutate: resumeAction, isLoading: resumeLoading } = useBulkResumeAction({ onSuccess }); const { mutate: restartCurrentAction, isLoading: restartCurrentLoading } = useBulkRestartAction({ onSuccess }); const { mutate: restartLatestAction, isLoading: restartLatestLoading } = useBulkRestartLatestAction({ onSuccess }); const { mutate: retryAction, isLoading: retryLoading } = useBulkRetryAction({ onSuccess, }); const { mutate: terminateAction, isLoading: terminateLoading } = useBulkTerminateAction({ onSuccess }); const { mutate: terminateWithReasonAction, isLoading: terminateWithReasonLoading, } = useBulkTerminateWithReasonAction({ onSuccess }); const isLoading = pauseLoading || resumeLoading || restartCurrentLoading || restartLatestLoading || retryLoading || terminateLoading || terminateWithReasonLoading; function onSuccess(data, variables, context) { const retval = { bulkErrorResults: Object.entries(data.bulkErrorResults).map( ([key, value]) => ({ workflowId: key, message: value, }) ), bulkSuccessfulResults: data.bulkSuccessfulResults.map((value) => ({ workflowId: value, })), }; setResults(retval); } function handleClose() { setResults(null); } return (
    {selectedRows.length} Workflows Selected. pauseAction({ body: JSON.stringify(selectedIds) }), }, { label: "Resume", handler: () => resumeAction({ body: JSON.stringify(selectedIds) }), }, { label: "Restart with current definitions", handler: () => restartCurrentAction({ body: JSON.stringify(selectedIds) }), }, { label: "Restart with latest definitions", handler: () => restartLatestAction({ body: JSON.stringify(selectedIds) }), }, { label: "Retry", handler: () => retryAction({ body: JSON.stringify(selectedIds) }), }, { label: "Terminate", handler: () => terminateAction({ body: JSON.stringify(selectedIds) }), }, { label: "Terminate with Reason", handler: () => { const reason = window.prompt("Termination Reason", ""); if (reason) { terminateWithReasonAction({ body: JSON.stringify(selectedIds), reason, }); } }, }, ]} > Bulk Action {(results || isLoading) && ( Batch Actions {isLoading && } {results && ( )} Close )}
    ); } ================================================ FILE: ui/src/pages/executions/ResultsTable.jsx ================================================ import React, { useState, useRef, useEffect } from "react"; import { Paper, NavLink, DataTable, LinearProgress, TertiaryButton, Text, } from "../../components"; import { Alert, AlertTitle } from "@material-ui/lab"; import { makeStyles } from "@material-ui/styles"; import BulkActionModule from "./BulkActionModule"; import executionsStyles from "./executionsStyles"; import sharedStyles from "../styles"; const useStyles = makeStyles({ ...executionsStyles, ...sharedStyles, }); const executionFields = [ { name: "startTime", type: "date" }, { name: "workflowId", grow: 2, renderer: (workflowId) => ( {workflowId} ), }, { name: "workflowType", grow: 2 }, { name: "version", grow: 0.5 }, { name: "correlationId", grow: 2 }, { name: "updateTime", type: "date" }, { name: "endTime", type: "date" }, { name: "status" }, { name: "input", grow: 2, wrap: true }, { name: "output", grow: 2 }, { name: "reasonForIncompletion" }, { name: "executionTime" }, { name: "event" }, { name: "failedReferenceTaskNames", grow: 2 }, { name: "externalInputPayloadStoragePath" }, { name: "externalOutputPayloadStoragePath" }, { name: "priority" }, ]; function ShowMore({ rowsPerPage, rowCount, onChangePage, onChangeRowsPerPage, currentPage, }) { return (
    onChangePage(currentPage + 1)}> Show More Results
    ); } export default function ResultsTable({ resultObj, error, busy, page, rowsPerPage, sort, setPage, setSort, setRowsPerPage, showMore, }) { const classes = useStyles(); let totalHits = 0; if (resultObj) { if (resultObj.totalHits) { totalHits = resultObj.totalHits; } else { if (resultObj.results) { totalHits = resultObj.results.length; } } } const [selectedRows, setSelectedRows] = useState([]); const [toggleCleared, setToggleCleared] = useState(false); const tableRef = useRef(null); const defaultSortField = sort ? sort.split(":")[0] : null; const defaultSortDirection = sort ? sort.split(":")[1] : null; useEffect(() => { setSelectedRows([]); setToggleCleared((t) => !t); }, [resultObj]); return ( {busy && } {error && ( Query Failed {error.message} )} {!resultObj && !error && ( Click "Search" to submit query. )} {resultObj && ( 0 && ` Page ${page} of ${totalHits}`} data={resultObj.results} columns={executionFields} defaultShowColumns={[ "startTime", "workflowType", "workflowId", "endTime", "status", ]} localStorageKey="executionsTable" keyField="workflowId" paginationServer paginationTotalRows={totalHits} paginationDefaultPage={page} paginationPerPage={rowsPerPage} onChangeRowsPerPage={(rowsPerPage) => setRowsPerPage(rowsPerPage)} onChangePage={(page) => setPage(page)} sortServer defaultSortField={defaultSortField} defaultSortAsc={defaultSortDirection === "ASC"} onSort={(column, sortDirection) => { setSort(column.id, sortDirection); }} selectableRows contextComponent={ } onSelectedRowsChange={({ selectedRows }) => setSelectedRows(selectedRows) } clearSelectedRows={toggleCleared} customStyles={{ header: { style: { overflow: "visible", }, }, contextMenu: { style: { display: "none", }, activeStyle: { display: "flex", }, }, }} paginationComponent={showMore ? ShowMore : null} /> )} ); } ================================================ FILE: ui/src/pages/executions/SearchTabs.jsx ================================================ import React from "react"; import { Tab, Tabs, NavLink } from "../../components"; export default function SearchTabs({ tabIndex }) { return ( ); } ================================================ FILE: ui/src/pages/executions/TaskResultsTable.jsx ================================================ import React, { useState, useRef, useEffect } from "react"; import { Paper, NavLink, DataTable, LinearProgress, TertiaryButton, Text, } from "../../components"; import { Alert, AlertTitle } from "@material-ui/lab"; import { makeStyles } from "@material-ui/styles"; import BulkActionModule from "./BulkActionModule"; import executionsStyles from "./executionsStyles"; import sharedStyles from "../styles"; import TaskLink from "../../components/TaskLink"; import { SEARCH_TASK_TYPES_SET } from "../../utils/constants"; const useStyles = makeStyles({ ...executionsStyles, ...sharedStyles, }); const executionFields = [ { name: "updateTime", label: "Update Time", type: "date" }, { name: "scheduledTime", label: "Scheduled Time", type: "date" }, { name: "startTime", label: "Start Time", type: "date" }, { name: "endTime", label: "End Time", type: "date" }, { name: "taskId", label: "Task ID", grow: 1.5, renderer: (taskId, row) => ( ), }, { name: "taskDefName", label: "Task Name", grow: 1.5, renderer: (taskDefName) => SEARCH_TASK_TYPES_SET.has(taskDefName) ? "-" : taskDefName, }, { name: "taskType", label: "Task Type", grow: 0.6, sortable: false, renderer: (taskType) => SEARCH_TASK_TYPES_SET.has(taskType) ? taskType : "SIMPLE", }, { name: "workflowId", label: "Workflow ID", grow: 2, renderer: (workflowId) => ( {workflowId} ), }, { name: "workflowType", label: "Workflow Name", grow: 1.5 }, { name: "executionTime", label: "Execution Time", grow: 0.6, sortable: false, }, { name: "queueWaitTime", label: "Queue Wait Time", grow: 0.6, sortable: false, }, { name: "workflowPriority", label: "Workflow Priority", grow: 0.6, sortable: false, }, { name: "status", label: "Status", sortable: false, }, { name: "input", label: "Input", grow: 3, sortable: false, wrap: true }, { name: "output", label: "Output", grow: 3, sortable: false, wrap: true }, { name: "reasonForIncompletion", label: "Reason for Incompletion", grow: 3, sortable: false, wrap: true, }, ]; function ShowMore({ rowsPerPage, rowCount, onChangePage, onChangeRowsPerPage, currentPage, }) { return (
    onChangePage(currentPage + 1)}> Show More Results
    ); } export default function ResultsTable({ resultObj, error, busy, page, rowsPerPage, sort, setPage, setSort, setRowsPerPage, showMore, }) { const classes = useStyles(); let totalHits = 0; if (resultObj) { if (resultObj.totalHits) { totalHits = resultObj.totalHits; } else { if (resultObj.results) { totalHits = resultObj.results.length; } } } const [selectedRows, setSelectedRows] = useState([]); const [toggleCleared, setToggleCleared] = useState(false); const tableRef = useRef(null); const defaultSortField = sort ? sort.split(":")[0] : null; const defaultSortDirection = sort ? sort.split(":")[1] : null; useEffect(() => { setSelectedRows([]); setToggleCleared((t) => !t); }, [resultObj]); return ( {busy && } {error && ( Query Failed {error.message} )} {!resultObj && !error && ( Click "Search" to submit query. )} {resultObj && ( 0 && ` Page ${page} of ${totalHits}`} data={resultObj.results} columns={executionFields} defaultShowColumns={[ "updateTime", "taskId", "taskDefName", "workflowType", "executionType", "taskType", "status", ]} localStorageKey="taskResultsTable" keyField="taskId" paginationServer paginationTotalRows={totalHits} paginationDefaultPage={page} paginationPerPage={rowsPerPage} onChangeRowsPerPage={(rowsPerPage) => setRowsPerPage(rowsPerPage)} onChangePage={(page) => setPage(page)} sortServer defaultSortField={defaultSortField} defaultSortAsc={defaultSortDirection === "ASC"} onSort={(column, sortDirection) => { setSort(column.id, sortDirection); }} selectableRows contextComponent={ } onSelectedRowsChange={({ selectedRows }) => setSelectedRows(selectedRows) } clearSelectedRows={toggleCleared} customStyles={{ header: { style: { overflow: "visible", }, }, contextMenu: { style: { display: "none", }, activeStyle: { display: "flex", }, }, }} paginationComponent={showMore ? ShowMore : null} /> )} ); } ================================================ FILE: ui/src/pages/executions/TaskSearch.jsx ================================================ import React, { useState } from "react"; import _ from "lodash"; import { FormControl, Grid, InputLabel } from "@material-ui/core"; import { Paper, Heading, PrimaryButton, Dropdown, Input, TaskNameInput, WorkflowNameInput, } from "../../components"; import { TASK_STATUSES, SEARCH_TASK_TYPES_SET } from "../../utils/constants"; import { useQueryState } from "react-router-use-location-state"; import SearchTabs from "./SearchTabs"; import TaskResultsTable from "./TaskResultsTable"; import DateRangePicker from "../../components/DateRangePicker"; import { DEFAULT_ROWS_PER_PAGE } from "../../components/DataTable"; import { useTaskSearch } from "../../data/task"; import { makeStyles } from "@material-ui/styles"; import clsx from "clsx"; import executionsStyles from "./executionsStyles"; import sharedStyles from "../styles"; const useStyles = makeStyles({ ...executionsStyles, ...sharedStyles, }); const DEFAULT_SORT = "startTime:DESC"; const MS_IN_DAY = 86400000; const taskTypeOptions = Array.from(SEARCH_TASK_TYPES_SET.values()); export default function WorkflowPanel() { const classes = useStyles(); const [freeText, setFreeText] = useQueryState("freeText", ""); const [status, setStatus] = useQueryState("status", []); const [taskName, setTaskName] = useQueryState("taskName", []); const [taskId, setTaskId] = useQueryState("taskId", ""); const [taskType, setTaskType] = useQueryState("taskType", []); const [startFrom, setStartFrom] = useQueryState("startFrom", ""); const [startTo, setStartTo] = useQueryState("startTo", ""); const [lookback, setLookback] = useQueryState("lookback", ""); const [workflowType, setWorkflowType] = useQueryState("workflowType", []); const [page, setPage] = useQueryState("page", 1); const [rowsPerPage, setRowsPerPage] = useQueryState( "rowsPerPage", DEFAULT_ROWS_PER_PAGE ); const [sort, setSort] = useQueryState("sort", DEFAULT_SORT); const [queryFT, setQueryFT] = useState(buildQuery); const { data: resultObj, error, isFetching, refetch, } = useTaskSearch({ page, rowsPerPage, sort, query: queryFT.query, freeText: queryFT.freeText, }); function buildQuery() { const clauses = []; if (!_.isEmpty(taskName)) { clauses.push(`taskDefName IN (${taskName.join(",")})`); } if (!_.isEmpty(taskType)) { clauses.push(`taskType IN (${taskType.join(",")})`); } if (!_.isEmpty(taskId)) { clauses.push(`taskId="${taskId}"`); } if (!_.isEmpty(status)) { clauses.push(`status IN (${status.join(",")})`); } if (!_.isEmpty(lookback)) { clauses.push(`updateTime>${new Date().getTime() - lookback * MS_IN_DAY}`); } if (!_.isEmpty(startFrom)) { clauses.push(`updateTime>${new Date(startFrom).getTime()}`); } if (!_.isEmpty(startTo)) { clauses.push(`updateTime<${new Date(startTo).getTime()}`); } if (!_.isEmpty(workflowType)) { clauses.push(`workflowType IN (${workflowType.join(",")})`); } return { query: clauses.join(" AND "), freeText: _.isEmpty(freeText) ? "*" : freeText, }; } function doSearch() { setPage(1); const oldQueryFT = queryFT; const newQueryFT = buildQuery(); setQueryFT(newQueryFT); // Only force refetch if query didn't change. Else let react-query detect difference and refetch automatically if (_.isEqual(oldQueryFT, newQueryFT)) { refetch(); } } const handlePage = (page) => { setPage(page); }; const handleSort = (changedColumn, direction) => { const sort = `${changedColumn}:${direction.toUpperCase()}`; setPage(1); setSort(sort); }; const handleRowsPerPage = (rowsPerPage) => { setPage(1); setRowsPerPage(rowsPerPage); }; const handleLookback = (val) => { setStartFrom(""); setStartTo(""); setLookback(val); }; const handleStartFrom = (val) => { setLookback(""); setStartFrom(val); }; const handleStartTo = (val) => { setLookback(""); setStartTo(val); }; return (
    Search Executions setTaskName(val)} value={taskName} /> setStatus(val)} value={status} /> setTaskType(val)} value={taskType} /> setWorkflowType(val)} value={workflowType} />   Search
    ); } ================================================ FILE: ui/src/pages/executions/WorkflowSearch.jsx ================================================ import React, { useState } from "react"; import _ from "lodash"; import { FormControl, Grid, InputLabel } from "@material-ui/core"; import { Paper, Heading, PrimaryButton, Dropdown, Input, WorkflowNameInput, } from "../../components"; import { workflowStatuses } from "../../utils/constants"; import { useQueryState } from "react-router-use-location-state"; import SearchTabs from "./SearchTabs"; import ResultsTable from "./ResultsTable"; import DateRangePicker from "../../components/DateRangePicker"; import { DEFAULT_ROWS_PER_PAGE } from "../../components/DataTable"; import { useWorkflowSearch } from "../../data/workflow"; import { makeStyles } from "@material-ui/styles"; import clsx from "clsx"; import executionsStyles from "./executionsStyles"; import sharedStyles from "../styles"; const useStyles = makeStyles({ ...executionsStyles, ...sharedStyles, }); const DEFAULT_SORT = "startTime:DESC"; const MS_IN_DAY = 86400000; export default function WorkflowPanel() { const classes = useStyles(); const [freeText, setFreeText] = useQueryState("freeText", ""); const [status, setStatus] = useQueryState("status", []); const [workflowType, setWorkflowType] = useQueryState("workflowType", []); const [workflowId, setWorkflowId] = useQueryState("workflowId", ""); const [startFrom, setStartFrom] = useQueryState("startFrom", ""); const [startTo, setStartTo] = useQueryState("startTo", ""); const [lookback, setLookback] = useQueryState("lookback", ""); const [page, setPage] = useQueryState("page", 1); const [rowsPerPage, setRowsPerPage] = useQueryState( "rowsPerPage", DEFAULT_ROWS_PER_PAGE ); const [sort, setSort] = useQueryState("sort", DEFAULT_SORT); const [queryFT, setQueryFT] = useState(buildQuery); const { data: resultObj, error, isFetching, refetch, } = useWorkflowSearch({ page, rowsPerPage, sort, query: queryFT.query, freeText: queryFT.freeText, }); function buildQuery() { const clauses = []; if (!_.isEmpty(workflowType)) { clauses.push(`workflowType IN (${workflowType.join(",")})`); } if (!_.isEmpty(workflowId)) { clauses.push(`workflowId="${workflowId}"`); } if (!_.isEmpty(status)) { clauses.push(`status IN (${status.join(",")})`); } if (!_.isEmpty(lookback)) { clauses.push(`startTime>${new Date().getTime() - lookback * MS_IN_DAY}`); } if (!_.isEmpty(startFrom)) { clauses.push(`startTime>${new Date(startFrom).getTime()}`); } if (!_.isEmpty(startTo)) { clauses.push(`startTime<${new Date(startTo).getTime()}`); } return { query: clauses.join(" AND "), freeText: _.isEmpty(freeText) ? "*" : freeText, }; } function doSearch() { setPage(1); const oldQueryFT = queryFT; const newQueryFT = buildQuery(); setQueryFT(newQueryFT); // Only force refetch if query didn't change. Else let react-query detect difference and refetch automatically if (_.isEqual(oldQueryFT, newQueryFT)) { refetch(); } } const handlePage = (page) => { setPage(page); }; const handleSort = (changedColumn, direction) => { const sort = `${changedColumn}:${direction.toUpperCase()}`; setPage(1); setSort(sort); }; const handleRowsPerPage = (rowsPerPage) => { setPage(1); setRowsPerPage(rowsPerPage); }; const handleLookback = (val) => { setStartFrom(""); setStartTo(""); setLookback(val); }; const handleStartFrom = (val) => { setLookback(""); setStartFrom(val); }; const handleStartTo = (val) => { setLookback(""); setStartTo(val); }; return (
    Search Executions setWorkflowType(val)} value={workflowType} /> setStatus(val)} value={status} />   Search
    ); } ================================================ FILE: ui/src/pages/executions/executionsStyles.js ================================================ export default { clickSearch: { width: "100%", padding: 30, display: "block", textAlign: "center", }, paper: { marginBottom: 30, }, heading: { marginBottom: 30, }, controls: { padding: 15, }, popupIndicator: { backgroundColor: "red", }, banner: { marginBottom: 15, }, }; ================================================ FILE: ui/src/pages/kitchensink/DataTableDemo.jsx ================================================ import React from "react"; import data from "./sampleMovieData"; import { DataTable } from "../../components"; export default () => { const columns = [ { name: "title" }, { name: "director" }, { name: "year" }, { name: "plot", grow: 0.5 }, ]; return ( <> ); }; ================================================ FILE: ui/src/pages/kitchensink/DiagramTest.jsx ================================================ import React, { Component } from "react"; import WorkflowDAG from "../../components/diagram/WorkflowDAG"; import WorkflowGraph from "../../components/diagram/WorkflowGraph"; const workflowDef = { tasks: [ { name: "fork_join", taskReferenceName: "fork", type: "FORK_JOIN", forkTasks: [ [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkChild_grp1a", }, { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkChild_grp1b", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp2", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp3", }, ], [ { name: "forkChild", type: "SIMPLE", taskReferenceName: "forkchild_grp4", }, ], ], }, { name: "join", taskReferenceName: "join", type: "JOIN", joinOn: ["forkChild_par1", "forkChild_par2", "forkChild_ser1"], }, { name: "decision", taskReferenceName: "decision", type: "DECISION", decisionCases: [ [ { name: "simple_task", type: "SIMPLE", taskReferenceName: "completed", }, ], [ { name: "simple_task", type: "SIMPLE", taskReferenceName: "failed", }, ], ], }, { name: "exclusive_join", taskReferenceName: "exclusiveJoin", type: "EXCLUSIVE_JOIN", joinOn: ["completed", "failed"], defaultExclusiveJoinTask: ["completed"], }, { name: "subworkflow", taskReferenceName: "subworkflow", type: "SUB_WORKFLOW", subworkflowParam: { name: "foo" }, }, { name: "dynamic_fork", taskReferenceName: "dynamic_fork", type: "FORK_JOIN_DYNAMIC", dynamicForkTasksParam: "dynamicTasks", dynamicForkTasksInputParamName: "dynamicTasksInput", }, { name: "join", taskReferenceName: "dynamic_join", type: "JOIN", }, ], }; class DiagramTest extends Component { constructor() { super(); this.state = { dag: new WorkflowDAG(null, workflowDef), }; } render() { const { dag } = this.state; return (
    ); } } export default DiagramTest; ================================================ FILE: ui/src/pages/kitchensink/Dropdown.jsx ================================================ import { useState } from "react"; import { Paper, Heading, Select } from "../../components"; import { MenuItem } from "@material-ui/core"; import top100Films from "./sampleMovieData"; import Dropdown from "../../components/Dropdown"; export default function () { const [value, setValue] = useState(10); const [dropdownValue, setDropdownValue] = useState(); const [dropdownValues, setDropdownValues] = useState([]); return ( Select option.title} onChange={(e, v) => setDropdownValue(v)} /> option.title} loading /> option.title} onChange={(e, v) => setDropdownValue(v)} /> option.title} onChange={(e, v) => setDropdownValue(v)} /> option.title} onChange={(e, v) => setDropdownValue(v)} /> option.title} style={{ width: 500 }} value={dropdownValues} onChange={(e, v) => setDropdownValues(v)} /> ); } ================================================ FILE: ui/src/pages/kitchensink/EnhancedTable.jsx ================================================ import React from "react"; import PropTypes from "prop-types"; import clsx from "clsx"; import { lighten, makeStyles } from "@material-ui/core/styles"; import { Table, TableBody, TableCell, TableContainer, TableHead, TablePagination, TableRow, TableSortLabel, Toolbar, Checkbox, FormControlLabel, Switch, } from "@material-ui/core"; import { Paper, Text, Heading } from "../../components"; function createData(name, calories, fat, carbs, protein) { return { name, calories, fat, carbs, protein }; } const rows = [ createData("Cupcake", 305, 3.7, 67, 4.3), createData("Donut", 452, 25.0, 51, 4.9), createData("Eclair", 262, 16.0, 24, 6.0), createData("Frozen yoghurt", 159, 6.0, 24, 4.0), createData("Gingerbread", 356, 16.0, 49, 3.9), createData("Honeycomb", 408, 3.2, 87, 6.5), createData("Ice cream sandwich", 237, 9.0, 37, 4.3), createData("Jelly Bean", 375, 0.0, 94, 0.0), createData("KitKat", 518, 26.0, 65, 7.0), createData("Lollipop", 392, 0.2, 98, 0.0), createData("Marshmallow", 318, 0, 81, 2.0), createData("Nougat", 360, 19.0, 9, 37.0), createData("Oreo", 437, 18.0, 63, 4.0), ]; function descendingComparator(a, b, orderBy) { if (b[orderBy] < a[orderBy]) { return -1; } if (b[orderBy] > a[orderBy]) { return 1; } return 0; } function getComparator(order, orderBy) { return order === "desc" ? (a, b) => descendingComparator(a, b, orderBy) : (a, b) => -descendingComparator(a, b, orderBy); } function stableSort(array, comparator) { const stabilizedThis = array.map((el, index) => [el, index]); stabilizedThis.sort((a, b) => { const order = comparator(a[0], b[0]); if (order !== 0) return order; return a[1] - b[1]; }); return stabilizedThis.map((el) => el[0]); } const headCells = [ { id: "name", numeric: false, disablePadding: true, label: "Dessert (100g serving)", }, { id: "calories", numeric: true, disablePadding: false, label: "Calories" }, { id: "fat", numeric: true, disablePadding: false, label: "Fat (g)" }, { id: "carbs", numeric: true, disablePadding: false, label: "Carbs (g)" }, { id: "protein", numeric: true, disablePadding: false, label: "Protein (g)" }, ]; function EnhancedTableHead(props) { const { classes, onSelectAllClick, order, orderBy, numSelected, rowCount, onRequestSort, } = props; const createSortHandler = (property) => (event) => { onRequestSort(event, property); }; return ( 0 && numSelected < rowCount} checked={rowCount > 0 && numSelected === rowCount} onChange={onSelectAllClick} inputProps={{ "aria-label": "select all desserts" }} /> {headCells.map((headCell) => ( {headCell.label} {orderBy === headCell.id ? ( {order === "desc" ? "sorted descending" : "sorted ascending"} ) : null} ))} ); } EnhancedTableHead.propTypes = { classes: PropTypes.object.isRequired, numSelected: PropTypes.number.isRequired, onRequestSort: PropTypes.func.isRequired, onSelectAllClick: PropTypes.func.isRequired, order: PropTypes.oneOf(["asc", "desc"]).isRequired, orderBy: PropTypes.string.isRequired, rowCount: PropTypes.number.isRequired, }; const useToolbarStyles = makeStyles((theme) => ({ root: { paddingLeft: theme.spacing(2), paddingRight: theme.spacing(1), }, highlight: theme.palette.type === "light" ? { color: theme.palette.secondary.main, backgroundColor: lighten(theme.palette.secondary.light, 0.85), } : { color: theme.palette.text.primary, backgroundColor: theme.palette.secondary.dark, }, title: { flex: "1 1 100%", }, })); const EnhancedTableToolbar = (props) => { const classes = useToolbarStyles(); const { numSelected } = props; return ( 0, })} > {numSelected > 0 ? {numSelected} selected : null} ); }; EnhancedTableToolbar.propTypes = { numSelected: PropTypes.number.isRequired, }; const useStyles = makeStyles((theme) => ({ root: { width: "100%", }, paper: { width: "100%", marginBottom: theme.spacing(2), }, table: { minWidth: 750, }, visuallyHidden: { border: 0, clip: "rect(0 0 0 0)", height: 1, margin: -1, overflow: "hidden", padding: 0, position: "absolute", top: 20, width: 1, }, })); export default function EnhancedTable() { const classes = useStyles(); const [order, setOrder] = React.useState("asc"); const [orderBy, setOrderBy] = React.useState("calories"); const [selected, setSelected] = React.useState([]); const [page, setPage] = React.useState(0); const [dense, setDense] = React.useState(false); const [rowsPerPage, setRowsPerPage] = React.useState(5); const handleRequestSort = (event, property) => { const isAsc = orderBy === property && order === "asc"; setOrder(isAsc ? "desc" : "asc"); setOrderBy(property); }; const handleSelectAllClick = (event) => { if (event.target.checked) { const newSelecteds = rows.map((n) => n.name); setSelected(newSelecteds); return; } setSelected([]); }; const handleClick = (event, name) => { const selectedIndex = selected.indexOf(name); let newSelected = []; if (selectedIndex === -1) { newSelected = newSelected.concat(selected, name); } else if (selectedIndex === 0) { newSelected = newSelected.concat(selected.slice(1)); } else if (selectedIndex === selected.length - 1) { newSelected = newSelected.concat(selected.slice(0, -1)); } else if (selectedIndex > 0) { newSelected = newSelected.concat( selected.slice(0, selectedIndex), selected.slice(selectedIndex + 1) ); } setSelected(newSelected); }; const handleChangePage = (event, newPage) => { setPage(newPage); }; const handleChangeRowsPerPage = (event) => { setRowsPerPage(parseInt(event.target.value, 10)); setPage(0); }; const handleChangeDense = (event) => { setDense(event.target.checked); }; const isSelected = (name) => selected.indexOf(name) !== -1; const emptyRows = rowsPerPage - Math.min(rowsPerPage, rows.length - page * rowsPerPage); return (
    Native MUI Table {stableSort(rows, getComparator(order, orderBy)) .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) .map((row, index) => { const isItemSelected = isSelected(row.name); const labelId = `enhanced-table-checkbox-${index}`; return ( handleClick(event, row.name)} role="checkbox" aria-checked={isItemSelected} tabIndex={-1} key={row.name} selected={isItemSelected} > {row.name} {row.calories} {row.fat} {row.carbs} {row.protein} ); })} {emptyRows > 0 && ( )}
    } label="Dense padding" />
    ); } ================================================ FILE: ui/src/pages/kitchensink/Examples.jsx ================================================ export default function Examples() { return null; } ================================================ FILE: ui/src/pages/kitchensink/Gantt.jsx ================================================ import React, { Component } from "react"; import Timeline from "react-vis-timeline-2"; import moment from "moment"; import { Paper } from "../../components"; function createItem(id, startTime) { return { id: id, group: id, content: "item " + id, start: startTime, end: startTime.clone().add(1, "minute"), }; } const initialGroups = [], initialItems = []; const now = moment().minutes(0).seconds(0).milliseconds(0); const itemCount = 20; for (let i = 0; i < itemCount; i++) { const start = now.clone().add(Math.random() * 200, "minutes"); initialGroups.push({ id: i, content: "group " + i }); initialItems.push(createItem(i, start)); } export default class Gantt extends Component { timelineRef = React.createRef(); constructor(props) { super(props); this.state = { selectedIds: [], }; } /* onAddItem = () => { var nextId = this.timelineRef.current.items.length + 1; const group = Math.floor(Math.random() * groupCount); this.timelineRef.current.items.add(createItem(nextId, group, names[group], moment())); this.timelineRef.current.timeline.fit(); }; */ onFit = () => { this.timelineRef.current.timeline.fit(); }; render() { return (

    This example demonstrate using groups.


     ); } clickHandler = () => { const { group } = this.props; var items = this.timelineRef.current.items.get(); const selectedIds = items .filter((item) => item.group === group) .map((item) => item.id); this.setState({ selectedIds, }); }; } ================================================ FILE: ui/src/pages/kitchensink/KitchenSink.jsx ================================================ import React, { useState } from "react"; import { Form, Formik } from "formik"; import { Checkbox, Grid, Switch, MenuItem, InputLabel, FormControl, IconButton, Toolbar, } from "@material-ui/core"; import DeleteIcon from "@material-ui/icons/Delete"; import { PrimaryButton, SecondaryButton, TertiaryButton, ButtonGroup, SplitButton, DropdownButton, Paper, Tab, Tabs, NavLink, Heading, Text, Input, Select, Button, } from "../../components"; import ZoomInIcon from "@material-ui/icons/ZoomIn"; import * as Yup from "yup"; import EnhancedTable from "./EnhancedTable"; import DataTableDemo from "./DataTableDemo"; import sharedStyles from "../styles"; import { makeStyles } from "@material-ui/styles"; import clsx from "clsx"; import FormikInput from "../../components/formik/FormikInput"; import FormikJsonInput from "../../components/formik/FormikJsonInput"; import Dropdown from "./Dropdown"; const useStyles = makeStyles(sharedStyles); export default function KitchenSink() { const classes = useStyles(); return (

    This is a Hawkins-like theme based on vanilla Material-UI.

     Gantt
    ); } const FormikSection = () => { const [formState, setFormState] = useState(); return ( Formik setFormState(values)} > 
    {JSON.stringify(formState)}
         ); }; const ToolbarSection = () => { return ( Toolbar Label {" "} ); }; const HeadingSection = () => { return ( Heading Level Zero Heading Level One Heading Level Two Heading Level Three Heading Level Four Heading Level Five Text Level Zero Text Level One Text Level Two
    Default <div>
    Default <p>
     ); }; const TabsSection = () => { const [tabIndex, setTabIndex] = useState(0); return ( Tabs Page Level Full Width setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> setTabIndex(3)} />
    Tab content {tabIndex}
     Fixed Width setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> setTabIndex(3)} />
    Tab content {tabIndex}
     Contextual Full Width setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> setTabIndex(3)} />
    Tab content {tabIndex}
     Fixed Width setTabIndex(0)} /> setTabIndex(1)} /> setTabIndex(2)} /> setTabIndex(3)} />
    Tab content {tabIndex}
         ); }; const Buttons = () => ( Button Primary Secondary Tertiary alert("you clicked 1"), }, { label: "Squash and merge", handler: () => alert("you clicked 2"), }, { label: "Rebase and merge", handler: () => alert("you clicked 3"), }, ]} onPrimaryClick={() => alert("main button")} > Split Button alert("you clicked 1"), }, { label: "Squash and merge", handler: () => alert("you clicked 2"), }, { label: "Rebase and merge", handler: () => alert("you clicked 3"), }, ]} > Dropdown Button ); const Toggles = () => { const [toggleChecked, setToggleChecked] = useState(false); return ( Toggle setToggleChecked(!toggleChecked)} color="primary" /> ); }; const Checkboxes = () => { const [toggleChecked, setToggleChecked] = useState(false); return ( Checkbox setToggleChecked(!toggleChecked)} color="primary" /> ); }; const Inputs = () => ( Input Input Label via FormControl/InputLabel ); ================================================ FILE: ui/src/pages/kitchensink/sampleMovieData.js ================================================ // Author https://github.com/yegor-sytnyk/movies-list export default [ { id: 1, title: "Beetlejuice", year: "1988", runtime: "92", genres: ["Comedy", "Fantasy"], director: "Tim Burton", actors: "Alec Baldwin, Geena Davis, Annie McEnroe, Maurice Page", plot: 'A couple of recently deceased ghosts contract the services of a "bio-exorcist" in order to remove the obnoxious new owners of their house.', posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTUwODE3MDE0MV5BMl5BanBnXkFtZTgwNTk1MjI4MzE@._V1_SX300.jpg", }, { id: 2, title: "The Cotton Club", year: "1984", runtime: "127", genres: ["Crime", "Drama", "Music"], director: "Francis Ford Coppola", actors: "Richard Gere, Gregory Hines, Diane Lane, Lonette McKee", plot: "The Cotton Club was a famous night club in Harlem. The story follows the people that visited the club, those that ran it, and is peppered with the Jazz music that made it so famous.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTU5ODAyNzA4OV5BMl5BanBnXkFtZTcwNzYwNTIzNA@@._V1_SX300.jpg", }, { id: 3, title: "The Shawshank Redemption", year: "1994", runtime: "142", genres: ["Crime", "Drama"], director: "Frank Darabont", actors: "Tim Robbins, Morgan Freeman, Bob Gunton, William Sadler", plot: "Two imprisoned men bond over a number of years, finding solace and eventual redemption through acts of common decency.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BODU4MjU4NjIwNl5BMl5BanBnXkFtZTgwMDU2MjEyMDE@._V1_SX300.jpg", }, { id: 4, title: "Crocodile Dundee", year: "1986", runtime: "97", genres: ["Adventure", "Comedy"], director: "Peter Faiman", actors: "Paul Hogan, Linda Kozlowski, John Meillon, David Gulpilil", plot: "An American reporter goes to the Australian outback to meet an eccentric crocodile poacher and invites him to New York City.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTg0MTU1MTg4NF5BMl5BanBnXkFtZTgwMDgzNzYxMTE@._V1_SX300.jpg", }, { id: 5, title: "Valkyrie", year: "2008", runtime: "121", genres: ["Drama", "History", "Thriller"], director: "Bryan Singer", actors: "Tom Cruise, Kenneth Branagh, Bill Nighy, Tom Wilkinson", plot: "A dramatization of the 20 July assassination and political coup plot by desperate renegade German Army officers against Hitler during World War II.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTg3Njc2ODEyN15BMl5BanBnXkFtZTcwNTAwMzc3NA@@._V1_SX300.jpg", }, { id: 6, title: "Ratatouille", year: "2007", runtime: "111", genres: ["Animation", "Comedy", "Family"], director: "Brad Bird, Jan Pinkava", actors: "Patton Oswalt, Ian Holm, Lou Romano, Brian Dennehy", plot: "A rat who can cook makes an unusual alliance with a young kitchen worker at a famous restaurant.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTMzODU0NTkxMF5BMl5BanBnXkFtZTcwMjQ4MzMzMw@@._V1_SX300.jpg", }, { id: 7, title: "City of God", year: "2002", runtime: "130", genres: ["Crime", "Drama"], director: "Fernando Meirelles, Kátia Lund", actors: "Alexandre Rodrigues, Leandro Firmino, Phellipe Haagensen, Douglas Silva", plot: "Two boys growing up in a violent neighborhood of Rio de Janeiro take different paths: one becomes a photographer, the other a drug dealer.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA4ODQ3ODkzNV5BMl5BanBnXkFtZTYwOTc4NDI3._V1_SX300.jpg", }, { id: 8, title: "Memento", year: "2000", runtime: "113", genres: ["Mystery", "Thriller"], director: "Christopher Nolan", actors: "Guy Pearce, Carrie-Anne Moss, Joe Pantoliano, Mark Boone Junior", plot: "A man juggles searching for his wife's murderer and keeping his short-term memory loss from being an obstacle.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNThiYjM3MzktMDg3Yy00ZWQ3LTk3YWEtN2M0YmNmNWEwYTE3XkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 9, title: "The Intouchables", year: "2011", runtime: "112", genres: ["Biography", "Comedy", "Drama"], director: "Olivier Nakache, Eric Toledano", actors: "François Cluzet, Omar Sy, Anne Le Ny, Audrey Fleurot", plot: "After he becomes a quadriplegic from a paragliding accident, an aristocrat hires a young man from the projects to be his caregiver.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTYxNDA3MDQwNl5BMl5BanBnXkFtZTcwNTU4Mzc1Nw@@._V1_SX300.jpg", }, { id: 10, title: "Stardust", year: "2007", runtime: "127", genres: ["Adventure", "Family", "Fantasy"], director: "Matthew Vaughn", actors: "Ian McKellen, Bimbo Hart, Alastair MacIntosh, David Kelly", plot: "In a countryside town bordering on a magical land, a young man makes a promise to his beloved that he'll retrieve a fallen star by venturing into the magical realm.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjkyMTE1OTYwNF5BMl5BanBnXkFtZTcwMDIxODYzMw@@._V1_SX300.jpg", }, { id: 11, title: "Apocalypto", year: "2006", runtime: "139", genres: ["Action", "Adventure", "Drama"], director: "Mel Gibson", actors: "Rudy Youngblood, Dalia Hernández, Jonathan Brewer, Morris Birdyellowhead", plot: "As the Mayan kingdom faces its decline, the rulers insist the key to prosperity is to build more temples and offer human sacrifices. Jaguar Paw, a young man captured for sacrifice, flees to avoid his fate.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNTM1NjYyNTY5OV5BMl5BanBnXkFtZTcwMjgwNTMzMQ@@._V1_SX300.jpg", }, { id: 12, title: "Taxi Driver", year: "1976", runtime: "113", genres: ["Crime", "Drama"], director: "Martin Scorsese", actors: "Diahnne Abbott, Frank Adu, Victor Argo, Gino Ardito", plot: "A mentally unstable Vietnam War veteran works as a night-time taxi driver in New York City where the perceived decadence and sleaze feeds his urge for violent action, attempting to save a preadolescent prostitute in the process.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNGQxNDgzZWQtZTNjNi00M2RkLWExZmEtNmE1NjEyZDEwMzA5XkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 13, title: "No Country for Old Men", year: "2007", runtime: "122", genres: ["Crime", "Drama", "Thriller"], director: "Ethan Coen, Joel Coen", actors: "Tommy Lee Jones, Javier Bardem, Josh Brolin, Woody Harrelson", plot: "Violence and mayhem ensue after a hunter stumbles upon a drug deal gone wrong and more than two million dollars in cash near the Rio Grande.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA5Njk3MjM4OV5BMl5BanBnXkFtZTcwMTc5MTE1MQ@@._V1_SX300.jpg", }, { id: 14, title: "Planet 51", year: "2009", runtime: "91", genres: ["Animation", "Adventure", "Comedy"], director: "Jorge Blanco, Javier Abad, Marcos Martínez", actors: "Jessica Biel, John Cleese, Gary Oldman, Dwayne Johnson", plot: "An alien civilization is invaded by Astronaut Chuck Baker, who believes that the planet was uninhabited. Wanted by the military, Baker must get back to his ship before it goes into orbit without him.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTUyOTAyNTA5Ml5BMl5BanBnXkFtZTcwODU2OTM0Mg@@._V1_SX300.jpg", }, { id: 15, title: "Looper", year: "2012", runtime: "119", genres: ["Action", "Crime", "Drama"], director: "Rian Johnson", actors: "Joseph Gordon-Levitt, Bruce Willis, Emily Blunt, Paul Dano", plot: "In 2074, when the mob wants to get rid of someone, the target is sent into the past, where a hired gun awaits - someone like Joe - who one day learns the mob wants to 'close the loop' by sending back Joe's future self for assassination.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTY3NTY0MjEwNV5BMl5BanBnXkFtZTcwNTE3NDA1OA@@._V1_SX300.jpg", }, { id: 16, title: "Corpse Bride", year: "2005", runtime: "77", genres: ["Animation", "Drama", "Family"], director: "Tim Burton, Mike Johnson", actors: "Johnny Depp, Helena Bonham Carter, Emily Watson, Tracey Ullman", plot: "When a shy groom practices his wedding vows in the inadvertent presence of a deceased young woman, she rises from the grave assuming he has married her.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTk1MTY1NjU4MF5BMl5BanBnXkFtZTcwNjIzMTEzMw@@._V1_SX300.jpg", }, { id: 17, title: "The Third Man", year: "1949", runtime: "93", genres: ["Film-Noir", "Mystery", "Thriller"], director: "Carol Reed", actors: "Joseph Cotten, Alida Valli, Orson Welles, Trevor Howard", plot: "Pulp novelist Holly Martins travels to shadowy, postwar Vienna, only to find himself investigating the mysterious death of an old friend, Harry Lime.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjMwNzMzMTQ0Ml5BMl5BanBnXkFtZTgwNjExMzUwNjE@._V1_SX300.jpg", }, { id: 18, title: "The Beach", year: "2000", runtime: "119", genres: ["Adventure", "Drama", "Romance"], director: "Danny Boyle", actors: "Leonardo DiCaprio, Daniel York, Patcharawan Patarakijjanon, Virginie Ledoyen", plot: "Twenty-something Richard travels to Thailand and finds himself in possession of a strange map. Rumours state that it leads to a solitary beach paradise, a tropical bliss - excited and intrigued, he sets out to find it.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BN2ViYTFiZmUtOTIxZi00YzIxLWEyMzUtYjQwZGNjMjNhY2IwXkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 19, title: "Scarface", year: "1983", runtime: "170", genres: ["Crime", "Drama"], director: "Brian De Palma", actors: "Al Pacino, Steven Bauer, Michelle Pfeiffer, Mary Elizabeth Mastrantonio", plot: "In Miami in 1980, a determined Cuban immigrant takes over a drug cartel and succumbs to greed.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjAzOTM4MzEwNl5BMl5BanBnXkFtZTgwMzU1OTc1MDE@._V1_SX300.jpg", }, { id: 20, title: "Sid and Nancy", year: "1986", runtime: "112", genres: ["Biography", "Drama", "Music"], director: "Alex Cox", actors: "Gary Oldman, Chloe Webb, David Hayman, Debby Bishop", plot: "Morbid biographical story of Sid Vicious, bassist with British punk group the Sex Pistols, and his girlfriend Nancy Spungen. When the Sex Pistols break up after their fateful US tour, ...", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjExNjA5NzY4M15BMl5BanBnXkFtZTcwNjQ2NzI5NA@@._V1_SX300.jpg", }, { id: 21, title: "Black Swan", year: "2010", runtime: "108", genres: ["Drama", "Thriller"], director: "Darren Aronofsky", actors: "Natalie Portman, Mila Kunis, Vincent Cassel, Barbara Hershey", plot: 'A committed dancer wins the lead role in a production of Tchaikovsky\'s "Swan Lake" only to find herself struggling to maintain her sanity.', posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNzY2NzI4OTE5MF5BMl5BanBnXkFtZTcwMjMyNDY4Mw@@._V1_SX300.jpg", }, { id: 22, title: "Inception", year: "2010", runtime: "148", genres: ["Action", "Adventure", "Sci-Fi"], director: "Christopher Nolan", actors: "Leonardo DiCaprio, Joseph Gordon-Levitt, Ellen Page, Tom Hardy", plot: "A thief, who steals corporate secrets through use of dream-sharing technology, is given the inverse task of planting an idea into the mind of a CEO.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjAxMzY3NjcxNF5BMl5BanBnXkFtZTcwNTI5OTM0Mw@@._V1_SX300.jpg", }, { id: 23, title: "The Deer Hunter", year: "1978", runtime: "183", genres: ["Drama", "War"], director: "Michael Cimino", actors: "Robert De Niro, John Cazale, John Savage, Christopher Walken", plot: "An in-depth examination of the ways in which the U.S. Vietnam War impacts and disrupts the lives of people in a small industrial town in Pennsylvania.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTYzYmRmZTQtYjk2NS00MDdlLTkxMDAtMTE2YTM2ZmNlMTBkXkEyXkFqcGdeQXVyNjU0OTQ0OTY@._V1_SX300.jpg", }, { id: 24, title: "Chasing Amy", year: "1997", runtime: "113", genres: ["Comedy", "Drama", "Romance"], director: "Kevin Smith", actors: "Ethan Suplee, Ben Affleck, Scott Mosier, Jason Lee", plot: "Holden and Banky are comic book artists. Everything's going good for them until they meet Alyssa, also a comic book artist. Holden falls for her, but his hopes are crushed when he finds out she's gay.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BZDM3MTg2MGUtZDM0MC00NzMwLWE5NjItOWFjNjA2M2I4YzgxXkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 25, title: "Django Unchained", year: "2012", runtime: "165", genres: ["Drama", "Western"], director: "Quentin Tarantino", actors: "Jamie Foxx, Christoph Waltz, Leonardo DiCaprio, Kerry Washington", plot: "With the help of a German bounty hunter, a freed slave sets out to rescue his wife from a brutal Mississippi plantation owner.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjIyNTQ5NjQ1OV5BMl5BanBnXkFtZTcwODg1MDU4OA@@._V1_SX300.jpg", }, { id: 26, title: "The Silence of the Lambs", year: "1991", runtime: "118", genres: ["Crime", "Drama", "Thriller"], director: "Jonathan Demme", actors: "Jodie Foster, Lawrence A. Bonney, Kasi Lemmons, Lawrence T. Wrentz", plot: "A young F.B.I. cadet must confide in an incarcerated and manipulative killer to receive his help on catching another serial killer who skins his victims.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQ2NzkzMDI4OF5BMl5BanBnXkFtZTcwMDA0NzE1NA@@._V1_SX300.jpg", }, { id: 27, title: "American Beauty", year: "1999", runtime: "122", genres: ["Drama", "Romance"], director: "Sam Mendes", actors: "Kevin Spacey, Annette Bening, Thora Birch, Wes Bentley", plot: "A sexually frustrated suburban father has a mid-life crisis after becoming infatuated with his daughter's best friend.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjM4NTI5NzYyNV5BMl5BanBnXkFtZTgwNTkxNTYxMTE@._V1_SX300.jpg", }, { id: 28, title: "Snatch", year: "2000", runtime: "102", genres: ["Comedy", "Crime"], director: "Guy Ritchie", actors: "Benicio Del Toro, Dennis Farina, Vinnie Jones, Brad Pitt", plot: "Unscrupulous boxing promoters, violent bookmakers, a Russian gangster, incompetent amateur robbers, and supposedly Jewish jewelers fight to track down a priceless stolen diamond.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTA2NDYxOGYtYjU1Mi00Y2QzLTgxMTQtMWI1MGI0ZGQ5MmU4XkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 29, title: "Midnight Express", year: "1978", runtime: "121", genres: ["Crime", "Drama", "Thriller"], director: "Alan Parker", actors: "Brad Davis, Irene Miracle, Bo Hopkins, Paolo Bonacelli", plot: "Billy Hayes, an American college student, is caught smuggling drugs out of Turkey and thrown into prison.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQyMDA5MzkyOF5BMl5BanBnXkFtZTgwOTYwNTcxMTE@._V1_SX300.jpg", }, { id: 30, title: "Pulp Fiction", year: "1994", runtime: "154", genres: ["Crime", "Drama"], director: "Quentin Tarantino", actors: "Tim Roth, Amanda Plummer, Laura Lovelace, John Travolta", plot: "The lives of two mob hit men, a boxer, a gangster's wife, and a pair of diner bandits intertwine in four tales of violence and redemption.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTkxMTA5OTAzMl5BMl5BanBnXkFtZTgwNjA5MDc3NjE@._V1_SX300.jpg", }, { id: 31, title: "Lock, Stock and Two Smoking Barrels", year: "1998", runtime: "107", genres: ["Comedy", "Crime"], director: "Guy Ritchie", actors: "Jason Flemyng, Dexter Fletcher, Nick Moran, Jason Statham", plot: "A botched card game in London triggers four friends, thugs, weed-growers, hard gangsters, loan sharks and debt collectors to collide with each other in a series of unexpected events, all for the sake of weed, cash and two antique shotguns.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTAyN2JmZmEtNjAyMy00NzYwLThmY2MtYWQ3OGNhNjExMmM4XkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 32, title: "Lucky Number Slevin", year: "2006", runtime: "110", genres: ["Crime", "Drama", "Mystery"], director: "Paul McGuigan", actors: "Josh Hartnett, Bruce Willis, Lucy Liu, Morgan Freeman", plot: "A case of mistaken identity lands Slevin into the middle of a war being plotted by two of the city's most rival crime bosses: The Rabbi and The Boss. Slevin is under constant surveillance by relentless Detective Brikowski as well as the infamous assassin Goodkat and finds himself having to hatch his own ingenious plot to get them before they get him.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMzc1OTEwMTk4OF5BMl5BanBnXkFtZTcwMTEzMDQzMQ@@._V1_SX300.jpg", }, { id: 33, title: "Rear Window", year: "1954", runtime: "112", genres: ["Mystery", "Thriller"], director: "Alfred Hitchcock", actors: "James Stewart, Grace Kelly, Wendell Corey, Thelma Ritter", plot: "A wheelchair-bound photographer spies on his neighbours from his apartment window and becomes convinced one of them has committed murder.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNGUxYWM3M2MtMGM3Mi00ZmRiLWE0NGQtZjE5ODI2OTJhNTU0XkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 34, title: "Pan's Labyrinth", year: "2006", runtime: "118", genres: ["Drama", "Fantasy", "War"], director: "Guillermo del Toro", actors: "Ivana Baquero, Sergi López, Maribel Verdú, Doug Jones", plot: "In the falangist Spain of 1944, the bookish young stepdaughter of a sadistic army officer escapes into an eerie but captivating fantasy world.", posterUrl: "", }, { id: 35, title: "Shutter Island", year: "2010", runtime: "138", genres: ["Mystery", "Thriller"], director: "Martin Scorsese", actors: "Leonardo DiCaprio, Mark Ruffalo, Ben Kingsley, Max von Sydow", plot: "In 1954, a U.S. marshal investigates the disappearance of a murderess who escaped from a hospital for the criminally insane.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTMxMTIyNzMxMV5BMl5BanBnXkFtZTcwOTc4OTI3Mg@@._V1_SX300.jpg", }, { id: 36, title: "Reservoir Dogs", year: "1992", runtime: "99", genres: ["Crime", "Drama", "Thriller"], director: "Quentin Tarantino", actors: "Harvey Keitel, Tim Roth, Michael Madsen, Chris Penn", plot: "After a simple jewelry heist goes terribly wrong, the surviving criminals begin to suspect that one of them is a police informant.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNjE5ZDJiZTQtOGE2YS00ZTc5LTk0OGUtOTg2NjdjZmVlYzE2XkEyXkFqcGdeQXVyMzM4MjM0Nzg@._V1_SX300.jpg", }, { id: 37, title: "The Shining", year: "1980", runtime: "146", genres: ["Drama", "Horror"], director: "Stanley Kubrick", actors: "Jack Nicholson, Shelley Duvall, Danny Lloyd, Scatman Crothers", plot: "A family heads to an isolated hotel for the winter where an evil and spiritual presence influences the father into violence, while his psychic son sees horrific forebodings from the past and of the future.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BODMxMjE3NTA4Ml5BMl5BanBnXkFtZTgwNDc0NTIxMDE@._V1_SX300.jpg", }, { id: 38, title: "Midnight in Paris", year: "2011", runtime: "94", genres: ["Comedy", "Fantasy", "Romance"], director: "Woody Allen", actors: "Owen Wilson, Rachel McAdams, Kurt Fuller, Mimi Kennedy", plot: "While on a trip to Paris with his fiancée's family, a nostalgic screenwriter finds himself mysteriously going back to the 1920s everyday at midnight.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTM4NjY1MDQwMl5BMl5BanBnXkFtZTcwNTI3Njg3NA@@._V1_SX300.jpg", }, { id: 39, title: "Les Misérables", year: "2012", runtime: "158", genres: ["Drama", "Musical", "Romance"], director: "Tom Hooper", actors: "Hugh Jackman, Russell Crowe, Anne Hathaway, Amanda Seyfried", plot: "In 19th-century France, Jean Valjean, who for decades has been hunted by the ruthless policeman Javert after breaking parole, agrees to care for a factory worker's daughter. The decision changes their lives forever.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTQ4NDI3NDg4M15BMl5BanBnXkFtZTcwMjY5OTI1OA@@._V1_SX300.jpg", }, { id: 40, title: "L.A. Confidential", year: "1997", runtime: "138", genres: ["Crime", "Drama", "Mystery"], director: "Curtis Hanson", actors: "Kevin Spacey, Russell Crowe, Guy Pearce, James Cromwell", plot: "As corruption grows in 1950s LA, three policemen - one strait-laced, one brutal, and one sleazy - investigate a series of murders with their own brand of justice.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNWEwNDhhNWUtYWMzNi00ZTNhLWFiZDAtMjBjZmJhMTU0ZTY2XkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 41, title: "Moneyball", year: "2011", runtime: "133", genres: ["Biography", "Drama", "Sport"], director: "Bennett Miller", actors: "Brad Pitt, Jonah Hill, Philip Seymour Hoffman, Robin Wright", plot: "Oakland A's general manager Billy Beane's successful attempt to assemble a baseball team on a lean budget by employing computer-generated analysis to acquire new players.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjAxOTU3Mzc1M15BMl5BanBnXkFtZTcwMzk1ODUzNg@@._V1_SX300.jpg", }, { id: 42, title: "The Hangover", year: "2009", runtime: "100", genres: ["Comedy"], director: "Todd Phillips", actors: "Bradley Cooper, Ed Helms, Zach Galifianakis, Justin Bartha", plot: "Three buddies wake up from a bachelor party in Las Vegas, with no memory of the previous night and the bachelor missing. They make their way around the city in order to find their friend before his wedding.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTU1MDA1MTYwMF5BMl5BanBnXkFtZTcwMDcxMzA1Mg@@._V1_SX300.jpg", }, { id: 43, title: "The Great Beauty", year: "2013", runtime: "141", genres: ["Drama"], director: "Paolo Sorrentino", actors: "Toni Servillo, Carlo Verdone, Sabrina Ferilli, Carlo Buccirosso", plot: "Jep Gambardella has seduced his way through the lavish nightlife of Rome for decades, but after his 65th birthday and a shock from the past, Jep looks past the nightclubs and parties to find a timeless landscape of absurd, exquisite beauty.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQ0ODg1OTQ2Nl5BMl5BanBnXkFtZTgwNTc2MDY1MDE@._V1_SX300.jpg", }, { id: 44, title: "Gran Torino", year: "2008", runtime: "116", genres: ["Drama"], director: "Clint Eastwood", actors: "Clint Eastwood, Christopher Carley, Bee Vang, Ahney Her", plot: "Disgruntled Korean War veteran Walt Kowalski sets out to reform his neighbor, a Hmong teenager who tried to steal Kowalski's prized possession: a 1972 Gran Torino.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTQyMTczMTAxMl5BMl5BanBnXkFtZTcwOTc1ODE0Mg@@._V1_SX300.jpg", }, { id: 45, title: "Mary and Max", year: "2009", runtime: "92", genres: ["Animation", "Comedy", "Drama"], director: "Adam Elliot", actors: "Toni Collette, Philip Seymour Hoffman, Barry Humphries, Eric Bana", plot: "A tale of friendship between two unlikely pen pals: Mary, a lonely, eight-year-old girl living in the suburbs of Melbourne, and Max, a forty-four-year old, severely obese man living in New York.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQ1NDIyNTA1Nl5BMl5BanBnXkFtZTcwMjc2Njk3OA@@._V1_SX300.jpg", }, { id: 46, title: "Flight", year: "2012", runtime: "138", genres: ["Drama", "Thriller"], director: "Robert Zemeckis", actors: "Nadine Velazquez, Denzel Washington, Carter Cabassa, Adam C. Edwards", plot: "An airline pilot saves almost all his passengers on his malfunctioning airliner which eventually crashed, but an investigation into the accident reveals something troubling.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTUxMjI1OTMxNl5BMl5BanBnXkFtZTcwNjc3NTY1OA@@._V1_SX300.jpg", }, { id: 47, title: "One Flew Over the Cuckoo's Nest", year: "1975", runtime: "133", genres: ["Drama"], director: "Milos Forman", actors: "Michael Berryman, Peter Brocco, Dean R. Brooks, Alonzo Brown", plot: "A criminal pleads insanity after getting into trouble again and once in the mental institution rebels against the oppressive nurse and rallies up the scared patients.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BYmJkODkwOTItZThjZC00MTE0LWIxNzQtYTM3MmQwMGI1OWFiXkEyXkFqcGdeQXVyNjUwNzk3NDc@._V1_SX300.jpg", }, { id: 48, title: "Requiem for a Dream", year: "2000", runtime: "102", genres: ["Drama"], director: "Darren Aronofsky", actors: "Ellen Burstyn, Jared Leto, Jennifer Connelly, Marlon Wayans", plot: "The drug-induced utopias of four Coney Island people are shattered when their addictions run deep.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTkzODMzODYwOF5BMl5BanBnXkFtZTcwODM2NjA2NQ@@._V1_SX300.jpg", }, { id: 49, title: "The Truman Show", year: "1998", runtime: "103", genres: ["Comedy", "Drama", "Sci-Fi"], director: "Peter Weir", actors: "Jim Carrey, Laura Linney, Noah Emmerich, Natascha McElhone", plot: "An insurance salesman/adjuster discovers his entire life is actually a television show.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMDIzODcyY2EtMmY2MC00ZWVlLTgwMzAtMjQwOWUyNmJjNTYyXkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 50, title: "The Artist", year: "2011", runtime: "100", genres: ["Comedy", "Drama", "Romance"], director: "Michel Hazanavicius", actors: "Jean Dujardin, Bérénice Bejo, John Goodman, James Cromwell", plot: "A silent movie star meets a young dancer, but the arrival of talking pictures sends their careers in opposite directions.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMzk0NzQxMTM0OV5BMl5BanBnXkFtZTcwMzU4MDYyNQ@@._V1_SX300.jpg", }, { id: 51, title: "Forrest Gump", year: "1994", runtime: "142", genres: ["Comedy", "Drama"], director: "Robert Zemeckis", actors: "Tom Hanks, Rebecca Williams, Sally Field, Michael Conner Humphreys", plot: "Forrest Gump, while not intelligent, has accidentally been present at many historic moments, but his true love, Jenny Curran, eludes him.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BYThjM2MwZGMtMzg3Ny00NGRkLWE4M2EtYTBiNWMzOTY0YTI4XkEyXkFqcGdeQXVyNDYyMDk5MTU@._V1_SX300.jpg", }, { id: 52, title: "The Hobbit: The Desolation of Smaug", year: "2013", runtime: "161", genres: ["Adventure", "Fantasy"], director: "Peter Jackson", actors: "Ian McKellen, Martin Freeman, Richard Armitage, Ken Stott", plot: "The dwarves, along with Bilbo Baggins and Gandalf the Grey, continue their quest to reclaim Erebor, their homeland, from Smaug. Bilbo Baggins is in possession of a mysterious and magical ring.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMzU0NDY0NDEzNV5BMl5BanBnXkFtZTgwOTIxNDU1MDE@._V1_SX300.jpg", }, { id: 53, title: "Vicky Cristina Barcelona", year: "2008", runtime: "96", genres: ["Drama", "Romance"], director: "Woody Allen", actors: "Rebecca Hall, Scarlett Johansson, Christopher Evan Welch, Chris Messina", plot: "Two girlfriends on a summer holiday in Spain become enamored with the same painter, unaware that his ex-wife, with whom he has a tempestuous relationship, is about to re-enter the picture.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTU2NDQ4MTg2MV5BMl5BanBnXkFtZTcwNDUzNjU3MQ@@._V1_SX300.jpg", }, { id: 54, title: "Slumdog Millionaire", year: "2008", runtime: "120", genres: ["Drama", "Romance"], director: "Danny Boyle, Loveleen Tandan", actors: "Dev Patel, Saurabh Shukla, Anil Kapoor, Rajendranath Zutshi", plot: 'A Mumbai teen reflects on his upbringing in the slums when he is accused of cheating on the Indian Version of "Who Wants to be a Millionaire?"', posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTU2NTA5NzI0N15BMl5BanBnXkFtZTcwMjUxMjYxMg@@._V1_SX300.jpg", }, { id: 55, title: "Lost in Translation", year: "2003", runtime: "101", genres: ["Drama"], director: "Sofia Coppola", actors: "Scarlett Johansson, Bill Murray, Akiko Takeshita, Kazuyoshi Minamimagoe", plot: "A faded movie star and a neglected young woman form an unlikely bond after crossing paths in Tokyo.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTI2NDI5ODk4N15BMl5BanBnXkFtZTYwMTI3NTE3._V1_SX300.jpg", }, { id: 56, title: "Match Point", year: "2005", runtime: "119", genres: ["Drama", "Romance", "Thriller"], director: "Woody Allen", actors: "Jonathan Rhys Meyers, Alexander Armstrong, Paul Kaye, Matthew Goode", plot: "At a turning point in his life, a former tennis pro falls for an actress who happens to be dating his friend and soon-to-be brother-in-law.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTMzNzY4MzE5NF5BMl5BanBnXkFtZTcwMzQ1MDMzMQ@@._V1_SX300.jpg", }, { id: 57, title: "Psycho", year: "1960", runtime: "109", genres: ["Horror", "Mystery", "Thriller"], director: "Alfred Hitchcock", actors: "Anthony Perkins, Vera Miles, John Gavin, Janet Leigh", plot: "A Phoenix secretary embezzles $40,000 from her employer's client, goes on the run, and checks into a remote motel run by a young man under the domination of his mother.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMDI3OWRmOTEtOWJhYi00N2JkLTgwNGItMjdkN2U0NjFiZTYwXkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 58, title: "North by Northwest", year: "1959", runtime: "136", genres: ["Action", "Adventure", "Crime"], director: "Alfred Hitchcock", actors: "Cary Grant, Eva Marie Saint, James Mason, Jessie Royce Landis", plot: "A hapless New York advertising executive is mistaken for a government agent by a group of foreign spies, and is pursued across the country while he looks for a way to survive.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjQwMTQ0MzgwNl5BMl5BanBnXkFtZTgwNjc4ODE4MzE@._V1_SX300.jpg", }, { id: 59, title: "Madagascar: Escape 2 Africa", year: "2008", runtime: "89", genres: ["Animation", "Action", "Adventure"], director: "Eric Darnell, Tom McGrath", actors: "Ben Stiller, Chris Rock, David Schwimmer, Jada Pinkett Smith", plot: "The animals try to fly back to New York City, but crash-land on an African wildlife refuge, where Alex is reunited with his parents.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjExMDA4NDcwMl5BMl5BanBnXkFtZTcwODAxNTQ3MQ@@._V1_SX300.jpg", }, { id: 60, title: "Despicable Me 2", year: "2013", runtime: "98", genres: ["Animation", "Adventure", "Comedy"], director: "Pierre Coffin, Chris Renaud", actors: "Steve Carell, Kristen Wiig, Benjamin Bratt, Miranda Cosgrove", plot: "When Gru, the world's most super-bad turned super-dad has been recruited by a team of officials to stop lethal muscle and a host of Gru's own, He has to fight back with new gadgetry, cars, and more minion madness.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjExNjAyNTcyMF5BMl5BanBnXkFtZTgwODQzMjQ3MDE@._V1_SX300.jpg", }, { id: 61, title: "Downfall", year: "2004", runtime: "156", genres: ["Biography", "Drama", "History"], director: "Oliver Hirschbiegel", actors: "Bruno Ganz, Alexandra Maria Lara, Corinna Harfouch, Ulrich Matthes", plot: "Traudl Junge, the final secretary for Adolf Hitler, tells of the Nazi dictator's final days in his Berlin bunker at the end of WWII.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM1OTI1MjE2Nl5BMl5BanBnXkFtZTcwMTEwMzc4NA@@._V1_SX300.jpg", }, { id: 62, title: "Madagascar", year: "2005", runtime: "86", genres: ["Animation", "Adventure", "Comedy"], director: "Eric Darnell, Tom McGrath", actors: "Ben Stiller, Chris Rock, David Schwimmer, Jada Pinkett Smith", plot: "Spoiled by their upbringing with no idea what wild life is really like, four animals from New York Central Zoo escape, unwittingly assisted by four absconding penguins, and find themselves in Madagascar, among a bunch of merry lemurs", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY4NDUwMzQxMF5BMl5BanBnXkFtZTcwMDgwNjgyMQ@@._V1_SX300.jpg", }, { id: 63, title: "Madagascar 3: Europe's Most Wanted", year: "2012", runtime: "93", genres: ["Animation", "Adventure", "Comedy"], director: "Eric Darnell, Tom McGrath, Conrad Vernon", actors: "Ben Stiller, Chris Rock, David Schwimmer, Jada Pinkett Smith", plot: "Alex, Marty, Gloria and Melman are still fighting to get home to their beloved Big Apple. Their journey takes them through Europe where they find the perfect cover: a traveling circus, which they reinvent - Madagascar style.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM2MTIzNzk2MF5BMl5BanBnXkFtZTcwMDcwMzQxNw@@._V1_SX300.jpg", }, { id: 64, title: "God Bless America", year: "2011", runtime: "105", genres: ["Comedy", "Crime"], director: "Bobcat Goldthwait", actors: "Joel Murray, Tara Lynne Barr, Melinda Page Hamilton, Mackenzie Brooke Smith", plot: "On a mission to rid society of its most repellent citizens, terminally ill Frank makes an unlikely accomplice in 16-year-old Roxy.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQwMTc1MzA4NF5BMl5BanBnXkFtZTcwNzQwMTgzNw@@._V1_SX300.jpg", }, { id: 65, title: "The Social Network", year: "2010", runtime: "120", genres: ["Biography", "Drama"], director: "David Fincher", actors: "Jesse Eisenberg, Rooney Mara, Bryan Barter, Dustin Fitzsimons", plot: "Harvard student Mark Zuckerberg creates the social networking site that would become known as Facebook, but is later sued by two brothers who claimed he stole their idea, and the co-founder who was later squeezed out of the business.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM2ODk0NDAwMF5BMl5BanBnXkFtZTcwNTM1MDc2Mw@@._V1_SX300.jpg", }, { id: 66, title: "The Pianist", year: "2002", runtime: "150", genres: ["Biography", "Drama", "War"], director: "Roman Polanski", actors: "Adrien Brody, Emilia Fox, Michal Zebrowski, Ed Stoppard", plot: "A Polish Jewish musician struggles to survive the destruction of the Warsaw ghetto of World War II.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTc4OTkyOTA3OF5BMl5BanBnXkFtZTYwMDIxNjk5._V1_SX300.jpg", }, { id: 67, title: "Alive", year: "1993", runtime: "120", genres: ["Adventure", "Biography", "Drama"], director: "Frank Marshall", actors: "Ethan Hawke, Vincent Spano, Josh Hamilton, Bruce Ramsay", plot: "Uruguayan rugby team stranded in the snow swept Andes are forced to use desperate measures to survive after a plane crash.", posterUrl: "", }, { id: 68, title: "Casablanca", year: "1942", runtime: "102", genres: ["Drama", "Romance", "War"], director: "Michael Curtiz", actors: "Humphrey Bogart, Ingrid Bergman, Paul Henreid, Claude Rains", plot: "In Casablanca, Morocco in December 1941, a cynical American expatriate meets a former lover, with unforeseen complications.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjQwNDYyNTk2N15BMl5BanBnXkFtZTgwMjQ0OTMyMjE@._V1_SX300.jpg", }, { id: 69, title: "American Gangster", year: "2007", runtime: "157", genres: ["Biography", "Crime", "Drama"], director: "Ridley Scott", actors: "Denzel Washington, Russell Crowe, Chiwetel Ejiofor, Josh Brolin", plot: "In 1970s America, a detective works to bring down the drug empire of Frank Lucas, a heroin kingpin from Manhattan, who is smuggling the drug into the country from the Far East.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTkyNzY5MDA5MV5BMl5BanBnXkFtZTcwMjg4MzI3MQ@@._V1_SX300.jpg", }, { id: 70, title: "Catch Me If You Can", year: "2002", runtime: "141", genres: ["Biography", "Crime", "Drama"], director: "Steven Spielberg", actors: "Leonardo DiCaprio, Tom Hanks, Christopher Walken, Martin Sheen", plot: "The true story of Frank Abagnale Jr. who, before his 19th birthday, successfully conned millions of dollars' worth of checks as a Pan Am pilot, doctor, and legal prosecutor.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY5MzYzNjc5NV5BMl5BanBnXkFtZTYwNTUyNTc2._V1_SX300.jpg", }, { id: 71, title: "American History X", year: "1998", runtime: "119", genres: ["Crime", "Drama"], director: "Tony Kaye", actors: "Edward Norton, Edward Furlong, Beverly D'Angelo, Jennifer Lien", plot: "A former neo-nazi skinhead tries to prevent his younger brother from going down the same wrong path that he did.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BZjA0MTM4MTQtNzY5MC00NzY3LWI1ZTgtYzcxMjkyMzU4MDZiXkEyXkFqcGdeQXVyNDYyMDk5MTU@._V1_SX300.jpg", }, { id: 72, title: "Casino", year: "1995", runtime: "178", genres: ["Biography", "Crime", "Drama"], director: "Martin Scorsese", actors: "Robert De Niro, Sharon Stone, Joe Pesci, James Woods", plot: "Greed, deception, money, power, and murder occur between two best friends, a mafia underboss and a casino owner, for a trophy wife over a gambling empire.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTcxOWYzNDYtYmM4YS00N2NkLTk0NTAtNjg1ODgwZjAxYzI3XkEyXkFqcGdeQXVyNTA4NzY1MzY@._V1_SX300.jpg", }, { id: 73, title: "Pirates of the Caribbean: At World's End", year: "2007", runtime: "169", genres: ["Action", "Adventure", "Fantasy"], director: "Gore Verbinski", actors: "Johnny Depp, Geoffrey Rush, Orlando Bloom, Keira Knightley", plot: "Captain Barbossa, Will Turner and Elizabeth Swann must sail off the edge of the map, navigate treachery and betrayal, find Jack Sparrow, and make their final alliances for one last decisive battle.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjIyNjkxNzEyMl5BMl5BanBnXkFtZTYwMjc3MDE3._V1_SX300.jpg", }, { id: 74, title: "Pirates of the Caribbean: On Stranger Tides", year: "2011", runtime: "136", genres: ["Action", "Adventure", "Fantasy"], director: "Rob Marshall", actors: "Johnny Depp, Penélope Cruz, Geoffrey Rush, Ian McShane", plot: "Jack Sparrow and Barbossa embark on a quest to find the elusive fountain of youth, only to discover that Blackbeard and his daughter are after it too.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjE5MjkwODI3Nl5BMl5BanBnXkFtZTcwNjcwMDk4NA@@._V1_SX300.jpg", }, { id: 75, title: "Crash", year: "2004", runtime: "112", genres: ["Crime", "Drama", "Thriller"], director: "Paul Haggis", actors: "Karina Arroyave, Dato Bakhtadze, Sandra Bullock, Don Cheadle", plot: "Los Angeles citizens with vastly separate lives collide in interweaving stories of race, loss and redemption.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BOTk1OTA1MjIyNV5BMl5BanBnXkFtZTcwODQxMTkyMQ@@._V1_SX300.jpg", }, { id: 76, title: "Pirates of the Caribbean: The Curse of the Black Pearl", year: "2003", runtime: "143", genres: ["Action", "Adventure", "Fantasy"], director: "Gore Verbinski", actors: "Johnny Depp, Geoffrey Rush, Orlando Bloom, Keira Knightley", plot: "Blacksmith Will Turner teams up with eccentric pirate \"Captain\" Jack Sparrow to save his love, the governor's daughter, from Jack's former pirate allies, who are now undead.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjAyNDM4MTc2N15BMl5BanBnXkFtZTYwNDk0Mjc3._V1_SX300.jpg", }, { id: 77, title: "The Lord of the Rings: The Return of the King", year: "2003", runtime: "201", genres: ["Action", "Adventure", "Drama"], director: "Peter Jackson", actors: "Noel Appleby, Ali Astin, Sean Astin, David Aston", plot: "Gandalf and Aragorn lead the World of Men against Sauron's army to draw his gaze from Frodo and Sam as they approach Mount Doom with the One Ring.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjE4MjA1NTAyMV5BMl5BanBnXkFtZTcwNzM1NDQyMQ@@._V1_SX300.jpg", }, { id: 78, title: "Oldboy", year: "2003", runtime: "120", genres: ["Drama", "Mystery", "Thriller"], director: "Chan-wook Park", actors: "Min-sik Choi, Ji-tae Yu, Hye-jeong Kang, Dae-han Ji", plot: "After being kidnapped and imprisoned for 15 years, Oh Dae-Su is released, only to find that he must find his captor in 5 days.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTI3NTQyMzU5M15BMl5BanBnXkFtZTcwMTM2MjgyMQ@@._V1_SX300.jpg", }, { id: 79, title: "Chocolat", year: "2000", runtime: "121", genres: ["Drama", "Romance"], director: "Lasse Hallström", actors: "Alfred Molina, Carrie-Anne Moss, Aurelien Parent Koenig, Antonio Gil", plot: "A woman and her daughter open a chocolate shop in a small French village that shakes up the rigid morality of the community.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA4MDI3NTQwMV5BMl5BanBnXkFtZTcwNjIzNDcyMQ@@._V1_SX300.jpg", }, { id: 80, title: "Casino Royale", year: "2006", runtime: "144", genres: ["Action", "Adventure", "Thriller"], director: "Martin Campbell", actors: "Daniel Craig, Eva Green, Mads Mikkelsen, Judi Dench", plot: "Armed with a licence to kill, Secret Agent James Bond sets out on his first mission as 007 and must defeat a weapons dealer in a high stakes game of poker at Casino Royale, but things are not what they seem.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM5MjI4NDExNF5BMl5BanBnXkFtZTcwMDM1MjMzMQ@@._V1_SX300.jpg", }, { id: 81, title: "WALL·E", year: "2008", runtime: "98", genres: ["Animation", "Adventure", "Family"], director: "Andrew Stanton", actors: "Ben Burtt, Elissa Knight, Jeff Garlin, Fred Willard", plot: "In the distant future, a small waste-collecting robot inadvertently embarks on a space journey that will ultimately decide the fate of mankind.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTczOTA3MzY2N15BMl5BanBnXkFtZTcwOTYwNjE2MQ@@._V1_SX300.jpg", }, { id: 82, title: "The Wolf of Wall Street", year: "2013", runtime: "180", genres: ["Biography", "Comedy", "Crime"], director: "Martin Scorsese", actors: "Leonardo DiCaprio, Jonah Hill, Margot Robbie, Matthew McConaughey", plot: "Based on the true story of Jordan Belfort, from his rise to a wealthy stock-broker living the high life to his fall involving crime, corruption and the federal government.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjIxMjgxNTk0MF5BMl5BanBnXkFtZTgwNjIyOTg2MDE@._V1_SX300.jpg", }, { id: 83, title: "Hellboy II: The Golden Army", year: "2008", runtime: "120", genres: ["Action", "Adventure", "Fantasy"], director: "Guillermo del Toro", actors: "Ron Perlman, Selma Blair, Doug Jones, John Alexander", plot: "The mythical world starts a rebellion against humanity in order to rule the Earth, so Hellboy and his team must save the world from the rebellious creatures.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA5NzgyMjc2Nl5BMl5BanBnXkFtZTcwOTU3MDI3MQ@@._V1_SX300.jpg", }, { id: 84, title: "Sunset Boulevard", year: "1950", runtime: "110", genres: ["Drama", "Film-Noir", "Romance"], director: "Billy Wilder", actors: "William Holden, Gloria Swanson, Erich von Stroheim, Nancy Olson", plot: "A hack screenwriter writes a screenplay for a former silent-film star who has faded into Hollywood obscurity.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTc3NDYzODAwNV5BMl5BanBnXkFtZTgwODg1MTczMTE@._V1_SX300.jpg", }, { id: 85, title: "I-See-You.Com", year: "2006", runtime: "92", genres: ["Comedy"], director: "Eric Steven Stahl", actors: "Beau Bridges, Rosanna Arquette, Mathew Botuchis, Shiri Appleby", plot: "A 17-year-old boy buys mini-cameras and displays the footage online at I-see-you.com. The cash rolls in as the site becomes a major hit. Everyone seems to have fun until it all comes crashing down....", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTYwMDUzNzA5Nl5BMl5BanBnXkFtZTcwMjQ2Njk3MQ@@._V1_SX300.jpg", }, { id: 86, title: "The Grand Budapest Hotel", year: "2014", runtime: "99", genres: ["Adventure", "Comedy", "Crime"], director: "Wes Anderson", actors: "Ralph Fiennes, F. Murray Abraham, Mathieu Amalric, Adrien Brody", plot: "The adventures of Gustave H, a legendary concierge at a famous hotel from the fictional Republic of Zubrowka between the first and second World Wars, and Zero Moustafa, the lobby boy who becomes his most trusted friend.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMzM5NjUxOTEyMl5BMl5BanBnXkFtZTgwNjEyMDM0MDE@._V1_SX300.jpg", }, { id: 87, title: "The Hitchhiker's Guide to the Galaxy", year: "2005", runtime: "109", genres: ["Adventure", "Comedy", "Sci-Fi"], director: "Garth Jennings", actors: "Bill Bailey, Anna Chancellor, Warwick Davis, Yasiin Bey", plot: 'Mere seconds before the Earth is to be demolished by an alien construction crew, journeyman Arthur Dent is swept off the planet by his friend Ford Prefect, a researcher penning a new edition of "The Hitchhiker\'s Guide to the Galaxy."', posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjEwOTk4NjU2MF5BMl5BanBnXkFtZTYwMDA3NzI3._V1_SX300.jpg", }, { id: 88, title: "Once Upon a Time in America", year: "1984", runtime: "229", genres: ["Crime", "Drama"], director: "Sergio Leone", actors: "Robert De Niro, James Woods, Elizabeth McGovern, Joe Pesci", plot: "A former Prohibition-era Jewish gangster returns to the Lower East Side of Manhattan over thirty years later, where he once again must confront the ghosts and regrets of his old life.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMGFkNWI4MTMtNGQ0OC00MWVmLTk3MTktOGYxN2Y2YWVkZWE2XkEyXkFqcGdeQXVyNjU0OTQ0OTY@._V1_SX300.jpg", }, { id: 89, title: "Oblivion", year: "2013", runtime: "124", genres: ["Action", "Adventure", "Mystery"], director: "Joseph Kosinski", actors: "Tom Cruise, Morgan Freeman, Olga Kurylenko, Andrea Riseborough", plot: "A veteran assigned to extract Earth's remaining resources begins to question what he knows about his mission and himself.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQwMDY0MTA4MF5BMl5BanBnXkFtZTcwNzI3MDgxOQ@@._V1_SX300.jpg", }, { id: 90, title: "V for Vendetta", year: "2005", runtime: "132", genres: ["Action", "Drama", "Thriller"], director: "James McTeigue", actors: "Natalie Portman, Hugo Weaving, Stephen Rea, Stephen Fry", plot: 'In a future British tyranny, a shadowy freedom fighter, known only by the alias of "V", plots to overthrow it with the help of a young woman.', posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BOTI5ODc3NzExNV5BMl5BanBnXkFtZTcwNzYxNzQzMw@@._V1_SX300.jpg", }, { id: 91, title: "Gattaca", year: "1997", runtime: "106", genres: ["Drama", "Sci-Fi", "Thriller"], director: "Andrew Niccol", actors: "Ethan Hawke, Uma Thurman, Gore Vidal, Xander Berkeley", plot: "A genetically inferior man assumes the identity of a superior one in order to pursue his lifelong dream of space travel.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNDQxOTc0MzMtZmRlOS00OWQ5LWI2ZDctOTAwNmMwOTYxYzlhXkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 92, title: "Silver Linings Playbook", year: "2012", runtime: "122", genres: ["Comedy", "Drama", "Romance"], director: "David O. Russell", actors: "Bradley Cooper, Jennifer Lawrence, Robert De Niro, Jacki Weaver", plot: "After a stint in a mental institution, former teacher Pat Solitano moves back in with his parents and tries to reconcile with his ex-wife. Things get more challenging when Pat meets Tiffany, a mysterious girl with problems of her own.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM2MTI5NzA3MF5BMl5BanBnXkFtZTcwODExNTc0OA@@._V1_SX300.jpg", }, { id: 93, title: "Alice in Wonderland", year: "2010", runtime: "108", genres: ["Adventure", "Family", "Fantasy"], director: "Tim Burton", actors: "Johnny Depp, Mia Wasikowska, Helena Bonham Carter, Anne Hathaway", plot: "Nineteen-year-old Alice returns to the magical world from her childhood adventure, where she reunites with her old friends and learns of her true destiny: to end the Red Queen's reign of terror.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTMwNjAxMTc0Nl5BMl5BanBnXkFtZTcwODc3ODk5Mg@@._V1_SX300.jpg", }, { id: 94, title: "Gandhi", year: "1982", runtime: "191", genres: ["Biography", "Drama"], director: "Richard Attenborough", actors: "Ben Kingsley, Candice Bergen, Edward Fox, John Gielgud", plot: "Gandhi's character is fully explained as a man of nonviolence. Through his patience, he is able to drive the British out of the subcontinent. And the stubborn nature of Jinnah and his commitment towards Pakistan is portrayed.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMzJiZDRmOWUtYjE2MS00Mjc1LTg1ZDYtNTQxYWJkZTg1OTM4XkEyXkFqcGdeQXVyNjUwNzk3NDc@._V1_SX300.jpg", }, { id: 95, title: "Pacific Rim", year: "2013", runtime: "131", genres: ["Action", "Adventure", "Sci-Fi"], director: "Guillermo del Toro", actors: "Charlie Hunnam, Diego Klattenhoff, Idris Elba, Rinko Kikuchi", plot: "As a war between humankind and monstrous sea creatures wages on, a former pilot and a trainee are paired up to drive a seemingly obsolete special weapon in a desperate effort to save the world from the apocalypse.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY3MTI5NjQ4Nl5BMl5BanBnXkFtZTcwOTU1OTU0OQ@@._V1_SX300.jpg", }, { id: 96, title: "Kiss Kiss Bang Bang", year: "2005", runtime: "103", genres: ["Comedy", "Crime", "Mystery"], director: "Shane Black", actors: "Robert Downey Jr., Val Kilmer, Michelle Monaghan, Corbin Bernsen", plot: "A murder mystery brings together a private eye, a struggling actress, and a thief masquerading as an actor.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTY5NDExMDA3M15BMl5BanBnXkFtZTYwNTc2MzA3._V1_SX300.jpg", }, { id: 97, title: "The Quiet American", year: "2002", runtime: "101", genres: ["Drama", "Mystery", "Romance"], director: "Phillip Noyce", actors: "Michael Caine, Brendan Fraser, Do Thi Hai Yen, Rade Serbedzija", plot: "An older British reporter vies with a young U.S. doctor for the affections of a beautiful Vietnamese woman.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjE2NTUxNTE3Nl5BMl5BanBnXkFtZTYwNTczMTg5._V1_SX300.jpg", }, { id: 98, title: "Cloud Atlas", year: "2012", runtime: "172", genres: ["Drama", "Sci-Fi"], director: "Tom Tykwer, Lana Wachowski, Lilly Wachowski", actors: "Tom Hanks, Halle Berry, Jim Broadbent, Hugo Weaving", plot: "An exploration of how the actions of individual lives impact one another in the past, present and future, as one soul is shaped from a killer into a hero, and an act of kindness ripples across centuries to inspire a revolution.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTczMTgxMjc4NF5BMl5BanBnXkFtZTcwNjM5MTA2OA@@._V1_SX300.jpg", }, { id: 99, title: "The Impossible", year: "2012", runtime: "114", genres: ["Drama", "Thriller"], director: "J.A. Bayona", actors: "Naomi Watts, Ewan McGregor, Tom Holland, Samuel Joslin", plot: "The story of a tourist family in Thailand caught in the destruction and chaotic aftermath of the 2004 Indian Ocean tsunami.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA5NTA3NzQ5Nl5BMl5BanBnXkFtZTcwOTYxNjY0OA@@._V1_SX300.jpg", }, { id: 100, title: "All Quiet on the Western Front", year: "1930", runtime: "136", genres: ["Drama", "War"], director: "Lewis Milestone", actors: "Louis Wolheim, Lew Ayres, John Wray, Arnold Lucy", plot: "A young soldier faces profound disillusionment in the soul-destroying horror of World War I.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNTM5OTg2NDY1NF5BMl5BanBnXkFtZTcwNTQ4MTMwNw@@._V1_SX300.jpg", }, { id: 101, title: "The English Patient", year: "1996", runtime: "162", genres: ["Drama", "Romance", "War"], director: "Anthony Minghella", actors: "Ralph Fiennes, Juliette Binoche, Willem Dafoe, Kristin Scott Thomas", plot: "At the close of WWII, a young nurse tends to a badly-burned plane crash victim. His past is shown in flashbacks, revealing an involvement in a fateful love affair.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNDg2OTcxNDE0OF5BMl5BanBnXkFtZTgwOTg2MDM0MDE@._V1_SX300.jpg", }, { id: 102, title: "Dallas Buyers Club", year: "2013", runtime: "117", genres: ["Biography", "Drama"], director: "Jean-Marc Vallée", actors: "Matthew McConaughey, Jennifer Garner, Jared Leto, Denis O'Hare", plot: "In 1985 Dallas, electrician and hustler Ron Woodroof works around the system to help AIDS patients get the medication they need after he is diagnosed with the disease.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTYwMTA4MzgyNF5BMl5BanBnXkFtZTgwMjEyMjE0MDE@._V1_SX300.jpg", }, { id: 103, title: "Frida", year: "2002", runtime: "123", genres: ["Biography", "Drama", "Romance"], director: "Julie Taymor", actors: "Salma Hayek, Mía Maestro, Alfred Molina, Antonio Banderas", plot: "A biography of artist Frida Kahlo, who channeled the pain of a crippling injury and her tempestuous marriage into her work.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTMyODUyMDY1OV5BMl5BanBnXkFtZTYwMDA2OTU3._V1_SX300.jpg", }, { id: 104, title: "Before Sunrise", year: "1995", runtime: "105", genres: ["Drama", "Romance"], director: "Richard Linklater", actors: "Ethan Hawke, Julie Delpy, Andrea Eckert, Hanno Pöschl", plot: "A young man and woman meet on a train in Europe, and wind up spending one evening together in Vienna. Unfortunately, both know that this will probably be their only night together.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTQyMTM3MTQxMl5BMl5BanBnXkFtZTcwMDAzNjQ4Mg@@._V1_SX300.jpg", }, { id: 105, title: "The Rum Diary", year: "2011", runtime: "120", genres: ["Comedy", "Drama"], director: "Bruce Robinson", actors: "Johnny Depp, Aaron Eckhart, Michael Rispoli, Amber Heard", plot: "American journalist Paul Kemp takes on a freelance job in Puerto Rico for a local newspaper during the 1960s and struggles to find a balance between island culture and the expatriates who live there.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM5ODA4MjYxM15BMl5BanBnXkFtZTcwMTM3NTE5Ng@@._V1_SX300.jpg", }, { id: 106, title: "The Last Samurai", year: "2003", runtime: "154", genres: ["Action", "Drama", "History"], director: "Edward Zwick", actors: "Ken Watanabe, Tom Cruise, William Atherton, Chad Lindberg", plot: "An American military advisor embraces the Samurai culture he was hired to destroy after he is captured in battle.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMzkyNzQ1Mzc0NV5BMl5BanBnXkFtZTcwODg3MzUzMw@@._V1_SX300.jpg", }, { id: 107, title: "Chinatown", year: "1974", runtime: "130", genres: ["Drama", "Mystery", "Thriller"], director: "Roman Polanski", actors: "Jack Nicholson, Faye Dunaway, John Huston, Perry Lopez", plot: "A private detective hired to expose an adulterer finds himself caught up in a web of deceit, corruption and murder.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BN2YyNDE5NzItMjAwNC00MGQxLTllNjktZGIzMWFkZjA3OWQ0XkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, { id: 108, title: "Calvary", year: "2014", runtime: "102", genres: ["Comedy", "Drama"], director: "John Michael McDonagh", actors: "Brendan Gleeson, Chris O'Dowd, Kelly Reilly, Aidan Gillen", plot: "After he is threatened during a confession, a good-natured priest must battle the dark forces closing in around him.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTc3MjQ1MjE2M15BMl5BanBnXkFtZTgwNTMzNjE4MTE@._V1_SX300.jpg", }, { id: 109, title: "Before Sunset", year: "2004", runtime: "80", genres: ["Drama", "Romance"], director: "Richard Linklater", actors: "Ethan Hawke, Julie Delpy, Vernon Dobtcheff, Louise Lemoine Torrès", plot: "Nine years after Jesse and Celine first met, they encounter each other again on the French leg of Jesse's book tour.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTQ1MjAwNTM5Ml5BMl5BanBnXkFtZTYwNDM0MTc3._V1_SX300.jpg", }, { id: 110, title: "Spirited Away", year: "2001", runtime: "125", genres: ["Animation", "Adventure", "Family"], director: "Hayao Miyazaki", actors: "Rumi Hiiragi, Miyu Irino, Mari Natsuki, Takashi Naitô", plot: "During her family's move to the suburbs, a sullen 10-year-old girl wanders into a world ruled by gods, witches, and spirits, and where humans are changed into beasts.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjYxMDcyMzIzNl5BMl5BanBnXkFtZTYwNDg2MDU3._V1_SX300.jpg", }, { id: 111, title: "Indochine", year: "1992", runtime: "159", genres: ["Drama", "Romance"], director: "Régis Wargnier", actors: "Catherine Deneuve, Vincent Perez, Linh Dan Pham, Jean Yanne", plot: "This story is set in 1930, at the time when French colonial rule in Indochina is ending. A widowed French woman who works in the rubber fields, raises a Vietnamese princess as if she was ...", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTM1MTkzNzA3NF5BMl5BanBnXkFtZTYwNTI2MzU5._V1_SX300.jpg", }, { id: 112, title: "Birdman or (The Unexpected Virtue of Ignorance)", year: "2014", runtime: "119", genres: ["Comedy", "Drama", "Romance"], director: "Alejandro G. Iñárritu", actors: "Michael Keaton, Emma Stone, Kenny Chin, Jamahl Garrison-Lowe", plot: "Illustrated upon the progress of his latest Broadway play, a former popular actor's struggle to cope with his current life as a wasted actor is shown.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BODAzNDMxMzAxOV5BMl5BanBnXkFtZTgwMDMxMjA4MjE@._V1_SX300.jpg", }, { id: 113, title: "Boyhood", year: "2014", runtime: "165", genres: ["Drama"], director: "Richard Linklater", actors: "Ellar Coltrane, Patricia Arquette, Elijah Smith, Lorelei Linklater", plot: "The life of Mason, from early childhood to his arrival at college.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTYzNDc2MDc0N15BMl5BanBnXkFtZTgwOTcwMDQ5MTE@._V1_SX300.jpg", }, { id: 114, title: "12 Angry Men", year: "1957", runtime: "96", genres: ["Crime", "Drama"], director: "Sidney Lumet", actors: "Martin Balsam, John Fiedler, Lee J. Cobb, E.G. Marshall", plot: "A jury holdout attempts to prevent a miscarriage of justice by forcing his colleagues to reconsider the evidence.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BODQwOTc5MDM2N15BMl5BanBnXkFtZTcwODQxNTEzNA@@._V1_SX300.jpg", }, { id: 115, title: "The Imitation Game", year: "2014", runtime: "114", genres: ["Biography", "Drama", "Thriller"], director: "Morten Tyldum", actors: "Benedict Cumberbatch, Keira Knightley, Matthew Goode, Rory Kinnear", plot: "During World War II, mathematician Alan Turing tries to crack the enigma code with help from fellow mathematicians.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNDkwNTEyMzkzNl5BMl5BanBnXkFtZTgwNTAwNzk3MjE@._V1_SX300.jpg", }, { id: 116, title: "Interstellar", year: "2014", runtime: "169", genres: ["Adventure", "Drama", "Sci-Fi"], director: "Christopher Nolan", actors: "Ellen Burstyn, Matthew McConaughey, Mackenzie Foy, John Lithgow", plot: "A team of explorers travel through a wormhole in space in an attempt to ensure humanity's survival.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjIxNTU4MzY4MF5BMl5BanBnXkFtZTgwMzM4ODI3MjE@._V1_SX300.jpg", }, { id: 117, title: "Big Nothing", year: "2006", runtime: "86", genres: ["Comedy", "Crime", "Thriller"], director: "Jean-Baptiste Andrea", actors: "David Schwimmer, Simon Pegg, Alice Eve, Natascha McElhone", plot: "A frustrated, unemployed teacher joining forces with a scammer and his girlfriend in a blackmailing scheme.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY5NTc2NjYwOV5BMl5BanBnXkFtZTcwMzk5OTY0MQ@@._V1_SX300.jpg", }, { id: 118, title: "Das Boot", year: "1981", runtime: "149", genres: ["Adventure", "Drama", "Thriller"], director: "Wolfgang Petersen", actors: "Jürgen Prochnow, Herbert Grönemeyer, Klaus Wennemann, Hubertus Bengsch", plot: "The claustrophobic world of a WWII German U-boat; boredom, filth, and sheer terror.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjE5Mzk5OTQ0Nl5BMl5BanBnXkFtZTYwNzUwMTQ5._V1_SX300.jpg", }, { id: 119, title: "Shrek 2", year: "2004", runtime: "93", genres: ["Animation", "Adventure", "Comedy"], director: "Andrew Adamson, Kelly Asbury, Conrad Vernon", actors: "Mike Myers, Eddie Murphy, Cameron Diaz, Julie Andrews", plot: "Princess Fiona's parents invite her and Shrek to dinner to celebrate her marriage. If only they knew the newlyweds were both ogres.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTk4MTMwNjI4M15BMl5BanBnXkFtZTcwMjExMzUyMQ@@._V1_SX300.jpg", }, { id: 120, title: "Sin City", year: "2005", runtime: "124", genres: ["Crime", "Thriller"], director: "Frank Miller, Robert Rodriguez, Quentin Tarantino", actors: "Jessica Alba, Devon Aoki, Alexis Bledel, Powers Boothe", plot: "A film that explores the dark and miserable town, Basin City, and tells the story of three different people, all caught up in violent corruption.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BODZmYjMwNzEtNzVhNC00ZTRmLTk2M2UtNzE1MTQ2ZDAxNjc2XkEyXkFqcGdeQXVyMTQxNzMzNDI@._V1_SX300.jpg", }, { id: 121, title: "Nebraska", year: "2013", runtime: "115", genres: ["Adventure", "Comedy", "Drama"], director: "Alexander Payne", actors: "Bruce Dern, Will Forte, June Squibb, Bob Odenkirk", plot: "An aging, booze-addled father makes the trip from Montana to Nebraska with his estranged son in order to claim a million-dollar Mega Sweepstakes Marketing prize.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTU2Mjk2NDkyMl5BMl5BanBnXkFtZTgwNTk0NzcyMDE@._V1_SX300.jpg", }, { id: 122, title: "Shrek", year: "2001", runtime: "90", genres: ["Animation", "Adventure", "Comedy"], director: "Andrew Adamson, Vicky Jenson", actors: "Mike Myers, Eddie Murphy, Cameron Diaz, John Lithgow", plot: "After his swamp is filled with magical creatures, an ogre agrees to rescue a princess for a villainous lord in order to get his land back.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTk2NTE1NTE0M15BMl5BanBnXkFtZTgwNjY4NTYxMTE@._V1_SX300.jpg", }, { id: 123, title: "Mr. & Mrs. Smith", year: "2005", runtime: "120", genres: ["Action", "Comedy", "Crime"], director: "Doug Liman", actors: "Brad Pitt, Angelina Jolie, Vince Vaughn, Adam Brody", plot: "A bored married couple is surprised to learn that they are both assassins hired by competing agencies to kill each other.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTUxMzcxNzQzOF5BMl5BanBnXkFtZTcwMzQxNjUyMw@@._V1_SX300.jpg", }, { id: 124, title: "Original Sin", year: "2001", runtime: "116", genres: ["Drama", "Mystery", "Romance"], director: "Michael Cristofer", actors: "Antonio Banderas, Angelina Jolie, Thomas Jane, Jack Thompson", plot: "A woman along with her lover, plan to con a rich man by marrying him and on earning his trust running away with all his money. Everything goes as planned until she actually begins to fall in love with him.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BODg3Mjg0MDY4M15BMl5BanBnXkFtZTcwNjY5MDQ2NA@@._V1_SX300.jpg", }, { id: 125, title: "Shrek Forever After", year: "2010", runtime: "93", genres: ["Animation", "Adventure", "Comedy"], director: "Mike Mitchell", actors: "Mike Myers, Eddie Murphy, Cameron Diaz, Antonio Banderas", plot: "Rumpelstiltskin tricks a mid-life crisis burdened Shrek into allowing himself to be erased from existence and cast in a dark alternate timeline where Rumpel rules supreme.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTY0OTU1NzkxMl5BMl5BanBnXkFtZTcwMzI2NDUzMw@@._V1_SX300.jpg", }, { id: 126, title: "Before Midnight", year: "2013", runtime: "109", genres: ["Drama", "Romance"], director: "Richard Linklater", actors: "Ethan Hawke, Julie Delpy, Seamus Davey-Fitzpatrick, Jennifer Prior", plot: "We meet Jesse and Celine nine years on in Greece. Almost two decades have passed since their first meeting on that train bound for Vienna.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjA5NzgxODE2NF5BMl5BanBnXkFtZTcwNTI1NTI0OQ@@._V1_SX300.jpg", }, { id: 127, title: "Despicable Me", year: "2010", runtime: "95", genres: ["Animation", "Adventure", "Comedy"], director: "Pierre Coffin, Chris Renaud", actors: "Steve Carell, Jason Segel, Russell Brand, Julie Andrews", plot: "When a criminal mastermind uses a trio of orphan girls as pawns for a grand scheme, he finds their love is profoundly changing him for the better.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY3NjY0MTQ0Nl5BMl5BanBnXkFtZTcwMzQ2MTc0Mw@@._V1_SX300.jpg", }, { id: 128, title: "Troy", year: "2004", runtime: "163", genres: ["Adventure"], director: "Wolfgang Petersen", actors: "Julian Glover, Brian Cox, Nathan Jones, Adoni Maropis", plot: "An adaptation of Homer's great epic, the film follows the assault on Troy by the united Greek forces and chronicles the fates of the men involved.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTk5MzU1MDMwMF5BMl5BanBnXkFtZTcwNjczODMzMw@@._V1_SX300.jpg", }, { id: 129, title: "The Hobbit: An Unexpected Journey", year: "2012", runtime: "169", genres: ["Adventure", "Fantasy"], director: "Peter Jackson", actors: "Ian McKellen, Martin Freeman, Richard Armitage, Ken Stott", plot: "A reluctant hobbit, Bilbo Baggins, sets out to the Lonely Mountain with a spirited group of dwarves to reclaim their mountain home - and the gold within it - from the dragon Smaug.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTcwNTE4MTUxMl5BMl5BanBnXkFtZTcwMDIyODM4OA@@._V1_SX300.jpg", }, { id: 130, title: "The Great Gatsby", year: "2013", runtime: "143", genres: ["Drama", "Romance"], director: "Baz Luhrmann", actors: "Lisa Adam, Frank Aldridge, Amitabh Bachchan, Steve Bisley", plot: "A writer and wall street trader, Nick, finds himself drawn to the past and lifestyle of his millionaire neighbor, Jay Gatsby.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTkxNTk1ODcxNl5BMl5BanBnXkFtZTcwMDI1OTMzOQ@@._V1_SX300.jpg", }, { id: 131, title: "Ice Age", year: "2002", runtime: "81", genres: ["Animation", "Adventure", "Comedy"], director: "Chris Wedge, Carlos Saldanha", actors: "Ray Romano, John Leguizamo, Denis Leary, Goran Visnjic", plot: "Set during the Ice Age, a sabertooth tiger, a sloth, and a wooly mammoth find a lost human infant, and they try to return him to his tribe.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjEyNzI1ODA0MF5BMl5BanBnXkFtZTYwODIxODY3._V1_SX300.jpg", }, { id: 132, title: "The Lord of the Rings: The Fellowship of the Ring", year: "2001", runtime: "178", genres: ["Action", "Adventure", "Drama"], director: "Peter Jackson", actors: "Alan Howard, Noel Appleby, Sean Astin, Sala Baker", plot: "A meek Hobbit from the Shire and eight companions set out on a journey to destroy the powerful One Ring and save Middle Earth from the Dark Lord Sauron.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNTEyMjAwMDU1OV5BMl5BanBnXkFtZTcwNDQyNTkxMw@@._V1_SX300.jpg", }, { id: 133, title: "The Lord of the Rings: The Two Towers", year: "2002", runtime: "179", genres: ["Action", "Adventure", "Drama"], director: "Peter Jackson", actors: "Bruce Allpress, Sean Astin, John Bach, Sala Baker", plot: "While Frodo and Sam edge closer to Mordor with the help of the shifty Gollum, the divided fellowship makes a stand against Sauron's new ally, Saruman, and his hordes of Isengard.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTAyNDU0NjY4NTheQTJeQWpwZ15BbWU2MDk4MTY2Nw@@._V1_SX300.jpg", }, { id: 134, title: "Ex Machina", year: "2015", runtime: "108", genres: ["Drama", "Mystery", "Sci-Fi"], director: "Alex Garland", actors: "Domhnall Gleeson, Corey Johnson, Oscar Isaac, Alicia Vikander", plot: "A young programmer is selected to participate in a ground-breaking experiment in synthetic intelligence by evaluating the human qualities of a breath-taking humanoid A.I.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTUxNzc0OTIxMV5BMl5BanBnXkFtZTgwNDI3NzU2NDE@._V1_SX300.jpg", }, { id: 135, title: "The Theory of Everything", year: "2014", runtime: "123", genres: ["Biography", "Drama", "Romance"], director: "James Marsh", actors: "Eddie Redmayne, Felicity Jones, Tom Prior, Sophie Perry", plot: "A look at the relationship between the famous physicist Stephen Hawking and his wife.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTAwMTU4MDA3NDNeQTJeQWpwZ15BbWU4MDk4NTMxNTIx._V1_SX300.jpg", }, { id: 136, title: "Shogun", year: "1980", runtime: "60", genres: ["Adventure", "Drama", "History"], director: "N/A", actors: "Richard Chamberlain, Toshirô Mifune, Yôko Shimada, Furankî Sakai", plot: "A English navigator becomes both a player and pawn in the complex political games in feudal Japan.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTY1ODI4NzYxMl5BMl5BanBnXkFtZTcwNDA4MzUxMQ@@._V1_SX300.jpg", }, { id: 137, title: "Spotlight", year: "2015", runtime: "128", genres: ["Biography", "Crime", "Drama"], director: "Tom McCarthy", actors: "Mark Ruffalo, Michael Keaton, Rachel McAdams, Liev Schreiber", plot: "The true story of how the Boston Globe uncovered the massive scandal of child molestation and cover-up within the local Catholic Archdiocese, shaking the entire Catholic Church to its core.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjIyOTM5OTIzNV5BMl5BanBnXkFtZTgwMDkzODE2NjE@._V1_SX300.jpg", }, { id: 138, title: "Vertigo", year: "1958", runtime: "128", genres: ["Mystery", "Romance", "Thriller"], director: "Alfred Hitchcock", actors: "James Stewart, Kim Novak, Barbara Bel Geddes, Tom Helmore", plot: "A San Francisco detective suffering from acrophobia investigates the strange activities of an old friend's wife, all the while becoming dangerously obsessed with her.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BNzY0NzQyNzQzOF5BMl5BanBnXkFtZTcwMTgwNTk4OQ@@._V1_SX300.jpg", }, { id: 139, title: "Whiplash", year: "2014", runtime: "107", genres: ["Drama", "Music"], director: "Damien Chazelle", actors: "Miles Teller, J.K. Simmons, Paul Reiser, Melissa Benoist", plot: "A promising young drummer enrolls at a cut-throat music conservatory where his dreams of greatness are mentored by an instructor who will stop at nothing to realize a student's potential.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTU4OTQ3MDUyMV5BMl5BanBnXkFtZTgwOTA2MjU0MjE@._V1_SX300.jpg", }, { id: 140, title: "The Lives of Others", year: "2006", runtime: "137", genres: ["Drama", "Thriller"], director: "Florian Henckel von Donnersmarck", actors: "Martina Gedeck, Ulrich Mühe, Sebastian Koch, Ulrich Tukur", plot: "In 1984 East Berlin, an agent of the secret police, conducting surveillance on a writer and his lover, finds himself becoming increasingly absorbed by their lives.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BNDUzNjYwNDYyNl5BMl5BanBnXkFtZTcwNjU3ODQ0MQ@@._V1_SX300.jpg", }, { id: 141, title: "Hotel Rwanda", year: "2004", runtime: "121", genres: ["Drama", "History", "War"], director: "Terry George", actors: "Xolani Mali, Don Cheadle, Desmond Dube, Hakeem Kae-Kazim", plot: "Paul Rusesabagina was a hotel manager who housed over a thousand Tutsi refugees during their struggle against the Hutu militia in Rwanda.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTI2MzQyNTc1M15BMl5BanBnXkFtZTYwMjExNjc3._V1_SX300.jpg", }, { id: 142, title: "The Martian", year: "2015", runtime: "144", genres: ["Adventure", "Drama", "Sci-Fi"], director: "Ridley Scott", actors: "Matt Damon, Jessica Chastain, Kristen Wiig, Jeff Daniels", plot: "An astronaut becomes stranded on Mars after his team assume him dead, and must rely on his ingenuity to find a way to signal to Earth that he is alive.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMTc2MTQ3MDA1Nl5BMl5BanBnXkFtZTgwODA3OTI4NjE@._V1_SX300.jpg", }, { id: 143, title: "To Kill a Mockingbird", year: "1962", runtime: "129", genres: ["Crime", "Drama"], director: "Robert Mulligan", actors: "Gregory Peck, John Megna, Frank Overton, Rosemary Murphy", plot: "Atticus Finch, a lawyer in the Depression-era South, defends a black man against an undeserved rape charge, and his kids against prejudice.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMjA4MzI1NDY2Nl5BMl5BanBnXkFtZTcwMTcyODc5Mw@@._V1_SX300.jpg", }, { id: 144, title: "The Hateful Eight", year: "2015", runtime: "187", genres: ["Crime", "Drama", "Mystery"], director: "Quentin Tarantino", actors: "Samuel L. Jackson, Kurt Russell, Jennifer Jason Leigh, Walton Goggins", plot: "In the dead of a Wyoming winter, a bounty hunter and his prisoner find shelter in a cabin currently inhabited by a collection of nefarious characters.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BMjA1MTc1NTg5NV5BMl5BanBnXkFtZTgwOTM2MDEzNzE@._V1_SX300.jpg", }, { id: 145, title: "A Separation", year: "2011", runtime: "123", genres: ["Drama", "Mystery"], director: "Asghar Farhadi", actors: "Peyman Moaadi, Leila Hatami, Sareh Bayat, Shahab Hosseini", plot: "A married couple are faced with a difficult decision - to improve the life of their child by moving to another country or to stay in Iran and look after a deteriorating parent who has Alzheimer's disease.", posterUrl: "http://ia.media-imdb.com/images/M/MV5BMTYzMzU4NDUwOF5BMl5BanBnXkFtZTcwMTM5MjA5Ng@@._V1_SX300.jpg", }, { id: 146, title: "The Big Short", year: "2015", runtime: "130", genres: ["Biography", "Comedy", "Drama"], director: "Adam McKay", actors: "Ryan Gosling, Rudy Eisenzopf, Casey Groves, Charlie Talbert", plot: "Four denizens in the world of high-finance predict the credit and housing bubble collapse of the mid-2000s, and decide to take on the big banks for their greed and lack of foresight.", posterUrl: "https://images-na.ssl-images-amazon.com/images/M/MV5BNDc4MThhN2EtZjMzNC00ZDJmLThiZTgtNThlY2UxZWMzNjdkXkEyXkFqcGdeQXVyNDk3NzU2MTQ@._V1_SX300.jpg", }, ]; ================================================ FILE: ui/src/pages/misc/TaskQueue.jsx ================================================ import { useRouteMatch } from "react-router-dom"; import sharedStyles from "../styles"; import { usePollData, useQueueSizes, useTaskNames } from "../../data/task"; import { makeStyles } from "@material-ui/styles"; import { Helmet } from "react-helmet"; import { usePushHistory } from "../../components/NavLink"; import { formatRelative } from "date-fns"; import { Paper, DataTable, LinearProgress, Heading, Dropdown, } from "../../components"; import _ from "lodash"; import { timestampRenderer } from "../../utils/helpers"; const useStyles = makeStyles(sharedStyles); function getSizesMap(sizes) { const retval = new Map(); for (let row of sizes) { if (row.isSuccess) { retval.set(row.data.domain, row.data.size); } } return retval; } export default function TaskDefinition() { const taskNames = useTaskNames(); const pushHistory = usePushHistory(); const classes = useStyles(); const match = useRouteMatch(); const taskName = match.params.name || ""; const { data: pollData, isFetching } = usePollData(taskName); const domains = pollData ? pollData.map((row) => row.domain) : null; const sizes = useQueueSizes(taskName, domains); const sizesMap = getSizesMap(sizes); const now = new Date(); function setTaskName(name) { if (name === null) { name = ""; } pushHistory(`/taskQueue/${name}`); } return (
    Conductor UI - Task Queue
    Task Queues setTaskName(val)} disableClearable getOptionSelected={(option, value) => { // Accept empty string if (value === "") return false; return value === option; }} value={taskName} />
    {isFetching && }
    {!_.isUndefined(pollData) && ( _.isEmpty(domain) ? "(Domain not set)" : domain, }, { name: "workerId", label: "Last Polled Worker" }, { name: "lastPollTime", label: "Last Poll Time", renderer: (time) => `${timestampRenderer(time)} (${formatRelative(time, now)})`, }, { name: "domain", id: "queueSize", label: "Queue Size", renderer: (domain) => sizesMap.get(domain), }, ]} /> )}
    ); } ================================================ FILE: ui/src/pages/styles.js ================================================ import { colors } from "../theme/variables"; export default { wrapper: { overflowY: "scroll", overflowX: "hidden", height: "100%", }, padded: { padding: 30, }, header: { backgroundColor: colors.gray14, padding: "20px 30px 0 30px", zIndex: 1, }, paddingBottom: { paddingBottom: 25, }, tabContent: { padding: 30, }, buttonRow: { marginBottom: 15, display: "flex", justifyContent: "flex-end", }, field: { marginBottom: 15, }, }; ================================================ FILE: ui/src/pages/workbench/ExecutionHistory.jsx ================================================ import { List, ListItem, ListItemText, Toolbar, IconButton, } from "@material-ui/core"; import { StatusBadge, Text, NavLink } from "../../components"; import { makeStyles } from "@material-ui/styles"; import { colors } from "../../theme/variables"; import _ from "lodash"; import { useInvalidateWorkflows, useWorkflowsByIds } from "../../data/workflow"; import { formatRelative } from "date-fns"; import RefreshIcon from "@material-ui/icons/Refresh"; const useStyles = makeStyles({ sidebar: { width: 360, border: "0px solid rgba(0, 0, 0, 0)", zIndex: 1, boxShadow: "0 2px 4px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%)", background: "#fff", display: "flex", flexDirection: "column", }, toolbar: { backgroundColor: colors.gray14, }, list: { overflowY: "auto", flex: 1, }, }); export default function ExecutionHistory({ run }) { const classes = useStyles(); const workflowRecords = run ? run.workflowRecords : []; const workflowIds = workflowRecords.map((record) => `${record.workflowId}`); const results = useWorkflowsByIds(workflowIds, { staleTime: 60000, }) || []; const resultsMap = new Map( results .filter((r) => r.isSuccess) .map((result) => [result.data.workflowId, result.data]) ); const invalidateWorkflows = useInvalidateWorkflows(); function handleRefresh() { invalidateWorkflows(workflowIds); } return (
    Execution History {Array.from(resultsMap.values()).map((workflow) => ( {workflow.workflowId} } secondary={ {" "} {formatRelative(new Date(workflow.startTime), new Date())} } secondaryTypographyProps={{ component: "div" }} /> ))} {_.isEmpty(workflowRecords) && ( No execution history. )}
    ); } ================================================ FILE: ui/src/pages/workbench/RunHistory.tsx ================================================ import { useImperativeHandle, useState, forwardRef } from "react"; import { useLocalStorage } from "../../utils/localstorage"; import { Text } from "../../components"; import { List, ListItem, ListItemText, ListItemSecondaryAction, Toolbar, IconButton, } from "@material-ui/core"; import { makeStyles } from "@material-ui/styles"; import { immutableReplaceAt } from "../../utils/helpers"; import { formatRelative } from "date-fns"; import DeleteIcon from "@material-ui/icons/DeleteForever"; import { colors } from "../../theme/variables"; import CloseIcon from "@material-ui/icons/Close"; import _ from "lodash"; import { useEnv } from "../../plugins/env"; const useStyles = makeStyles({ sidebar: { width: 300, border: "0px solid rgba(0, 0, 0, 0)", zIndex: 1, boxShadow: "0 2px 4px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%)", background: "#fff", display: "flex", flexDirection: "column", }, toolbar: { backgroundColor: colors.gray14, }, title: { fontWeight: "bold", flex: 1, }, list: { overflowY: "auto", cursor: "pointer", flex: 1, }, }); type RunPayload = any; type RunEntry = { runPayload: RunPayload; workflowRecords: WorkflowRecord[]; createTime: number; }; type WorkflowRecord = { workflowId: string; }; type RunHistoryProps = { onRunSelected: (run: RunEntry | undefined) => void; }; const RUN_HISTORY_SCHEMA_VER = 1; const RunHistory = forwardRef((props: RunHistoryProps, ref) => { const { onRunSelected } = props; const { stack } = useEnv(); const classes = useStyles(); const [selectedCreateTime, setSelectedCreateTime] = useState< number | undefined >(undefined); const [runHistory, setRunHistory]: readonly [ RunEntry[], (v: RunEntry[]) => void ] = useLocalStorage(`runHistory_${stack}_${RUN_HISTORY_SCHEMA_VER}`, []); useImperativeHandle(ref, () => ({ pushNewRun: (runPayload: RunPayload) => { const createTime = new Date().getTime(); const newRun = { runPayload: runPayload, workflowRecords: [], createTime: createTime, }; setRunHistory([newRun, ...runHistory]); setSelectedCreateTime(createTime); return newRun; }, updateRun: (createTime: number, workflowId: string) => { const idx = runHistory.findIndex((v) => v.createTime === createTime); const currRun = runHistory[idx]; const oldRecords = currRun.workflowRecords; const updatedRun = { runPayload: currRun.runPayload, workflowRecords: [ { workflowId: workflowId, }, ...oldRecords, ], createTime: currRun.createTime, }; setRunHistory(immutableReplaceAt(runHistory, idx, updatedRun)); onRunSelected(updatedRun); }, })); function handleSelectRun(run: RunEntry) { if (onRunSelected) onRunSelected(run); setSelectedCreateTime(run.createTime); } function handleDeleteAll() { if (window.confirm("Delete all run history in this browser?")) { setRunHistory([]); } } function handleDeleteItem(run: RunEntry) { const newHistory = runHistory.filter( (v) => v.createTime !== run.createTime ); if (newHistory.length > 0) { setSelectedCreateTime(newHistory[0].createTime); onRunSelected(newHistory[0]); } else { console.log("Empty history"); setSelectedCreateTime(undefined); onRunSelected(undefined); } setRunHistory(newHistory); } return (
    Run History {runHistory.map((run) => ( handleSelectRun(run)} > handleDeleteItem(run)}> ))} {_.isEmpty(runHistory) && No saved runs.}
    ); }); export default RunHistory; ================================================ FILE: ui/src/pages/workbench/Workbench.jsx ================================================ import { useState, useRef } from "react"; import { makeStyles } from "@material-ui/styles"; import { Helmet } from "react-helmet"; import RunHistory from "./RunHistory"; import WorkbenchForm from "./WorkbenchForm"; import { colors } from "../../theme/variables"; import { useStartWorkflow } from "../../data/workflow"; import ExecutionHistory from "./ExecutionHistory"; const useStyles = makeStyles({ wrapper: { height: "100%", overflow: "hidden", display: "flex", flexDirection: "row", position: "relative", }, name: { width: "50%", }, submitButton: { float: "right", }, toolbar: { backgroundColor: colors.gray14, }, workflowName: { fontWeight: "bold", }, main: { flex: 1, display: "flex", flexDirection: "column", }, row: { display: "flex", flexDirection: "row", }, fields: { margin: 30, flex: 1, display: "flex", flexDirection: "column", gap: 15, }, runInfo: { marginLeft: -350, }, }); export default function Workbench() { const classes = useStyles(); const runHistoryRef = useRef(); const [run, setRun] = useState(undefined); const { mutate: startWorkflow } = useStartWorkflow({ onSuccess: (workflowId, variables) => { runHistoryRef.current.updateRun(variables.createTime, workflowId); }, }); const handleRunSelect = (run) => { setRun(run); }; const handleSaveRun = (runPayload) => { const newRun = runHistoryRef.current.pushNewRun(runPayload); setRun(newRun); return newRun; }; const handleExecuteRun = (createTime, runPayload) => { startWorkflow({ createTime, body: runPayload, }); }; return ( <> Conductor UI - Workbench
    ); } ================================================ FILE: ui/src/pages/workbench/WorkbenchForm.jsx ================================================ import { Text, Pill } from "../../components"; import { Toolbar, IconButton, Tooltip } from "@material-ui/core"; import FormikInput from "../../components/formik/FormikInput"; import FormikJsonInput from "../../components/formik/FormikJsonInput"; import { makeStyles } from "@material-ui/styles"; import _ from "lodash"; import { Form, setNestedObjectValues, withFormik } from "formik"; import { useWorkflowDef } from "../../data/workflow"; import FormikVersionDropdown from "../../components/formik/FormikVersionDropdown"; import PlayArrowIcon from "@material-ui/icons/PlayArrow"; import PlaylistAddIcon from "@material-ui/icons/PlaylistAdd"; import SaveIcon from "@material-ui/icons/Save"; import { colors } from "../../theme/variables"; import { timestampRenderer } from "../../utils/helpers"; import * as Yup from "yup"; import FormikWorkflowNameInput from "../../components/formik/FormikWorkflowNameInput"; const useStyles = makeStyles({ name: { width: "50%", }, submitButton: { float: "right", }, toolbar: { backgroundColor: colors.gray14, }, workflowName: { fontWeight: "bold", }, main: { flex: 1, display: "flex", flexDirection: "column", overflow: "auto", }, fields: { width: "100%", padding: 30, flex: 1, display: "flex", flexDirection: "column", overflowX: "hidden", overflowY: "auto", gap: 15, }, }); Yup.addMethod(Yup.string, "isJson", function () { return this.test("is-json", "is not valid json", (value) => { if (_.isEmpty(value)) return true; try { JSON.parse(value); } catch (e) { return false; } return true; }); }); const validationSchema = Yup.object({ workflowName: Yup.string().required("Workflow Name is required"), workflowInput: Yup.string().isJson(), taskToDomain: Yup.string().isJson(), }); export default withFormik({ enableReinitialize: true, mapPropsToValues: ({ selectedRun }) => runPayloadToFormData(_.get(selectedRun, "runPayload")), validationSchema: validationSchema, })(WorkbenchForm); function WorkbenchForm(props) { const { values, validateForm, setTouched, setFieldValue, dirty, selectedRun, saveRun, executeRun, } = props; const classes = useStyles(); const { workflowName, workflowVersion } = values; const createTime = selectedRun ? selectedRun.createTime : undefined; const { refetch } = useWorkflowDef(workflowName, workflowVersion, null, { onSuccess: populateInput, enabled: false, }); function triggerPopulateInput() { refetch(); } function populateInput(workflowDef) { let bootstrap = {}; if (!_.isEmpty(values.workflowInput)) { const existing = JSON.parse(values.workflowInput); bootstrap = _.pickBy(existing, (v) => v !== ""); } if (workflowDef.inputParameters) { for (let param of workflowDef.inputParameters) { if (!_.has(bootstrap, param)) { bootstrap[param] = ""; } } setFieldValue("workflowInput", JSON.stringify(bootstrap, null, 2)); } } function handleRun() { validateForm().then((errors) => { if (Object.keys(errors).length === 0) { const payload = formDataToRunPayload(values); if (!dirty && createTime) { console.log("Executing pre-existing run. Append workflowRecord"); executeRun(createTime, payload); } else { console.log("Executing new run. Save first then execute"); const newRun = saveRun(payload); executeRun(newRun.createTime, payload); } } else { // Handle validation error manually (not using handleSubmit) setTouched(setNestedObjectValues(errors, true)); } }); } function handleSave() { validateForm().then((errors) => { if (Object.keys(errors).length === 0) { const payload = formDataToRunPayload(values); saveRun(payload); } else { setTouched(setNestedObjectValues(errors, true)); } }); } return (
    Workflow Workbench
    {dirty && } {createTime && Created: {timestampRenderer(createTime)}}
    ); } function runPayloadToFormData(runPayload) { return { workflowName: _.get(runPayload, "name", ""), workflowVersion: _.get(runPayload, "version", ""), workflowInput: _.has(runPayload, "input") ? JSON.stringify(runPayload.input, null, 2) : "", correlationId: _.get(runPayload, "correlationId", ""), taskToDomain: _.has(runPayload, "taskToDomain") ? JSON.stringify(runPayload.taskToDomain, null, 2) : "", }; } function formDataToRunPayload(form) { let runPayload = { name: form.workflowName, }; if (form.workflowVersion) { runPayload.version = form.workflowVersion; } if (form.workflowInput) { runPayload.input = JSON.parse(form.workflowInput); } if (form.correlationId) { runPayload.correlationId = form.correlationId; } if (form.taskToDomain) { runPayload.taskToDomain = JSON.parse(form.taskToDomain); } return runPayload; } // runHistoryRef.current.pushRun(runPayload); ================================================ FILE: ui/src/plugins/AppBarModules.jsx ================================================ export default function AppBarModules() { return null; } ================================================ FILE: ui/src/plugins/AppLogo.jsx ================================================ import React from "react"; import { makeStyles } from "@material-ui/core/styles"; import { getBasename } from "../utils/helpers"; import { cleanDuplicateSlash } from "./fetch"; const useStyles = makeStyles((theme) => ({ logo: { height: 37, width: 175, marginRight: 30, }, })); export default function AppLogo() { const classes = useStyles(); const logoPath = getBasename() + 'logo.svg'; return Conductor; } ================================================ FILE: ui/src/plugins/CustomAppBarButtons.jsx ================================================ export default function CustomAppBarButtons() { return <>; } ================================================ FILE: ui/src/plugins/CustomRoutes.jsx ================================================ export default function CustomRoutes() { return <>; } ================================================ FILE: ui/src/plugins/constants.js ================================================ ================================================ FILE: ui/src/plugins/customTypeRenderers.jsx ================================================ export const customTypeRenderers = {}; ================================================ FILE: ui/src/plugins/env.js ================================================ export function useEnv() { return { stack: "default", defaultStack: "default", }; } ================================================ FILE: ui/src/plugins/fetch.js ================================================ import { getBasename } from "../utils/helpers"; import { useEnv } from "./env"; export function useFetchContext() { const { stack } = useEnv(); return { stack, ready: true, }; } export function fetchWithContext( path, context, fetchParams, isJsonResponse = true ) { const newParams = { ...fetchParams }; const basename = getBasename(); const newPath = basename + `api/${path}`; const cleanPath = cleanDuplicateSlash(newPath); // Cleanup duplicated slashes return fetch(cleanPath, newParams) .then((res) => Promise.all([res, res.text()])) .then(([res, text]) => { if (!res.ok) { // get error message from body or default to response status const error = text || res.status; return Promise.reject(error); } else if (!text || text.length === 0) { return null; } else if (!isJsonResponse) { return text; } else { try { return JSON.parse(text); } catch (e) { return text; } } }); } export function cleanDuplicateSlash(path) { return path.replace(/([^:]\/)\/+/g, "$1"); } ================================================ FILE: ui/src/react-app-env.d.ts ================================================ /// ================================================ FILE: ui/src/schema/task.js ================================================ export const NEW_TASK_TEMPLATE = { name: "", description: "Edit or extend this sample task. Set the task name to get started", retryCount: 3, timeoutSeconds: 3600, inputKeys: [], outputKeys: [], timeoutPolicy: "TIME_OUT_WF", retryLogic: "FIXED", retryDelaySeconds: 60, responseTimeoutSeconds: 600, rateLimitPerFrequency: 0, rateLimitFrequencyInSeconds: 1, ownerEmail: "", }; export function configureMonaco(monaco) { // No-op } ================================================ FILE: ui/src/schema/workflow.js ================================================ /* eslint-disable no-template-curly-in-string */ export const NEW_WORKFLOW_TEMPLATE = { name: "", description: "Edit or extend this sample workflow. Set the workflow name to get started", version: 1, tasks: [ { name: "get_population_data", taskReferenceName: "get_population_data", inputParameters: { http_request: { uri: "https://datausa.io/api/data?drilldowns=Nation&measures=Population", method: "GET", }, }, type: "HTTP", }, ], inputParameters: [], outputParameters: { data: "${get_population_data.output.response.body.data}", source: "${get_population_data.output.response.body.source}", }, schemaVersion: 2, restartable: true, workflowStatusListenerEnabled: false, ownerEmail: "example@email.com", timeoutPolicy: "ALERT_ONLY", timeoutSeconds: 0, }; const WORKFLOW_SCHEMA = { $schema: "http://json-schema.org/draft-07/schema", $id: "http://example.com/example.json", type: "object", title: "The root schema", description: "The root schema comprises the entire JSON document.", default: {}, examples: [ { name: "first_sample_workflow", description: "First Sample Workflow", version: 1, tasks: [ { name: "get_population_data", taskReferenceName: "get_population_data", inputParameters: { http_request: { uri: "https://datausa.io/api/data?drilldowns=Nation&measures=Population", method: "GET", }, }, type: "HTTP", }, ], inputParameters: [], outputParameters: { data: "${get_population_data.output.response.body.data}", source: "${get_population_data.output.response.body.source}", }, schemaVersion: 2, restartable: true, workflowStatusListenerEnabled: false, ownerEmail: "example@email.com", timeoutPolicy: "ALERT_ONLY", timeoutSeconds: 0, }, ], required: ["name", "version", "tasks", "schemaVersion"], properties: { name: { $id: "#/properties/name", default: "", description: "Workflow Name - should be without spaces or special characters. Underscores and periods are allowed.", examples: ["first_sample_workflow"], maxLength: 100, pattern: "^[\\w\\.]+$", title: "Workflow Name", type: "string", }, description: { $id: "#/properties/description", type: "string", title: "Workflow Description", description: "An brief description of your workflow for reference.", default: "", examples: ["First Sample Workflow"], }, version: { $id: "#/properties/version", default: 0, description: "An explanation about the purpose of this instance.", examples: [1], title: "The version schema", minimum: 1, type: "integer", }, tasks: { $id: "#/properties/tasks", type: "array", title: "Workflow Tasks", description: "This list holds the tasks for your workflow.", default: [], examples: [ [ { name: "get_population_data", taskReferenceName: "get_population_data", inputParameters: { http_request: { uri: "https://datausa.io/api/data?drilldowns=Nation&measures=Population", method: "GET", }, }, type: "HTTP", }, ], ], additionalItems: true, items: { $id: "#/properties/tasks/items", anyOf: [ { $id: "#/properties/tasks/items/anyOf/0", type: "object", title: "The first anyOf schema", description: "Workflow task details", default: { name: "", taskReferenceName: "", inputParameters: {}, type: "SIMPLE", }, examples: [ { name: "get_population_data", taskReferenceName: "get_population_data", inputParameters: { http_request: { uri: "https://datausa.io/api/data?drilldowns=Nation&measures=Population", method: "GET", }, }, type: "HTTP", }, ], required: ["name", "taskReferenceName", "inputParameters", "type"], properties: { name: { $id: "#/properties/tasks/items/anyOf/0/properties/name", type: "string", title: "Task name", description: "Task name", default: "", examples: ["get_population_data"], }, taskReferenceName: { $id: "#/properties/tasks/items/anyOf/0/properties/taskReferenceName", type: "string", title: "Task Reference Name", description: "A unique task reference name for this task in the entire workflow", default: "", examples: ["get_population_data"], }, inputParameters: { $id: "#/properties/tasks/items/anyOf/0/properties/inputParameters", type: "object", title: "Input Parameters", description: "Task input parameters", default: {}, examples: [ { http_request: { uri: "https://datausa.io/api/data?drilldowns=Nation&measures=Population", method: "GET", }, }, ], required: [], properties: {}, additionalProperties: true, }, type: { $id: "#/properties/tasks/items/anyOf/0/properties/type", type: "string", title: "Task Type", description: "Task type", default: "", examples: ["HTTP"], }, }, additionalProperties: true, }, ], }, }, inputParameters: { $id: "#/properties/inputParameters", type: "array", title: "Workflow Input Parameters", description: "An explanation about the purpose of this instance.", default: [], examples: [[]], additionalItems: true, items: { $id: "#/properties/inputParameters/items", }, }, outputParameters: { $id: "#/properties/outputParameters", type: "object", title: "The outputParameters schema", description: "An explanation about the purpose of this instance.", default: {}, examples: [ { data: "${get_population_data.output.response.body.data}", source: "${get_population_data.output.response.body.source}", }, ], required: [], properties: {}, additionalProperties: true, }, schemaVersion: { $id: "#/properties/schemaVersion", type: "integer", title: "Schema Version", description: "Fixed schema version", default: 2, examples: [2], }, restartable: { $id: "#/properties/restartable", type: "boolean", title: "Workflow restartable", description: "Specify if the workflow is restartable.", default: true, examples: [true, false], }, workflowStatusListenerEnabled: { $id: "#/properties/workflowStatusListenerEnabled", type: "boolean", title: "The workflowStatusListenerEnabled schema", description: "An explanation about the purpose of this instance.", default: false, examples: [true, false], }, ownerEmail: { $id: "#/properties/ownerEmail", type: "string", title: "The ownerEmail schema", description: "An explanation about the purpose of this instance.", default: "", examples: ["example@email.com"], }, timeoutPolicy: { $id: "#/properties/timeoutPolicy", type: "string", title: "The timeoutPolicy schema", description: "An explanation about the purpose of this instance.", default: "", examples: ["ALERT_ONLY", "TIME_OUT_WF"], }, timeoutSeconds: { $id: "#/properties/timeoutSeconds", type: "integer", title: "The timeoutSeconds schema", description: "An explanation about the purpose of this instance.", default: 0, examples: [0], }, }, additionalProperties: true, }; export const JSON_FILE_NAME = "file:///workflow.json"; export function configureMonaco(monaco) { monaco.languages.typescript.javascriptDefaults.setEagerModelSync(true); // noinspection JSUnresolvedVariable monaco.languages.typescript.javascriptDefaults.setCompilerOptions({ target: monaco.languages.typescript.ScriptTarget.ES6, allowNonTsExtensions: true, }); let modelUri = monaco.Uri.parse(JSON_FILE_NAME); monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: true, schemas: [ { uri: "http://conductor.tmp/schemas/workflow.json", // id of the first schema fileMatch: [modelUri.toString()], // associate with our model schema: WORKFLOW_SCHEMA, }, ], }); } ================================================ FILE: ui/src/serviceWorker.js ================================================ // This optional code is used to register a service worker. // register() is not called by default. // This lets the app load faster on subsequent visits in production, and gives // it offline capabilities. However, it also means that developers (and users) // will only see deployed updates on subsequent visits to a page, after all the // existing tabs open on the page have been closed, since previously cached // resources are updated in the background. // To learn more about the benefits of this model and instructions on how to // opt-in, read https://bit.ly/CRA-PWA const isLocalhost = Boolean( window.location.hostname === "localhost" || // [::1] is the IPv6 localhost address. window.location.hostname === "[::1]" || // 127.0.0.0/8 are considered localhost for IPv4. window.location.hostname.match( /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/ ) ); export function register(config) { if (process.env.NODE_ENV === "production" && "serviceWorker" in navigator) { // The URL constructor is available in all browsers that support SW. const publicUrl = new URL(process.env.PUBLIC_URL, window.location.href); if (publicUrl.origin !== window.location.origin) { // Our service worker won't work if PUBLIC_URL is on a different origin // from what our page is served on. This might happen if a CDN is used to // serve assets; see https://github.com/facebook/create-react-app/issues/2374 return; } window.addEventListener("load", () => { const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`; if (isLocalhost) { // This is running on localhost. Let's check if a service worker still exists or not. checkValidServiceWorker(swUrl, config); // Add some additional logging to localhost, pointing developers to the // service worker/PWA documentation. navigator.serviceWorker.ready.then(() => { console.log( "This web app is being served cache-first by a service " + "worker. To learn more, visit https://bit.ly/CRA-PWA" ); }); } else { // Is not localhost. Just register service worker registerValidSW(swUrl, config); } }); } } function registerValidSW(swUrl, config) { navigator.serviceWorker .register(swUrl) .then((registration) => { registration.onupdatefound = () => { const installingWorker = registration.installing; if (installingWorker == null) { return; } installingWorker.onstatechange = () => { if (installingWorker.state === "installed") { if (navigator.serviceWorker.controller) { // At this point, the updated precached content has been fetched, // but the previous service worker will still serve the older // content until all client tabs are closed. console.log( "New content is available and will be used when all " + "tabs for this page are closed. See https://bit.ly/CRA-PWA." ); // Execute callback if (config && config.onUpdate) { config.onUpdate(registration); } } else { // At this point, everything has been precached. // It's the perfect time to display a // "Content is cached for offline use." message. console.log("Content is cached for offline use."); // Execute callback if (config && config.onSuccess) { config.onSuccess(registration); } } } }; }; }) .catch((error) => { console.error("Error during service worker registration:", error); }); } function checkValidServiceWorker(swUrl, config) { // Check if the service worker can be found. If it can't reload the page. fetch(swUrl, { headers: { "Service-Worker": "script" }, }) .then((response) => { // Ensure service worker exists, and that we really are getting a JS file. const contentType = response.headers.get("content-type"); if ( response.status === 404 || (contentType != null && contentType.indexOf("javascript") === -1) ) { // No service worker found. Probably a different app. Reload the page. navigator.serviceWorker.ready.then((registration) => { registration.unregister().then(() => { window.location.reload(); }); }); } else { // Service worker found. Proceed as normal. registerValidSW(swUrl, config); } }) .catch(() => { console.log( "No internet connection found. App is running in offline mode." ); }); } export function unregister() { if ("serviceWorker" in navigator) { navigator.serviceWorker.ready .then((registration) => { registration.unregister(); }) .catch((error) => { console.error(error.message); }); } } ================================================ FILE: ui/src/setupProxy.js ================================================ const { createProxyMiddleware } = require("http-proxy-middleware"); const target = process.env.WF_SERVER || "http://localhost:8080"; module.exports = function (app) { app.use( "/api", createProxyMiddleware({ target: target, //pathRewrite: { "^/api/": "/" }, changeOrigin: true, }) ); }; ================================================ FILE: ui/src/setupTests.js ================================================ // jest-dom adds custom jest matchers for asserting on DOM nodes. // allows you to do things like: // expect(element).toHaveTextContent(/react/i) // learn more: https://github.com/testing-library/jest-dom import "@testing-library/jest-dom/extend-expect"; ================================================ FILE: ui/src/theme/colorOverrides.js ================================================ import * as colors from "./colors"; const brandAliases = { brand00: colors.indigo00, brand01: colors.indigo01, brand02: colors.indigo02, brand03: colors.indigo03, brand04: colors.indigo04, brand05: colors.indigo05, brand06: colors.indigo06, brand07: colors.indigo07, brand08: colors.indigo08, brand09: colors.indigo09, brand10: colors.indigo10, brand11: colors.indigo11, brand12: colors.indigo12, brand13: colors.indigo13, brand14: colors.indigo14, }; const brandShortcuts = { brand: brandAliases.brand07, bgBrand: brandAliases.brand07, bgBrandLight: brandAliases.brand09, bgBrandDark: brandAliases.brand05, brandXLight: colors.indigoXLight, brandXXLight: colors.indigoXXLight, }; const failureAliases = { failure: colors.red07, failureLight: colors.red09, failureDark: colors.red05, }; export const colorOverrides = { ...colors, ...brandAliases, ...brandShortcuts, ...failureAliases, }; ================================================ FILE: ui/src/theme/colors.js ================================================ // Backgrounds / Black exports.black = "#050505"; // Transparents / Black / 00-Black-Light (70%) exports.blackLight = "rgba(5,5,5,0.7)"; // Transparents / Black / 01-Black-Xlight (40%) exports.blackXLight = "rgba(5,5,5,0.4)"; // Transparents / Black / 02-Black-Xxlight (10%) exports.blackXXLight = "rgba(5,5,5,0.1)"; // Backgrounds / Blue / Blue-00 (Xxdark) exports.blue00 = "#00101f"; // Backgrounds / Blue / Blue-01 exports.blue01 = "#05192b"; // Backgrounds / Blue / Blue-02 exports.blue02 = "#092743"; // Backgrounds / Blue / Blue-03 (Xdark) exports.blue03 = "#0d365c"; // Backgrounds / Blue / Blue-04 exports.blue04 = "#12487a"; // Backgrounds / Blue / Blue-05 (Dark) exports.blue05 = "#165b99"; // Backgrounds / Blue / Blue-06 exports.blue06 = "#1b6fb9"; // Backgrounds / Blue / -Blue-07 (Base) exports.blue07 = "#1f83db"; // Backgrounds / Blue / Blue-08 exports.blue08 = "#5995e1"; // Backgrounds / Blue / Blue-09 (Light) exports.blue09 = "#7ea7e7"; // Backgrounds / Blue / Blue-10 exports.blue10 = "#9dbaec"; // Backgrounds / Blue / Blue-11 (Xlight) exports.blue11 = "#bacdf2"; // Backgrounds / Blue / Blue-12 exports.blue12 = "#d2def6"; // Backgrounds / Blue / Blue-13 exports.blue13 = "#eaf0fb"; // Backgrounds / Blue / Blue-14 (Xxlight) exports.blue14 = "#f7fafd"; // Transparents / Blue / 00-Blue-Light (70%) exports.blueLight = "rgba(31,131,219,0.7)"; // Transparents / Blue / 01-Blue-Xlight (40%) exports.blueXLight = "rgba(31,131,219,0.4)"; // Transparents / Blue / 02-Blue-Xxlight (10%) exports.blueXXLight = "rgba(31,131,219,0.1)"; // Backgrounds / Cyan / Cyan-00 (Xxdark) exports.cyan00 = "#001b1e"; // Backgrounds / Cyan / Cyan-01 exports.cyan01 = "#042529"; // Backgrounds / Cyan / Cyan-02 exports.cyan02 = "#08373d"; // Backgrounds / Cyan / Cyan-03 (Xdark) exports.cyan03 = "#0f4a52"; // Backgrounds / Cyan / Cyan-04 exports.cyan04 = "#17616c"; // Backgrounds / Cyan / Cyan-05 (Dark) exports.cyan05 = "#207986"; // Backgrounds / Cyan / Cyan-06 exports.cyan06 = "#2991a2"; // Backgrounds / Cyan / -Cyan-07 (Base) exports.cyan07 = "#32abbe"; // Backgrounds / Cyan / Cyan-08 exports.cyan08 = "#5fb8c8"; // Backgrounds / Cyan / Cyan-09 (Light) exports.cyan09 = "#80c5d2"; // Backgrounds / Cyan / Cyan-10 exports.cyan10 = "#9ed2dc"; // Backgrounds / Cyan / Cyan-11 (Xlight) exports.cyan11 = "#badfe6"; // Backgrounds / Cyan / Cyan-12 exports.cyan12 = "#d2eaef"; // Backgrounds / Cyan / Cyan-13 exports.cyan13 = "#eaf5f8"; // Backgrounds / Cyan / Cyan-14 (Xxlight) exports.cyan14 = "#f7fcfd"; // Transparents / Cyan / 00-Cyan-Light (70%) exports.cyanLight = "rgba(50,171,190,0.7)"; // Transparents / Cyan / 01-Cyan-Xlight (40%) exports.cyanXLight = "rgba(50,171,190,0.4)"; // Transparents / Cyan / 02-Cyan-Xxlight (10%) exports.cyanXXLight = "rgba(50,171,190,0.1)"; // Backgrounds / Grape / Grape-00 (Xxdark) exports.grape00 = "#18001f"; // Backgrounds / Grape / Grape-01 exports.grape01 = "#200b2a"; // Backgrounds / Grape / Grape-02 exports.grape02 = "#33143f"; // Backgrounds / Grape / Grape-03 (Xdark) exports.grape03 = "#481d56"; // Backgrounds / Grape / Grape-04 exports.grape04 = "#602871"; // Backgrounds / Grape / Grape-05 (Dark) exports.grape05 = "#7a338d"; // Backgrounds / Grape / Grape-06 exports.grape06 = "#943eab"; // Backgrounds / Grape / -Grape-07 (Base) exports.grape07 = "#b04ac9"; // Backgrounds / Grape / Grape-08 exports.grape08 = "#be68d2"; // Backgrounds / Grape / Grape-09 (Light) exports.grape09 = "#cb84da"; // Backgrounds / Grape / Grape-10 exports.grape10 = "#d89fe3"; // Backgrounds / Grape / Grape-11 (Xlight) exports.grape11 = "#e4baeb"; // Backgrounds / Grape / Grape-12 exports.grape12 = "#edd2f2"; // Backgrounds / Grape / Grape-13 exports.grape13 = "#f7e9f9"; // Backgrounds / Grape / Grape-14 (Xxlight) exports.grape14 = "#fcf7fd"; // Transparents / Grape / 00-Grape-Light (70%) exports.grapeLight = "rgba(176,74,201,0.7)"; // Transparents / Grape / 01-Grape-Xlight (40%) exports.grapeXLight = "rgba(176,74,201,0.4)"; // Transparents / Grape / 02-Grape-Xxlight (10%) exports.grapeXXLight = "rgba(176,74,201,0.1)"; // Backgrounds / Gray / Gray-00 (Xxdark) exports.gray00 = "#0f0f0f"; // Backgrounds / Gray / Gray-01 exports.gray01 = "#181818"; // Backgrounds / Gray / Gray-02 exports.gray02 = "#242424"; // Backgrounds / Gray / Gray-03 (Xdark) exports.gray03 = "#323232"; // Backgrounds / Gray / Gray-04 exports.gray04 = "#424242"; // Backgrounds / Gray / Gray-05 (Dark) exports.gray05 = "#535353"; // Backgrounds / Gray / Gray-06 exports.gray06 = "#646464"; // Backgrounds / Gray / -Gray-07 (Base) exports.gray07 = "#767676"; // Backgrounds / Gray / Gray-08 exports.gray08 = "#8a8a8a"; // Backgrounds / Gray / Gray-09 (Light) exports.gray09 = "#9e9e9e"; // Backgrounds / Gray / Gray-10 exports.gray10 = "#b3b3b3"; // Backgrounds / Gray / Gray-11 (Xlight) exports.gray11 = "#c8c8c8"; // Backgrounds / Gray / Gray-12 exports.gray12 = "#dbdbdb"; // Backgrounds / Gray / Gray-13 exports.gray13 = "#efefef"; // Backgrounds / Gray / Gray-14 (Xxlight) exports.gray14 = "#fafafa"; // Transparents / Gray / 00-Gray-Light (70%) exports.grayLight = "rgba(118,118,118,0.7)"; // Transparents / Gray / 01-Gray-Xlight (40%) exports.grayXLight = "rgba(118,118,118,0.4)"; // Transparents / Gray / 02-Gray-Xxlight (10%) exports.grayXXLight = "rgba(118,118,118,0.1)"; // Backgrounds / Green / Green-00 (Xxdark) exports.green00 = "#121e00"; // Backgrounds / Green / Green-01 exports.green01 = "#192a07"; // Backgrounds / Green / Green-02 exports.green02 = "#28400f"; // Backgrounds / Green / Green-03 (Xdark) exports.green03 = "#385714"; // Backgrounds / Green / Green-04 exports.green04 = "#4c731a"; // Backgrounds / Green / Green-05 (Dark) exports.green05 = "#61911f"; // Backgrounds / Green / Green-06 exports.green06 = "#76af25"; // Backgrounds / Green / -Green-07 (Base) exports.green07 = "#8ccf2a"; // Backgrounds / Green / Green-08 exports.green08 = "#a1d753"; // Backgrounds / Green / Green-09 (Light) exports.green09 = "#b4de74"; // Backgrounds / Green / Green-10 exports.green10 = "#c6e593"; // Backgrounds / Green / Green-11 (Xlight) exports.green11 = "#d7edb2"; // Backgrounds / Green / Green-12 exports.green12 = "#e5f3cd"; // Backgrounds / Green / Green-13 exports.green13 = "#f3f9e8"; // Backgrounds / Green / Green-14 (Xxlight) exports.green14 = "#fbfdf7"; // Transparents / Green / 00-Green-Light (70%) exports.greenLight = "rgba(140,207,42,0.7)"; // Transparents / Green / 01-Green-Xlight (40%) exports.greenXLight = "rgba(140,207,42,0.4)"; // Transparents / Green / 02-Green-Xxlight (10%) exports.greenXXLight = "rgba(140,207,42,0.1)"; // Backgrounds / Indigo / Indigo-00 (Xxdark) exports.indigo00 = "#00071f"; // Backgrounds / Indigo / Indigo-01 exports.indigo01 = "#07122c"; // Backgrounds / Indigo / Indigo-02 exports.indigo02 = "#0f1e44"; // Backgrounds / Indigo / Indigo-03 (Xdark) exports.indigo03 = "#192b5e"; // Backgrounds / Indigo / Indigo-04 exports.indigo04 = "#24397e"; // Backgrounds / Indigo / Indigo-05 (Dark) exports.indigo05 = "#30499f"; // Backgrounds / Indigo / Indigo-06 exports.indigo06 = "#3c59c1"; // Backgrounds / Indigo / -Indigo-07 (Base) exports.indigo07 = "#4969e4"; // Backgrounds / Indigo / Indigo-08 exports.indigo08 = "#6f7ee9"; // Backgrounds / Indigo / Indigo-09 (Light) exports.indigo09 = "#8e94ed"; // Backgrounds / Indigo / Indigo-10 exports.indigo10 = "#a9abf1"; // Backgrounds / Indigo / Indigo-11 (Xlight) exports.indigo11 = "#c2c2f5"; // Backgrounds / Indigo / Indigo-12 exports.indigo12 = "#d7d7f8"; // Backgrounds / Indigo / Indigo-13 exports.indigo13 = "#ebedfb"; // Backgrounds / Indigo / Indigo-14 (Xxlight) exports.indigo14 = "#f7f9fd"; // Transparents / Indigo / 00-Indigo-Light (70%) exports.indigoLight = "rgba(73,105,228,0.7)"; // Transparents / Indigo / 01-Indigo-Xlight (40%) exports.indigoXLight = "rgba(73,105,228,0.4)"; // Transparents / Indigo / 02-Indigo-Xxlight (10%) exports.indigoXXLight = "rgba(73,105,228,0.1)"; // Backgrounds / Lime / Lime-00 (Xxdark) exports.lime00 = "#001f06"; // Backgrounds / Lime / Lime-01 exports.lime01 = "#05290f"; // Backgrounds / Lime / Lime-02 exports.lime02 = "#0c3c19"; // Backgrounds / Lime / Lime-03 (Xdark) exports.lime03 = "#145124"; // Backgrounds / Lime / Lime-04 exports.lime04 = "#1f6930"; // Backgrounds / Lime / Lime-05 (Dark) exports.lime05 = "#2a833c"; // Backgrounds / Lime / Lime-06 exports.lime06 = "#359e4a"; // Backgrounds / Lime / -Lime-07 (Base) exports.lime07 = "#41b957"; // Backgrounds / Lime / Lime-08 exports.lime08 = "#65c470"; // Backgrounds / Lime / Lime-09 (Light) exports.lime09 = "#84d08a"; // Backgrounds / Lime / Lime-10 exports.lime10 = "#a0dba3"; // Backgrounds / Lime / Lime-11 (Xlight) exports.lime11 = "#bbe5bd"; // Backgrounds / Lime / Lime-12 exports.lime12 = "#d2efd4"; // Backgrounds / Lime / Lime-13 exports.lime13 = "#e9f8eb"; // Backgrounds / Lime / Lime-14 (Xxlight) exports.lime14 = "#f6fdf8"; // Transparents / Lime / 00-Lime-Light (70%) exports.limeLight = "rgba(65,185,87,0.7)"; // Transparents / Lime / 01-Lime-Xlight (40%) exports.limeXLight = "rgba(65,185,87,0.4)"; // Transparents / Lime / 02-Lime-Xxlight (10%) exports.limeXXLight = "rgba(65,185,87,0.1)"; // Backgrounds / Orange / Orange-00 (Xxdark) exports.orange00 = "#1e0c00"; // Backgrounds / Orange / Orange-01 exports.orange01 = "#2b1505"; // Backgrounds / Orange / Orange-02 exports.orange02 = "#46210d"; // Backgrounds / Orange / Orange-03 (Xdark) exports.orange03 = "#622e10"; // Backgrounds / Orange / Orange-04 exports.orange04 = "#853d12"; // Backgrounds / Orange / Orange-05 (Dark) exports.orange05 = "#a94d14"; // Backgrounds / Orange / Orange-06 exports.orange06 = "#cf5d14"; // Backgrounds / Orange / -Orange-07 (Base) exports.orange07 = "#f66e13"; // Backgrounds / Orange / Orange-08 exports.orange08 = "#fd853f"; // Backgrounds / Orange / Orange-09 (Light) exports.orange09 = "#ff9c62"; // Backgrounds / Orange / Orange-10 exports.orange10 = "#ffb284"; // Backgrounds / Orange / Orange-11 (Xlight) exports.orange11 = "#ffc8a7"; // Backgrounds / Orange / Orange-12 exports.orange12 = "#ffdbc5"; // Backgrounds / Orange / Orange-13 exports.orange13 = "#ffeee5"; // Backgrounds / Orange / Orange-14 (Xxlight) exports.orange14 = "#fdf9f7"; // Transparents / Orange / 00-Orange-Light (70%) exports.orangeLight = "rgba(246,110,19,0.7)"; // Transparents / Orange / 01-Orange-Xlight (40%) exports.orangeXLight = "rgba(246,110,19,0.4)"; // Transparents / Orange / 02-Orange-Xxlight (10%) exports.orangeXXLight = "rgba(246,110,19,0.1)"; // Backgrounds / Pear / Pear-00 (Xxdark) exports.pear00 = "#1e1d00"; // Backgrounds / Pear / Pear-01 exports.pear01 = "#2a2a07"; // Backgrounds / Pear / Pear-02 exports.pear02 = "#42410e"; // Backgrounds / Pear / Pear-03 (Xdark) exports.pear03 = "#5d5a12"; // Backgrounds / Pear / Pear-04 exports.pear04 = "#7c7815"; // Backgrounds / Pear / Pear-05 (Dark) exports.pear05 = "#9d9718"; // Backgrounds / Pear / Pear-06 exports.pear06 = "#bfb71b"; // Backgrounds / Pear / -Pear-07 (Base) exports.pear07 = "#e3d91c"; // Backgrounds / Pear / Pear-08 exports.pear08 = "#eade4f"; // Backgrounds / Pear / Pear-09 (Light) exports.pear09 = "#f0e472"; // Backgrounds / Pear / Pear-10 exports.pear10 = "#f6e993"; // Backgrounds / Pear / Pear-11 (Xlight) exports.pear11 = "#f9efb2"; // Backgrounds / Pear / Pear-12 exports.pear12 = "#fcf4cd"; // Backgrounds / Pear / Pear-13 exports.pear13 = "#fdf9e8"; // Backgrounds / Pear / Pear-14 (Xxlight) exports.pear14 = "#fdfcf7"; // Transparents / Pear / 00-Pear-Light (70%) exports.pearLight = "rgba(227,217,28,0.7)"; // Transparents / Pear / 01-Pear-Xlight (40%) exports.pearXLight = "rgba(227,217,28,0.4)"; // Transparents / Pear / 02-Pear-Xxlight (10%) exports.pearXXLight = "rgba(227,217,28,0.1)"; // Backgrounds / Pink / Pink-00 (Xxdark) exports.pink00 = "#1e000a"; // Backgrounds / Pink / Pink-01 exports.pink01 = "#280a14"; // Backgrounds / Pink / Pink-02 exports.pink02 = "#3f1221"; // Backgrounds / Pink / Pink-03 (Xdark) exports.pink03 = "#58192f"; // Backgrounds / Pink / Pink-04 exports.pink04 = "#75223f"; // Backgrounds / Pink / Pink-05 (Dark) exports.pink05 = "#942b50"; // Backgrounds / Pink / Pink-06 exports.pink06 = "#b53461"; // Backgrounds / Pink / -Pink-07 (Base) exports.pink07 = "#d63d73"; // Backgrounds / Pink / Pink-08 exports.pink08 = "#e06187"; // Backgrounds / Pink / Pink-09 (Light) exports.pink09 = "#e87f9c"; // Backgrounds / Pink / Pink-10 exports.pink10 = "#f09cb1"; // Backgrounds / Pink / Pink-11 (Xlight) exports.pink11 = "#f5b8c6"; // Backgrounds / Pink / Pink-12 exports.pink12 = "#f9d1da"; // Backgrounds / Pink / Pink-13 exports.pink13 = "#fce9ee"; // Backgrounds / Pink / Pink-14 (Xxlight) exports.pink14 = "#fdf7f9"; // Transparents / Pink / 00-Pink-Light (70%) exports.pinkLight = "rgba(214,61,115,0.7)"; // Transparents / Pink / 01-Pink-Xlight (40%) exports.pinkXLight = "rgba(214,61,115,0.4)"; // Transparents / Pink / 02-Pink-Xxlight (10%) exports.pinkXXLight = "rgba(214,61,115,0.1)"; // Backgrounds / Red / Red-00 (Xxdark) exports.red00 = "#1e0002"; // Backgrounds / Red / Red-01 exports.red01 = "#2a0805"; // Backgrounds / Red / Red-02 exports.red02 = "#420e0b"; // Backgrounds / Red / Red-03 (Xdark) exports.red03 = "#5d110f"; // Backgrounds / Red / Red-04 exports.red04 = "#7d1311"; // Backgrounds / Red / Red-05 (Dark) exports.red05 = "#9e1313"; // Backgrounds / Red / Red-06 exports.red06 = "#c11014"; // Backgrounds / Red / -Red-07 (Base) exports.red07 = "#e50914"; // Backgrounds / Red / Red-08 exports.red08 = "#f04c38"; // Backgrounds / Red / Red-09 (Light) exports.red09 = "#f9715a"; // Backgrounds / Red / Red-10 exports.red10 = "#ff927d"; // Backgrounds / Red / Red-11 (Xlight) exports.red11 = "#ffb2a2"; // Backgrounds / Red / Red-12 exports.red12 = "#ffcdc3"; // Backgrounds / Red / Red-13 exports.red13 = "#ffe8e4"; // Backgrounds / Red / Red-14 (Xxlight) exports.red14 = "#fdf7f8"; // Transparents / Red / 00-Red-Light (70%) exports.redLight = "rgba(229,9,20,0.7)"; // Transparents / Red / 01-Red-Xlight (40%) exports.redXLight = "rgba(229,9,20,0.4)"; // Transparents / Red / 02-Red-Xxlight (10%) exports.redXXLight = "rgba(229,9,20,0.1)"; // Backgrounds / Violet / Violet-00 (Xxdark) exports.violet00 = "#08001e"; // Backgrounds / Violet / Violet-01 exports.violet01 = "#110b2b"; // Backgrounds / Violet / Violet-02 exports.violet02 = "#1d1643"; // Backgrounds / Violet / Violet-03 (Xdark) exports.violet03 = "#2a1f5d"; // Backgrounds / Violet / Violet-04 exports.violet04 = "#3b297c"; // Backgrounds / Violet / Violet-05 (Dark) exports.violet05 = "#4c349d"; // Backgrounds / Violet / Violet-06 exports.violet06 = "#5e3fbf"; // Backgrounds / Violet / -Violet-07 (Base) exports.violet07 = "#714be2"; // Backgrounds / Violet / Violet-08 exports.violet08 = "#8c66e7"; // Backgrounds / Violet / Violet-09 (Light) exports.violet09 = "#a481ec"; // Backgrounds / Violet / Violet-10 exports.violet10 = "#ba9cf1"; // Backgrounds / Violet / Violet-11 (Xlight) exports.violet11 = "#ceb8f5"; // Backgrounds / Violet / Violet-12 exports.violet12 = "#dfd0f8"; // Backgrounds / Violet / Violet-13 exports.violet13 = "#f0e9fb"; // Backgrounds / Violet / Violet-14 (Xxlight) exports.violet14 = "#f9f7fd"; // Transparents / Violet / 00-Violet-Light (70%) exports.violetLight = "rgba(113,75,226,0.7)"; // Transparents / Violet / 01-Violet-Xlight (40%) exports.violetXLight = "rgba(113,75,226,0.4)"; // Transparents / Violet / 02-Violet-Xxlight (10%) exports.violetXXLight = "rgba(113,75,226,0.1)"; // Backgrounds / White exports.white = "#FFFFFF"; // Transparents / White / 00-White-Light (70%) exports.whiteLight = "rgba(255,255,255,0.7)"; // Transparents / White / 01-White-Xlight (40%) exports.whiteXLight = "rgba(255,255,255,0.4)"; // Transparents / White / 02-White-Xxlight (10%) exports.whiteXXLight = "rgba(255,255,255,0.1)"; // Backgrounds / Yellow / Yellow-00 (Xxdark) exports.yellow00 = "#1e1400"; // Backgrounds / Yellow / Yellow-01 exports.yellow01 = "#2c1e06"; // Backgrounds / Yellow / Yellow-02 exports.yellow02 = "#47300d"; // Backgrounds / Yellow / Yellow-03 (Xdark) exports.yellow03 = "#64430f"; // Backgrounds / Yellow / Yellow-04 exports.yellow04 = "#875a11"; // Backgrounds / Yellow / Yellow-05 (Dark) exports.yellow05 = "#ac7210"; // Backgrounds / Yellow / Yellow-06 exports.yellow06 = "#d38a0c"; // Backgrounds / Yellow / -Yellow-07 (Base) exports.yellow07 = "#fba404"; // Backgrounds / Yellow / Yellow-08 exports.yellow08 = "#ffb141"; // Backgrounds / Yellow / Yellow-09 (Light) exports.yellow09 = "#ffbf66"; // Backgrounds / Yellow / Yellow-10 exports.yellow10 = "#ffcd89"; // Backgrounds / Yellow / Yellow-11 (Xlight) exports.yellow11 = "#ffdbaa"; // Backgrounds / Yellow / Yellow-12 exports.yellow12 = "#ffe7c8"; // Backgrounds / Yellow / Yellow-13 exports.yellow13 = "#fff4e6"; // Backgrounds / Yellow / Yellow-14 (Xxlight) exports.yellow14 = "#fdfbf7"; // Transparents / Yellow / 00-Yellow-Light (70%) exports.yellowLight = "rgba(251,164,4,0.7)"; // Transparents / Yellow / 01-Yellow-Xlight (40%) exports.yellowXLight = "rgba(251,164,4,0.4)"; // Transparents / Yellow / 02-Yellow-Xxlight (10%) exports.yellowXXLight = "rgba(251,164,4,0.1)"; ================================================ FILE: ui/src/theme/index.js ================================================ export { Provider as ThemeProvider } from "./provider"; export { default as theme } from "./theme"; ================================================ FILE: ui/src/theme/provider.jsx ================================================ import React from "react"; import { MuiThemeProvider } from "@material-ui/core/styles"; import { theme } from "./"; export const Provider = ({ children, ...rest }) => { return ( {children} ); }; ================================================ FILE: ui/src/theme/theme.js ================================================ import { unstable_createMuiStrictModeTheme as createMuiTheme } from "@material-ui/core/styles"; import { borders, colors, spacings, breakpoints, fontSizes, lineHeights, fontWeights, fontFamily, } from "./variables"; function toNumber(v) { return parseFloat(v); } const spacingFn = (factor) => { const unit = toNumber(spacings.space0); // Support theme.spacing('space3') if (typeof factor === "string") { return toNumber(spacings[factor]); } if (typeof factor === "number") { // Support theme.spacing(2) return unit * factor; } return unit; }; const colorFn = (color) => colors[color]; const baseThemeOptions = { palette: { type: "light", primary: { main: colors.brand, light: colors.bgBrandLight, dark: colors.bgBrandDark, contrastText: colors.white, }, secondary: { main: colors.white, light: colors.bgBrandLight, dark: colors.bgBrandDark, contrastText: colors.black, }, text: { primary: colors.black, secondary: colors.blackXLight, disabled: colors.blackXXLight, hint: colors.blackXXLight, }, grey: { 50: colors.gray14, 100: colors.gray13, 200: colors.gray12, 300: colors.gray11, 400: colors.gray10, 500: colors.gray09, 600: colors.gray07, 700: colors.gray06, 800: colors.gray04, 900: colors.gray02, A100: colors.gray12, A200: colors.gray08, A400: colors.gray03, A700: colors.gray06, }, error: { main: colors.failure, light: colors.failureLight, dark: colors.failureDark, contrastText: colors.white, }, background: { paper: colors.white, default: colors.gray14, }, divider: colors.blackXXLight, }, typography: { fontFamily: fontFamily.fontFamilySans, fontSize: toNumber(fontSizes.fontSize2), htmlFontSize: toNumber(fontSizes.fontSize2), fontWeightLight: fontWeights.fontWeight0, fontWeightRegular: fontWeights.fontWeight0, fontWeightMedium: fontWeights.fontWeight1, fontWeightBold: fontWeights.fontWeight2, h1: { fontSize: fontSizes.fontSize10, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, h2: { fontSize: fontSizes.fontSize9, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, h3: { fontSize: fontSizes.fontSize8, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, h4: { fontSize: fontSizes.fontSize7, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, h5: { fontSize: fontSizes.fontSize6, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, h6: { fontSize: fontSizes.fontSize5, lineHeight: lineHeights.lineHeight0, fontWeight: fontWeights.fontWeight2, }, body1: { fontSize: fontSizes.fontSize4, lineHeight: lineHeights.lineHeight1, }, body2: { fontSize: fontSizes.fontSize3, lineHeight: lineHeights.lineHeight1, }, caption: { fontSize: fontSizes.fontSize2, lineHeight: lineHeights.lineHeight1, fontWeight: fontWeights.fontWeight1, }, button: { fontSize: fontSizes.fontSize2, fontWeight: fontWeights.fontWeight1, }, }, breakpoints: { // this looks wrong, but it's not // material's breakpoints are a range, so the below basically says // xs is from 0 to breakpoints.large values: { xs: 0, sm: toNumber(breakpoints.xsmall), md: toNumber(breakpoints.small), lg: toNumber(breakpoints.medium), xl: toNumber(breakpoints.large), }, }, shape: { borderRadius: toNumber(borders.radiusSmall), }, color: colorFn, spacing: spacingFn, props: { MuiButtonBase: { disableRipple: true, }, MuiFormControl: { variant: "outlined", }, MuiMenu: { transitionDuration: 0, elevation: 3, }, MuiTextField: { variant: "outlined", InputProps: { labelWidth: 0, }, }, MuiInputLabel: { shrink: true, disableAnimation: true, }, MuiOutlinedInput: { notched: false, }, MuiPaper: { elevation: 3, }, MuiPopover: { elevation: 3, }, }, }; const baseTheme = createMuiTheme(baseThemeOptions); // Keep overrides in separate object so we can reference attributes of baseTheme. const overrides = { overrides: { MuiSvgIcon: { root: { fontSize: fontSizes.fontSize6, }, fontSizeSmall: { fontSize: fontSizes.fontSize1, }, }, MuiAvatar: { root: { fontSize: "2.4rem", }, }, MuiButton: { root: { textDecoration: "none !important", textTransform: "none", paddingTop: baseTheme.spacing("space1"), paddingBottom: baseTheme.spacing("space1"), paddingLeft: baseTheme.spacing("space2"), paddingRight: baseTheme.spacing("space2"), border: "1px solid transparent", transition: "none", "&$focusVisible": { boxShadow: "none", position: "relative", "&:after": { content: '""', display: "block", position: "absolute", width: "calc(100% + 6px)", height: "calc(100% + 6px)", borderRadius: borders.radiusSmall, border: borders.blueRegular2px, top: -5, left: -5, }, }, }, text: { paddingTop: baseTheme.spacing("space1"), paddingBottom: baseTheme.spacing("space1"), paddingLeft: baseTheme.spacing("space2"), paddingRight: baseTheme.spacing("space2"), "&:hover": { backgroundColor: baseTheme.palette.grey.A100, }, }, textSizeSmall: { fontSize: "0.8125rem", }, outlined: { paddingTop: baseTheme.spacing("space1"), paddingBottom: baseTheme.spacing("space1"), paddingLeft: baseTheme.spacing("space2"), paddingRight: baseTheme.spacing("space2"), }, outlinedPrimary: { border: borders.blackRegular1px, }, outlinedSecondary: { border: borders.blackLight1px, color: baseTheme.palette.secondary.contrastText, "&:hover": { border: borders.blackLight1px + " !important", backgroundColor: baseTheme.palette.grey.A100, }, }, contained: { "&:disabled": { backgroundColor: colors.bgBrandLight, color: baseTheme.palette.common.white, }, boxShadow: "none !important", "&:active": { boxShadow: "none !important", }, }, containedPrimary: { color: `${colors.white} !important`, }, }, MuiCheckbox: { root: { fontSize: fontSizes.fontSize4, padding: baseTheme.spacing("space1"), }, colorSecondary: { color: colors.blackLight, "&$checked": { color: baseTheme.palette.primary.main, }, "&$disabled": { color: colors.blackXLight, }, }, }, MuiChip: { root: { borderRadius: borders.radiusSmall, height: 24, fontSize: fontSizes.fontSize2, fontWeight: fontWeights.fontWeight1, }, label: { paddingLeft: baseTheme.spacing("space1"), paddingRight: baseTheme.spacing("space1"), }, sizeSmall: { fontSize: fontSizes.fontSize0, height: 20, }, deleteIcon: { height: "100%", padding: 3, margin: 0, backgroundColor: "rgba(5, 5, 5, 0.1)", borderRadius: `0 ${borders.radiusSmall} ${borders.radiusSmall} 0`, width: 24, boxSizing: "border-box", textAlign: "center", fill: baseTheme.palette.common.white, borderLeftWidth: 1, borderLeftStyle: "solid", borderLeftColor: "rgba(5, 5, 5, 0.1)", }, deleteIconColorPrimary: { color: colors.white, }, colorSecondary: { color: colors.white, backgroundColor: colors.lime07, }, }, MuiRadio: { root: { padding: baseTheme.spacing("space1"), }, }, MuiInputBase: { root: { fontSize: fontSizes.fontSize2, }, input: { "&[type=number]::-webkit-inner-spin-button ": { appearance: "none", margin: 0, }, }, }, MuiOutlinedInput: { notchedOutline: { borderColor: colors.blackXXLight, top: 0, "& legend": { // force-disable notched legends display: "none", }, }, root: { "&:hover $notchedOutline": { borderColor: colors.blackXXLight, }, "&.$Mui-disabled": { backgroundColor: colors.grayXXLight, borderColor: colors.blackXXLight, color: colors.blackLight, }, "&.$Mui-disabled .MuiOutlinedInput-notchedOutline": { borderColor: "inherit", }, backgroundColor: baseTheme.palette.background.paper, }, input: { padding: `${baseTheme.spacing("space2")}px ${baseTheme.spacing( "space2" )}px`, }, }, MuiFormControl: { root: { display: "block", }, }, MuiFormControlLabel: { label: { fontSize: fontSizes.fontSize3, lineHeight: lineHeights.lineHeight1, }, }, MuiInputLabel: { root: { display: "none", pointerEvents: "none", color: baseTheme.palette.text.primary, "&.$Mui-disabled": { color: colors.blackXLight, }, }, outlined: { "&$shrink": { display: "block", transform: "none", position: "relative", fontWeight: fontWeights.fontWeight1, fontSize: fontSizes.fontSize2, paddingLeft: 0, paddingBottom: 8, }, "&$focused": { // focused attr under MuiInputLabel does not work color: baseTheme.palette.text.primary, }, }, }, MuiFormHelperText: { contained: { margin: 0, marginTop: baseTheme.spacing("space1"), }, }, MuiSelect: { icon: { fontSize: fontSizes.fontSize5, marginTop: 3, color: baseTheme.palette.text.primary, }, selectMenu: {}, }, MuiPickersClockNumber: { clockNumber: { top: 6, }, }, MuiMenuItem: { root: { color: baseTheme.palette.text.primary, fontSize: fontSizes.fontSize1, "&:hover": { backgroundColor: baseTheme.palette.grey[100], }, "&:focus": { backgroundColor: baseTheme.palette.grey[100], }, "&$selected": { backgroundColor: baseTheme.palette.grey[200], "&:hover": { backgroundColor: baseTheme.palette.grey[200], }, "&:focus": { backgroundColor: baseTheme.palette.grey[200], }, }, }, dense: { paddingTop: 0, paddingBottom: 0, }, }, MuiSnackbarContent: { root: { backgroundColor: baseTheme.palette.primary.main, paddingTop: 0, paddingBottom: 0, marginRight: baseTheme.spacing("space3"), marginLeft: baseTheme.spacing("space3"), borderRadius: baseTheme.shape.borderRadius, boxShadow: "none", }, action: { "& button": { color: baseTheme.palette.common.white, }, }, }, MuiSwitch: { root: { padding: 0, height: 20, width: 40, "&:hover": { "& > $track": { backgroundColor: colors.gray05, }, "& > $checked + $track": { backgroundColor: colors.brand05, }, }, }, thumb: { borderRadius: 8, width: 16, height: 16, boxShadow: "0px 1px 2px 0px rgba(0, 0, 0, 0.4), 0px 0px 1px 0px rgba(0, 0, 0, 0.4)", }, track: { backgroundColor: colors.gray07, borderRadius: 10, opacity: 1, }, switchBase: { padding: 2, "&$checked": { transform: "translateX(100%)", "& + $track": { opacity: 1, }, }, }, colorPrimary: { "&$checked": { color: baseTheme.palette.common.white, }, "&$checked + $track": { backgroundColor: baseTheme.palette.primary.main, }, }, }, MuiTab: { root: { textTransform: "none", "&$selected": { color: "black", }, }, }, MuiTabs: { indicator: { height: 4, }, root: { minHeight: 0, }, }, MuiListItemText: { secondary: { fontSize: fontSizes.fontSize2, }, primary: { fontSize: fontSizes.fontSize2, }, }, MuiListSubheader: { root: { fontSize: fontSizes.fontSize2, lineHeight: lineHeights.lineHeight1, paddingTop: baseTheme.spacing("space0"), paddingBottom: baseTheme.spacing("space0"), }, }, MuiTableCell: { root: { fontSize: fontSizes.fontSize2, }, head: { //border: 'none', fontWeight: fontWeights.fontWeight1, color: colors.gray05, }, }, MuiTableRow: { root: { "&.Mui-selected:hover": { backgroundColor: colors.gray12, }, "&.Mui-selected": { backgroundColor: `${colors.gray12} !important`, }, }, }, MuiDialogTitle: { root: { backgroundColor: baseTheme.palette.grey[50], padding: `${baseTheme.spacing("space5")}px ${baseTheme.spacing( "space4" )}px`, borderBottom: `1px solid ${colors.blackXXLight}`, }, }, MuiDialogContent: { root: { padding: baseTheme.spacing("space5"), }, }, MuiDialogActions: { root: { backgroundColor: baseTheme.palette.grey[50], padding: `${baseTheme.spacing("space3")}px ${baseTheme.spacing( "space5" )}px`, borderTop: `1px solid ${colors.blackXXLight}`, margin: 0, "button + button": { marginLeft: baseTheme.spacing("space1"), }, }, }, MuiToolbar: { root: { gap: 8, }, }, MuiAppBar: { colorPrimary: { backgroundColor: colors.white, color: colors.gray00, }, root: { zIndex: 999, paddingLeft: 20, paddingRight: 20, boxShadow: "0 4px 8px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%)", height: 80, "& .MuiButton-label": { color: colors.black, }, "& .MuiLink-underlineHover:hover": { textDecoration: "none !important", }, }, }, MuiAutocomplete: { input: { padding: "12px 16px !important", }, paper: { fontSize: fontSizes.fontSize2, }, popupIndicator: { fontSize: fontSizes.fontSize5, color: baseTheme.palette.text.primary, }, clearIndicator: { fontSize: fontSizes.fontSize5, }, inputRoot: { padding: "0px !important", }, listbox: { backgroundColor: baseTheme.palette.common.white, }, tag: { "&:first-child": { marginLeft: 8, }, }, }, MuiTablePagination: { select: { paddingRight: "32px !important", }, selectRoot: { top: 1, }, }, }, }; const finalTheme = createMuiTheme({ ...baseTheme, ...overrides, }); export default finalTheme; ================================================ FILE: ui/src/theme/variables.js ================================================ export { colorOverrides as colors } from "./colorOverrides"; export const fontSizes = { fontSize0: "10px", fontSize1: "12px", fontSize2: "13px", fontSize3: "14px", fontSize4: "16px", fontSize5: "18px", fontSize6: "20px", fontSize7: "24px", fontSize8: "28px", fontSize9: "32px", fontSize10: "40px", fontSize11: "52px", fontSize12: "68px", fontSize13: "88px", }; export const lineHeights = { lineHeight0: 1.25, lineHeight1: 1.5, }; export const fontWeights = { fontWeight0: 400, fontWeight1: 600, fontWeight2: 700, fontWeight3: 800, }; export const fontFamily = { fontFamilySans: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"', fontFamilyMono: "monospace", }; export const spacings = { space0: "4px", space1: "8px", space2: "12px", space3: "16px", space4: "20px", space5: "24px", space6: "32px", space7: "48px", space8: "80px", space9: "144px", }; export const breakpoints = { xsmall: "599px", small: "1023px", medium: "1439px", large: "1919px", xlarge: "3840px", }; export const borders = { radiusSmall: "4px", blueRegular2px: "2px solid rgba(31,131,219,1)", blackRegular1px: "1px solid rgba(5,5,5,1)", blackLight1px: "1px solid rgba(5,5,5,0.7)", }; ================================================ FILE: ui/src/utils/constants.js ================================================ export const workflowStatuses = [ "RUNNING", "COMPLETED", "FAILED", "TIMED_OUT", "TERMINATED", "PAUSED", ]; export const TASK_STATUSES = [ "IN_PROGRESS", "CANCELED", "FAILED", "FAILED_WITH_TERMINAL_ERROR", "COMPLETED", "COMPLETED_WITH_ERRORS", "SCHEDULED", "TIMED_OUT", "SKIPPED", ]; export const TASK_TYPES = [ "ARCHER", "DECISION", "DO_WHILE", "DYNAMIC", "DYNIMO", "EAAS", "EVENT", "EXCLUSIVE_JOIN", "FORK_JOIN", "FORK_JOIN_DYNAMIC", "HTTP", "INLINE", "JOIN", "JSON_JQ_TRANSFORM", "LAMBDA", "SIMPLE", "SUB_WORKFLOW", "SWITCH", "TERMINATE", "TITUS", "TITUS_TASK", "WAIT", ]; export const SEARCH_TASK_TYPES_SET = modifyTaskTypes(TASK_TYPES); function modifyTaskTypes(taskTypes) { const newTaskTypes = taskTypes.filter( (ele) => ele !== "FORK_JOIN_DYNAMIC" && ele !== "SIMPLE" ); const fjIdx = newTaskTypes.findIndex((ele) => ele === "FORK_JOIN"); newTaskTypes[fjIdx] = "FORK"; return new Set(newTaskTypes); } ================================================ FILE: ui/src/utils/helpers.js ================================================ import { format, formatDuration, intervalToDuration } from "date-fns"; import _ from "lodash"; import packageJson from '../../package.json'; export function timestampRenderer(date) { if (_.isNil(date)) return null; const parsed = new Date(date); if (parsed.getTime() === 0) return null; // 0 epoch (UTC 1970-1-1) return format(parsed, "yyyy-MM-dd HH:mm:ss"); } export function timestampMsRenderer(date) { if (_.isNil(date)) return null; const parsed = new Date(date); if (parsed.getTime() === 0) return null; // 0 epoch (UTC 1970-1-1) return format(parsed, "yyyy-MM-dd HH:mm:ss.SSS"); } export function durationRenderer(durationMs) { const duration = intervalToDuration({ start: 0, end: durationMs }); if (durationMs > 5000) { return formatDuration(duration); } else { return `${durationMs}ms`; } } export function taskHasResult(task) { const keys = Object.keys(task); return !(keys.length === 1 && keys[0] === "workflowTask"); } export function astToQuery(node) { // leaf node if (node.operator !== undefined) { return node.field + node.operator + node.value; } else if (node.combinator !== undefined) { const clauses = node.rules .filter((rule) => !(rule.rules && rule.rules.length === 0)) // Ignore empty groups .map((rule) => astToQuery(rule)); const wrapper = clauses.length > 1; let combinator = node.combinator.toUpperCase(); return `${wrapper ? "(" : ""}${clauses.join(` ${combinator} `)}${ wrapper ? ")" : "" }`; } else { return ""; } } export function isFailedTask(status) { return ( status === "FAILED" || status === "FAILED_WITH_TERMINAL_ERROR" || status === "TIMED_OUT" || status === "CANCELED" ); } export function defaultCompare(x, y) { if (x === undefined && y === undefined) return 0; if (x === undefined) return 1; if (y === undefined) return -1; if (x < y) return -1; if (x > y) return 1; return 0; } export function immutableReplaceAt(array, index, value) { const ret = array.slice(0); ret[index] = value; return ret; } export function isEmptyIterable(iterable) { // eslint-disable-next-line no-unused-vars, no-unreachable-loop for (const _ of iterable) { return false; } return true; } export function getBasename() { let basename = '/'; try{ basename = new URL(packageJson.homepage).pathname; } catch(e) {} return _.isEmpty(basename) ? '/' : basename; } ================================================ FILE: ui/src/utils/localstorage.ts ================================================ import { useState } from "react"; // If key is null/undefined, hook behaves exactly like useState export const useLocalStorage = (key: string, initialValue: any) => { const initialString = JSON.stringify(initialValue); const [storedValue, setStoredValue] = useState(() => { if (key) { const item = window.localStorage.getItem(key); return item ? JSON.parse(item) : initialValue; } else { return initialValue; } }); const setValue = (value: any) => { // Allow value to be a function so we have same API as useState const valueToStore = value instanceof Function ? value(storedValue) : value; // Save state setStoredValue(valueToStore); if (key) { const stringToStore = JSON.stringify(valueToStore); if (stringToStore === initialString) { window.localStorage.removeItem(key); } else { window.localStorage.setItem(key, stringToStore); } } }; return [storedValue, setValue] as const; }; ================================================ FILE: ui/src/utils/path.js ================================================ import { isEmptyIterable } from "./helpers"; class Path { constructor(pathname) { this.search = new URLSearchParams(); this.pathname = pathname; } toString() { return ( this.pathname + (isEmptyIterable(this.search) ? "" : `?${this.search.toString()}`) ); } } export default Path; ================================================ FILE: ui/test-karbon.sh ================================================ ================================================ FILE: ui/tsconfig.json ================================================ { "compilerOptions": { "target": "es5", "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "esModuleInterop": true, "allowSyntheticDefaultImports": true, "strict": true, "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "module": "esnext", "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, "noEmit": true, "jsx": "react-jsx" }, "include": ["src"] }