\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *
\n * http://www.apache.org/licenses/LICENSE-2.0\n *
\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotations.protogen;\n\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * ProtoEnum annotates an enum type that will be exposed via the GRPC API as a native Protocol\n * Buffers enum.\n */\n@Retention(RetentionPolicy.RUNTIME)\n@Target(ElementType.TYPE)\npublic @interface ProtoEnum {}\n" }, { "path": "annotations/src/main/java/com/netflix/conductor/annotations/protogen/ProtoField.java", "content": "/*\n * Copyright 2022 Netflix, Inc.\n *
\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *
\n * http://www.apache.org/licenses/LICENSE-2.0\n *
\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotations.protogen;\n\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * ProtoField annotates a field inside an struct with metadata on how to expose it on its\n * corresponding Protocol Buffers struct. For a field to be exposed in a ProtoBuf struct, the\n * containing struct must also be annotated with a {@link ProtoMessage} or {@link ProtoEnum} tag.\n */\n@Retention(RetentionPolicy.RUNTIME)\n@Target(ElementType.FIELD)\npublic @interface ProtoField {\n /**\n * Mandatory. Sets the Protocol Buffer ID for this specific field. Once a field has been\n * annotated with a given ID, the ID can never change to a different value or the resulting\n * Protocol Buffer struct will not be backwards compatible.\n *\n * @return the numeric ID for the field\n */\n int id();\n}\n" }, { "path": "annotations/src/main/java/com/netflix/conductor/annotations/protogen/ProtoMessage.java", "content": "/*\n * Copyright 2022 Netflix, Inc.\n *
\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *
\n * http://www.apache.org/licenses/LICENSE-2.0\n *
\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotations.protogen;\n\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * ProtoMessage annotates a given Java class so it becomes exposed via the GRPC API as a native\n * Protocol Buffers struct. The annotated class must be a POJO.\n */\n@Retention(RetentionPolicy.RUNTIME)\n@Target(ElementType.TYPE)\npublic @interface ProtoMessage {\n /**\n * Sets whether the generated mapping code will contain a helper to translate the POJO for this\n * class into the equivalent ProtoBuf object.\n *\n * @return whether this class will generate a mapper to ProtoBuf objects\n */\n boolean toProto() default true;\n\n /**\n * Sets whether the generated mapping code will contain a helper to translate the ProtoBuf\n * object for this class into the equivalent POJO.\n *\n * @return whether this class will generate a mapper from ProtoBuf objects\n */\n boolean fromProto() default true;\n\n /**\n * Sets whether this is a wrapper class that will be used to encapsulate complex nested type\n * interfaces. Wrapper classes are not directly exposed by the ProtoBuf API and must be mapped\n * manually.\n *\n * @return whether this is a wrapper class\n */\n boolean wrapper() default false;\n}\n" }, { "path": "annotations-processor/README.md", "content": "[Annotations Processor](docs/docs/reference-docs/annotations-processor.md)" }, { "path": "annotations-processor/build.gradle", "content": "\nsourceSets {\n example\n}\n\ndependencies {\n implementation project(':conductor-annotations')\n api 'com.google.guava:guava:32.1.2-jre'\n api 'com.squareup:javapoet:1.13.+'\n api 'com.github.jknack:handlebars:4.3.+'\n api 'com.google.protobuf:protobuf-java:3.21.12'\n api 'javax.annotation:javax.annotation-api:1.3.2'\n api gradleApi()\n\n exampleImplementation sourceSets.main.output\n exampleImplementation project(':conductor-annotations')\n}\n\ntask exampleJar(type: Jar) {\n archiveFileName = 'example.jar'\n from sourceSets.example.output.classesDirs\n}\n\ntestClasses.finalizedBy(exampleJar)\n" }, { "path": "annotations-processor/src/example/java/com/example/Example.java", "content": "/*\n * Copyright 2022 Netflix, Inc.\n *
\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *
\n * http://www.apache.org/licenses/LICENSE-2.0\n *
\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.example;\n\nimport com.netflix.conductor.annotations.protogen.ProtoField;\nimport com.netflix.conductor.annotations.protogen.ProtoMessage;\n\n@ProtoMessage\npublic class Example {\n @ProtoField(id = 1)\n public String name;\n\n @ProtoField(id = 2)\n public Long count;\n}\n" }, { "path": "annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/AbstractMessage.java", "content": "/*\n * Copyright 2022 Netflix, Inc.\n *
\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *
\n * http://www.apache.org/licenses/LICENSE-2.0\n *
\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\n\nimport com.netflix.conductor.annotations.protogen.ProtoEnum;\nimport com.netflix.conductor.annotations.protogen.ProtoMessage;\nimport com.netflix.conductor.annotationsprocessor.protogen.types.MessageType;\nimport com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper;\n\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeSpec;\n\npublic abstract class AbstractMessage {\n protected Class clazz;\n protected MessageType type;\n protected List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport javax.lang.model.element.Modifier;\n\nimport com.netflix.conductor.annotationsprocessor.protogen.types.MessageType;\n\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\nimport com.squareup.javapoet.TypeSpec;\n\npublic class Enum extends AbstractMessage {\n public enum MapType {\n FROM_PROTO(\"fromProto\"),\n TO_PROTO(\"toProto\");\n\n private final String methodName;\n\n MapType(String m) {\n methodName = m;\n }\n\n public String getMethodName() {\n return methodName;\n }\n }\n\n public Enum(Class cls, MessageType parent) {\n super(cls, parent);\n\n int protoIndex = 0;\n for (java.lang.reflect.Field field : cls.getDeclaredFields()) {\n if (field.isEnumConstant()) fields.add(new EnumField(protoIndex++, field));\n }\n }\n\n @Override\n public String getProtoClass() {\n return \"enum\";\n }\n\n private MethodSpec javaMap(MapType mt, TypeName from, TypeName to) {\n MethodSpec.Builder method = MethodSpec.methodBuilder(mt.getMethodName());\n method.addModifiers(Modifier.PUBLIC);\n method.returns(to);\n method.addParameter(from, \"from\");\n\n method.addStatement(\"$T to\", to);\n method.beginControlFlow(\"switch (from)\");\n\n for (Field field : fields) {\n String fromName = (mt == MapType.TO_PROTO) ? field.getName() : field.getProtoName();\n String toName = (mt == MapType.TO_PROTO) ? field.getProtoName() : field.getName();\n method.addStatement(\"case $L: to = $T.$L; break\", fromName, to, toName);\n }\n\n method.addStatement(\n \"default: throw new $T(\\\"Unexpected enum constant: \\\" + from)\",\n IllegalArgumentException.class);\n method.endControlFlow();\n method.addStatement(\"return to\");\n return method.build();\n }\n\n @Override\n protected void javaMapFromProto(TypeSpec.Builder type) {\n type.addMethod(\n javaMap(\n MapType.FROM_PROTO,\n this.type.getJavaProtoType(),\n TypeName.get(this.clazz)));\n }\n\n @Override\n protected void javaMapToProto(TypeSpec.Builder type) {\n type.addMethod(\n javaMap(MapType.TO_PROTO, TypeName.get(this.clazz), this.type.getJavaProtoType()));\n }\n\n public class EnumField extends Field {\n protected EnumField(int index, java.lang.reflect.Field field) {\n super(index, field);\n }\n\n @Override\n public String getProtoTypeDeclaration() {\n return String.format(\"%s = %d\", getProtoName(), getProtoIndex());\n }\n }\n}\n"
},
{
"path": "annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/Message.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.util.Set;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport javax.lang.model.element.Modifier;\n\nimport com.netflix.conductor.annotations.protogen.ProtoField;\nimport com.netflix.conductor.annotations.protogen.ProtoMessage;\nimport com.netflix.conductor.annotationsprocessor.protogen.types.AbstractType;\nimport com.netflix.conductor.annotationsprocessor.protogen.types.MessageType;\nimport com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeSpec;\n\npublic class Message extends AbstractMessage {\n public Message(Class cls, MessageType parent) {\n super(cls, parent);\n\n for (java.lang.reflect.Field field : clazz.getDeclaredFields()) {\n ProtoField ann = field.getAnnotation(ProtoField.class);\n if (ann == null) continue;\n\n fields.add(new MessageField(ann.id(), field));\n }\n }\n\n protected ProtoMessage getAnnotation() {\n return (ProtoMessage) this.clazz.getAnnotation(ProtoMessage.class);\n }\n\n @Override\n public String getProtoClass() {\n return \"message\";\n }\n\n @Override\n protected void javaMapToProto(TypeSpec.Builder type) {\n if (!getAnnotation().toProto() || getAnnotation().wrapper()) return;\n\n ClassName javaProtoType = (ClassName) this.type.getJavaProtoType();\n MethodSpec.Builder method = MethodSpec.methodBuilder(\"toProto\");\n method.addModifiers(Modifier.PUBLIC);\n method.returns(javaProtoType);\n method.addParameter(this.clazz, \"from\");\n\n method.addStatement(\n \"$T to = $T.newBuilder()\", javaProtoType.nestedClass(\"Builder\"), javaProtoType);\n\n for (Field field : this.fields) {\n if (field instanceof MessageField) {\n AbstractType fieldType = ((MessageField) field).getAbstractType();\n fieldType.mapToProto(field.getName(), method);\n }\n }\n\n method.addStatement(\"return to.build()\");\n type.addMethod(method.build());\n }\n\n @Override\n protected void javaMapFromProto(TypeSpec.Builder type) {\n if (!getAnnotation().fromProto() || getAnnotation().wrapper()) return;\n\n MethodSpec.Builder method = MethodSpec.methodBuilder(\"fromProto\");\n method.addModifiers(Modifier.PUBLIC);\n method.returns(this.clazz);\n method.addParameter(this.type.getJavaProtoType(), \"from\");\n\n method.addStatement(\"$T to = new $T()\", this.clazz, this.clazz);\n\n for (Field field : this.fields) {\n if (field instanceof MessageField) {\n AbstractType fieldType = ((MessageField) field).getAbstractType();\n fieldType.mapFromProto(field.getName(), method);\n }\n }\n\n method.addStatement(\"return to\");\n type.addMethod(method.build());\n }\n\n public static class MessageField extends Field {\n protected AbstractType type;\n\n protected MessageField(int index, java.lang.reflect.Field field) {\n super(index, field);\n }\n\n public AbstractType getAbstractType() {\n if (type == null) {\n type = TypeMapper.INSTANCE.get(field.getGenericType());\n }\n return type;\n }\n\n private static Pattern CAMEL_CASE_RE = Pattern.compile(\"(?<=[a-z])[A-Z]\");\n\n private static String toUnderscoreCase(String input) {\n Matcher m = CAMEL_CASE_RE.matcher(input);\n StringBuilder sb = new StringBuilder();\n while (m.find()) {\n m.appendReplacement(sb, \"_\" + m.group());\n }\n m.appendTail(sb);\n return sb.toString().toLowerCase();\n }\n\n @Override\n public String getProtoTypeDeclaration() {\n return String.format(\n \"%s %s = %d\",\n getAbstractType().getProtoType(), toUnderscoreCase(getName()), getProtoIndex());\n }\n\n @Override\n public void getDependencies(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\nimport com.netflix.conductor.annotationsprocessor.protogen.types.TypeMapper;\n\nimport com.squareup.javapoet.ClassName;\n\npublic class ProtoFile {\n public static String PROTO_SUFFIX = \"Pb\";\n\n private ClassName baseClass;\n private AbstractMessage message;\n private String filePath;\n\n private String protoPackageName;\n private String javaPackageName;\n private String goPackageName;\n\n public ProtoFile(\n Class object,\n String protoPackageName,\n String javaPackageName,\n String goPackageName) {\n this.protoPackageName = protoPackageName;\n this.javaPackageName = javaPackageName;\n this.goPackageName = goPackageName;\n\n String className = object.getSimpleName() + PROTO_SUFFIX;\n this.filePath = \"model/\" + object.getSimpleName().toLowerCase() + \".proto\";\n this.baseClass = ClassName.get(this.javaPackageName, className);\n this.message = new Message(object, TypeMapper.INSTANCE.baseClass(baseClass, filePath));\n }\n\n public String getJavaClassName() {\n return baseClass.simpleName();\n }\n\n public String getFilePath() {\n return filePath;\n }\n\n public String getProtoPackageName() {\n return protoPackageName;\n }\n\n public String getJavaPackageName() {\n return javaPackageName;\n }\n\n public String getGoPackageName() {\n return goPackageName;\n }\n\n public AbstractMessage getMessage() {\n return message;\n }\n\n public Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.io.File;\nimport java.io.FileWriter;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.net.URL;\nimport java.net.URLClassLoader;\nimport java.util.*;\n\nimport javax.annotation.Generated;\nimport javax.lang.model.element.Modifier;\n\nimport com.netflix.conductor.annotations.protogen.ProtoMessage;\n\nimport com.github.jknack.handlebars.EscapingStrategy;\nimport com.github.jknack.handlebars.Handlebars;\nimport com.github.jknack.handlebars.Template;\nimport com.github.jknack.handlebars.io.ClassPathTemplateLoader;\nimport com.github.jknack.handlebars.io.TemplateLoader;\nimport com.google.common.reflect.ClassPath;\nimport com.squareup.javapoet.AnnotationSpec;\nimport com.squareup.javapoet.JavaFile;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeSpec;\n\npublic class ProtoGen {\n private static final String GENERATOR_NAME =\n \"com.netflix.conductor.annotationsprocessor.protogen\";\n\n private String protoPackageName;\n private String javaPackageName;\n private String goPackageName;\n private List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.io.File;\nimport java.io.IOException;\n\npublic class ProtoGenTask {\n private String protoPackage;\n private String javaPackage;\n private String goPackage;\n\n private File protosDir;\n private File mapperDir;\n private String mapperPackage;\n\n private File sourceJar;\n private String sourcePackage;\n\n public String getProtoPackage() {\n return protoPackage;\n }\n\n public void setProtoPackage(String protoPackage) {\n this.protoPackage = protoPackage;\n }\n\n public String getJavaPackage() {\n return javaPackage;\n }\n\n public void setJavaPackage(String javaPackage) {\n this.javaPackage = javaPackage;\n }\n\n public String getGoPackage() {\n return goPackage;\n }\n\n public void setGoPackage(String goPackage) {\n this.goPackage = goPackage;\n }\n\n public File getProtosDir() {\n return protosDir;\n }\n\n public void setProtosDir(File protosDir) {\n this.protosDir = protosDir;\n }\n\n public File getMapperDir() {\n return mapperDir;\n }\n\n public void setMapperDir(File mapperDir) {\n this.mapperDir = mapperDir;\n }\n\n public String getMapperPackage() {\n return mapperPackage;\n }\n\n public void setMapperPackage(String mapperPackage) {\n this.mapperPackage = mapperPackage;\n }\n\n public File getSourceJar() {\n return sourceJar;\n }\n\n public void setSourceJar(File sourceJar) {\n this.sourceJar = sourceJar;\n }\n\n public String getSourcePackage() {\n return sourcePackage;\n }\n\n public void setSourcePackage(String sourcePackage) {\n this.sourcePackage = sourcePackage;\n }\n\n public void generate() {\n ProtoGen generator = new ProtoGen(protoPackage, javaPackage, goPackage);\n try {\n generator.processPackage(sourceJar, sourcePackage);\n generator.writeMapper(mapperDir, mapperPackage);\n generator.writeProtos(protosDir);\n } catch (IOException e) {\n System.err.printf(\"protogen: failed with %s\\n\", e);\n }\n }\n\n public static void main(String[] args) {\n if (args == null || args.length < 8) {\n throw new RuntimeException(\n \"protogen configuration incomplete, please provide all required (8) inputs\");\n }\n ProtoGenTask task = new ProtoGenTask();\n int argsId = 0;\n task.setProtoPackage(args[argsId++]);\n task.setJavaPackage(args[argsId++]);\n task.setGoPackage(args[argsId++]);\n task.setProtosDir(new File(args[argsId++]));\n task.setMapperDir(new File(args[argsId++]));\n task.setMapperPackage(args[argsId++]);\n task.setSourceJar(new File(args[argsId++]));\n task.setSourcePackage(args[argsId]);\n System.out.println(\"Running protogen with arguments: \" + task);\n task.generate();\n System.out.println(\"protogen completed.\");\n }\n\n @Override\n public String toString() {\n return \"ProtoGenTask{\"\n + \"protoPackage='\"\n + protoPackage\n + '\\''\n + \", javaPackage='\"\n + javaPackage\n + '\\''\n + \", goPackage='\"\n + goPackage\n + '\\''\n + \", protosDir=\"\n + protosDir\n + \", mapperDir=\"\n + mapperDir\n + \", mapperPackage='\"\n + mapperPackage\n + '\\''\n + \", sourceJar=\"\n + sourceJar\n + \", sourcePackage='\"\n + sourcePackage\n + '\\''\n + '}';\n }\n}\n"
},
{
"path": "annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/AbstractType.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.Set;\n\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\n\npublic abstract class AbstractType {\n Type javaType;\n TypeName javaProtoType;\n\n AbstractType(Type javaType, TypeName javaProtoType) {\n this.javaType = javaType;\n this.javaProtoType = javaProtoType;\n }\n\n public Type getJavaType() {\n return javaType;\n }\n\n public TypeName getJavaProtoType() {\n return javaProtoType;\n }\n\n public abstract String getProtoType();\n\n public abstract TypeName getRawJavaType();\n\n public abstract void mapToProto(String field, MethodSpec.Builder method);\n\n public abstract void mapFromProto(String field, MethodSpec.Builder method);\n\n public abstract void getDependencies(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.Set;\n\nimport javax.lang.model.element.Modifier;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\n\npublic class ExternMessageType extends MessageType {\n private String externProtoType;\n\n public ExternMessageType(\n Type javaType, ClassName javaProtoType, String externProtoType, String protoFilePath) {\n super(javaType, javaProtoType, protoFilePath);\n this.externProtoType = externProtoType;\n }\n\n @Override\n public String getProtoType() {\n return externProtoType;\n }\n\n @Override\n public void generateAbstractMethods(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.Set;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\n\nabstract class GenericType extends AbstractType {\n public GenericType(Type type) {\n super(type, null);\n }\n\n protected Class getRawType() {\n ParameterizedType tt = (ParameterizedType) this.getJavaType();\n return (Class) tt.getRawType();\n }\n\n protected AbstractType resolveGenericParam(int idx) {\n ParameterizedType tt = (ParameterizedType) this.getJavaType();\n Type[] types = tt.getActualTypeArguments();\n\n AbstractType abstractType = TypeMapper.INSTANCE.get(types[idx]);\n if (abstractType instanceof GenericType) {\n return WrappedType.wrap((GenericType) abstractType);\n }\n return abstractType;\n }\n\n public abstract String getWrapperSuffix();\n\n public abstract AbstractType getValueType();\n\n public abstract TypeName resolveJavaProtoType();\n\n @Override\n public TypeName getRawJavaType() {\n return ClassName.get(getRawType());\n }\n\n @Override\n public void getDependencies(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.stream.Collectors;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.ParameterizedTypeName;\nimport com.squareup.javapoet.TypeName;\n\npublic class ListType extends GenericType {\n private AbstractType valueType;\n\n public ListType(Type type) {\n super(type);\n }\n\n @Override\n public String getWrapperSuffix() {\n return \"List\";\n }\n\n @Override\n public AbstractType getValueType() {\n if (valueType == null) {\n valueType = resolveGenericParam(0);\n }\n return valueType;\n }\n\n @Override\n public void mapToProto(String field, MethodSpec.Builder method) {\n AbstractType subtype = getValueType();\n if (subtype instanceof ScalarType) {\n method.addStatement(\n \"to.$L( from.$L() )\",\n protoMethodName(\"addAll\", field),\n javaMethodName(\"get\", field));\n } else {\n method.beginControlFlow(\n \"for ($T elem : from.$L())\",\n subtype.getJavaType(),\n javaMethodName(\"get\", field));\n method.addStatement(\"to.$L( toProto(elem) )\", protoMethodName(\"add\", field));\n method.endControlFlow();\n }\n }\n\n @Override\n public void mapFromProto(String field, MethodSpec.Builder method) {\n AbstractType subtype = getValueType();\n Type entryType = subtype.getJavaType();\n Class collector = TypeMapper.PROTO_LIST_TYPES.get(getRawType());\n\n if (subtype instanceof ScalarType) {\n if (entryType.equals(String.class)) {\n method.addStatement(\n \"to.$L( from.$L().stream().collect($T.toCollection($T::new)) )\",\n javaMethodName(\"set\", field),\n protoMethodName(\"get\", field) + \"List\",\n Collectors.class,\n collector);\n } else {\n method.addStatement(\n \"to.$L( from.$L() )\",\n javaMethodName(\"set\", field),\n protoMethodName(\"get\", field) + \"List\");\n }\n } else {\n method.addStatement(\n \"to.$L( from.$L().stream().map(this::fromProto).collect($T.toCollection($T::new)) )\",\n javaMethodName(\"set\", field),\n protoMethodName(\"get\", field) + \"List\",\n Collectors.class,\n collector);\n }\n }\n\n @Override\n public TypeName resolveJavaProtoType() {\n return ParameterizedTypeName.get(\n (ClassName) getRawJavaType(), getValueType().getJavaProtoType());\n }\n\n @Override\n public String getProtoType() {\n return \"repeated \" + getValueType().getProtoType();\n }\n}\n"
},
{
"path": "annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/MapType.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.ParameterizedTypeName;\nimport com.squareup.javapoet.TypeName;\n\npublic class MapType extends GenericType {\n private AbstractType keyType;\n private AbstractType valueType;\n\n public MapType(Type type) {\n super(type);\n }\n\n @Override\n public String getWrapperSuffix() {\n return \"Map\";\n }\n\n @Override\n public AbstractType getValueType() {\n if (valueType == null) {\n valueType = resolveGenericParam(1);\n }\n return valueType;\n }\n\n public AbstractType getKeyType() {\n if (keyType == null) {\n keyType = resolveGenericParam(0);\n }\n return keyType;\n }\n\n @Override\n public void mapToProto(String field, MethodSpec.Builder method) {\n AbstractType valueType = getValueType();\n if (valueType instanceof ScalarType) {\n method.addStatement(\n \"to.$L( from.$L() )\",\n protoMethodName(\"putAll\", field),\n javaMethodName(\"get\", field));\n } else {\n TypeName typeName =\n ParameterizedTypeName.get(\n Map.Entry.class,\n getKeyType().getJavaType(),\n getValueType().getJavaType());\n method.beginControlFlow(\n \"for ($T pair : from.$L().entrySet())\", typeName, javaMethodName(\"get\", field));\n method.addStatement(\n \"to.$L( pair.getKey(), toProto( pair.getValue() ) )\",\n protoMethodName(\"put\", field));\n method.endControlFlow();\n }\n }\n\n @Override\n public void mapFromProto(String field, MethodSpec.Builder method) {\n AbstractType valueType = getValueType();\n if (valueType instanceof ScalarType) {\n method.addStatement(\n \"to.$L( from.$L() )\",\n javaMethodName(\"set\", field),\n protoMethodName(\"get\", field) + \"Map\");\n } else {\n Type keyType = getKeyType().getJavaType();\n Type valueTypeJava = getValueType().getJavaType();\n TypeName valueTypePb = getValueType().getJavaProtoType();\n\n ParameterizedTypeName entryType =\n ParameterizedTypeName.get(\n ClassName.get(Map.Entry.class), TypeName.get(keyType), valueTypePb);\n ParameterizedTypeName mapType =\n ParameterizedTypeName.get(Map.class, keyType, valueTypeJava);\n ParameterizedTypeName hashMapType =\n ParameterizedTypeName.get(HashMap.class, keyType, valueTypeJava);\n String mapName = field + \"Map\";\n\n method.addStatement(\"$T $L = new $T()\", mapType, mapName, hashMapType);\n method.beginControlFlow(\n \"for ($T pair : from.$L().entrySet())\",\n entryType,\n protoMethodName(\"get\", field) + \"Map\");\n method.addStatement(\"$L.put( pair.getKey(), fromProto( pair.getValue() ) )\", mapName);\n method.endControlFlow();\n method.addStatement(\"to.$L($L)\", javaMethodName(\"set\", field), mapName);\n }\n }\n\n @Override\n public TypeName resolveJavaProtoType() {\n return ParameterizedTypeName.get(\n (ClassName) getRawJavaType(),\n getKeyType().getJavaProtoType(),\n getValueType().getJavaProtoType());\n }\n\n @Override\n public String getProtoType() {\n AbstractType keyType = getKeyType();\n AbstractType valueType = getValueType();\n if (!(keyType instanceof ScalarType)) {\n throw new IllegalArgumentException(\n \"cannot map non-scalar map key: \" + this.getJavaType());\n }\n return String.format(\"map<%s, %s>\", keyType.getProtoType(), valueType.getProtoType());\n }\n}\n"
},
{
"path": "annotations-processor/src/main/java/com/netflix/conductor/annotationsprocessor/protogen/types/MessageType.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.List;\nimport java.util.Set;\n\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\n\npublic class MessageType extends AbstractType {\n private String protoFilePath;\n\n public MessageType(Type javaType, ClassName javaProtoType, String protoFilePath) {\n super(javaType, javaProtoType);\n this.protoFilePath = protoFilePath;\n }\n\n @Override\n public String getProtoType() {\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.Set;\n\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\n\npublic class ScalarType extends AbstractType {\n private String protoType;\n\n public ScalarType(Type javaType, TypeName javaProtoType, String protoType) {\n super(javaType, javaProtoType);\n this.protoType = protoType;\n }\n\n @Override\n public String getProtoType() {\n return protoType;\n }\n\n @Override\n public TypeName getRawJavaType() {\n return getJavaProtoType();\n }\n\n @Override\n public void mapFromProto(String field, MethodSpec.Builder method) {\n method.addStatement(\n \"to.$L( from.$L() )\", javaMethodName(\"set\", field), protoMethodName(\"get\", field));\n }\n\n private boolean isNullableType() {\n final Type jt = getJavaType();\n return jt.equals(Boolean.class)\n || jt.equals(Byte.class)\n || jt.equals(Character.class)\n || jt.equals(Short.class)\n || jt.equals(Integer.class)\n || jt.equals(Long.class)\n || jt.equals(Double.class)\n || jt.equals(Float.class)\n || jt.equals(String.class);\n }\n\n @Override\n public void mapToProto(String field, MethodSpec.Builder method) {\n final boolean nullable = isNullableType();\n String getter =\n (getJavaType().equals(boolean.class) || getJavaType().equals(Boolean.class))\n ? javaMethodName(\"is\", field)\n : javaMethodName(\"get\", field);\n\n if (nullable) method.beginControlFlow(\"if (from.$L() != null)\", getter);\n\n method.addStatement(\"to.$L( from.$L() )\", protoMethodName(\"set\", field), getter);\n\n if (nullable) method.endControlFlow();\n }\n\n @Override\n public void getDependencies(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.*;\n\nimport com.google.protobuf.Any;\nimport com.squareup.javapoet.ClassName;\nimport com.squareup.javapoet.TypeName;\n\npublic class TypeMapper {\n static Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen.types;\n\nimport java.lang.reflect.Type;\nimport java.util.Set;\n\nimport javax.lang.model.element.Modifier;\n\nimport com.squareup.javapoet.MethodSpec;\nimport com.squareup.javapoet.TypeName;\n\npublic class WrappedType extends AbstractType {\n private AbstractType realType;\n private MessageType wrappedType;\n\n public static WrappedType wrap(GenericType realType) {\n Type valueType = realType.getValueType().getJavaType();\n if (!(valueType instanceof Class))\n throw new IllegalArgumentException(\"cannot wrap primitive type: \" + valueType);\n\n String className = ((Class) valueType).getSimpleName() + realType.getWrapperSuffix();\n MessageType wrappedType = TypeMapper.INSTANCE.get(className);\n if (wrappedType == null)\n throw new IllegalArgumentException(\"missing wrapper class: \" + className);\n return new WrappedType(realType, wrappedType);\n }\n\n public WrappedType(AbstractType realType, MessageType wrappedType) {\n super(realType.getJavaType(), wrappedType.getJavaProtoType());\n this.realType = realType;\n this.wrappedType = wrappedType;\n }\n\n @Override\n public String getProtoType() {\n return wrappedType.getProtoType();\n }\n\n @Override\n public TypeName getRawJavaType() {\n return realType.getRawJavaType();\n }\n\n @Override\n public void mapToProto(String field, MethodSpec.Builder method) {\n wrappedType.mapToProto(field, method);\n }\n\n @Override\n public void mapFromProto(String field, MethodSpec.Builder method) {\n wrappedType.mapFromProto(field, method);\n }\n\n @Override\n public void getDependencies(Set \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.annotationsprocessor.protogen;\n\nimport java.io.File;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\n\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\nimport com.google.common.collect.Lists;\nimport com.google.common.io.Files;\nimport com.google.common.io.Resources;\n\nimport static org.junit.Assert.*;\n\npublic class ProtoGenTest {\n private static final Charset charset = StandardCharsets.UTF_8;\n\n @Rule public TemporaryFolder folder = new TemporaryFolder();\n\n @Test\n public void happyPath() throws Exception {\n File rootDir = folder.getRoot();\n String protoPackage = \"protoPackage\";\n String javaPackage = \"abc.protogen.example\";\n String goPackage = \"goPackage\";\n String sourcePackage = \"com.example\";\n String mapperPackage = \"mapperPackage\";\n\n File jarFile = new File(\"./build/libs/example.jar\");\n assertTrue(jarFile.exists());\n\n File mapperDir = new File(rootDir, \"mapperDir\");\n mapperDir.mkdirs();\n\n File protosDir = new File(rootDir, \"protosDir\");\n protosDir.mkdirs();\n\n File modelDir = new File(protosDir, \"model\");\n modelDir.mkdirs();\n\n ProtoGen generator = new ProtoGen(protoPackage, javaPackage, goPackage);\n generator.processPackage(jarFile, sourcePackage);\n generator.writeMapper(mapperDir, mapperPackage);\n generator.writeProtos(protosDir);\n\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\n\ndependencies {\n implementation project(':conductor-common')\n implementation project(':conductor-core')\n compileOnly 'org.springframework.boot:spring-boot-starter'\n\n implementation \"com.amazonaws:aws-java-sdk-s3:${revAwsSdk}\"\n implementation \"org.apache.commons:commons-lang3\"\n}\n"
},
{
"path": "awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.s3.config;\n\nimport org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;\nimport org.springframework.boot.context.properties.EnableConfigurationProperties;\nimport org.springframework.context.annotation.Bean;\nimport org.springframework.context.annotation.Configuration;\n\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage;\nimport com.netflix.conductor.core.utils.IDGenerator;\nimport com.netflix.conductor.s3.storage.S3PayloadStorage;\n\nimport com.amazonaws.services.s3.AmazonS3;\nimport com.amazonaws.services.s3.AmazonS3ClientBuilder;\n\n@Configuration\n@EnableConfigurationProperties(S3Properties.class)\n@ConditionalOnProperty(name = \"conductor.external-payload-storage.type\", havingValue = \"s3\")\npublic class S3Configuration {\n\n @Bean\n public ExternalPayloadStorage s3ExternalPayloadStorage(\n IDGenerator idGenerator, S3Properties properties, AmazonS3 s3Client) {\n return new S3PayloadStorage(idGenerator, properties, s3Client);\n }\n\n @ConditionalOnProperty(\n name = \"conductor.external-payload-storage.s3.use_default_client\",\n havingValue = \"true\",\n matchIfMissing = true)\n @Bean\n public AmazonS3 amazonS3(S3Properties properties) {\n return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build();\n }\n}\n"
},
{
"path": "awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.s3.config;\n\nimport java.time.Duration;\nimport java.time.temporal.ChronoUnit;\n\nimport org.springframework.boot.context.properties.ConfigurationProperties;\nimport org.springframework.boot.convert.DurationUnit;\n\n@ConfigurationProperties(\"conductor.external-payload-storage.s3\")\npublic class S3Properties {\n\n /** The s3 bucket name where the payloads will be stored */\n private String bucketName = \"conductor_payloads\";\n\n /** The time (in seconds) for which the signed url will be valid */\n @DurationUnit(ChronoUnit.SECONDS)\n private Duration signedUrlExpirationDuration = Duration.ofSeconds(5);\n\n /** The AWS region of the s3 bucket */\n private String region = \"us-east-1\";\n\n public String getBucketName() {\n return bucketName;\n }\n\n public void setBucketName(String bucketName) {\n this.bucketName = bucketName;\n }\n\n public Duration getSignedUrlExpirationDuration() {\n return signedUrlExpirationDuration;\n }\n\n public void setSignedUrlExpirationDuration(Duration signedUrlExpirationDuration) {\n this.signedUrlExpirationDuration = signedUrlExpirationDuration;\n }\n\n public String getRegion() {\n return region;\n }\n\n public void setRegion(String region) {\n this.region = region;\n }\n}\n"
},
{
"path": "awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.s3.storage;\n\nimport java.io.InputStream;\nimport java.net.URISyntaxException;\nimport java.util.Date;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.common.run.ExternalStorageLocation;\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage;\nimport com.netflix.conductor.core.exception.NonTransientException;\nimport com.netflix.conductor.core.exception.TransientException;\nimport com.netflix.conductor.core.utils.IDGenerator;\nimport com.netflix.conductor.s3.config.S3Properties;\n\nimport com.amazonaws.HttpMethod;\nimport com.amazonaws.SdkClientException;\nimport com.amazonaws.services.s3.AmazonS3;\nimport com.amazonaws.services.s3.model.*;\n\n/**\n * An implementation of {@link ExternalPayloadStorage} using AWS S3 for storing large JSON payload\n * data.\n *\n * NOTE: The S3 client assumes that access to S3 is configured on the instance.\n *\n * @see \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.config;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.springframework.beans.factory.annotation.Autowired;\nimport org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;\nimport org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;\nimport org.springframework.boot.context.properties.EnableConfigurationProperties;\nimport org.springframework.context.annotation.Bean;\nimport org.springframework.context.annotation.Configuration;\n\nimport com.netflix.conductor.core.config.ConductorProperties;\nimport com.netflix.conductor.core.events.EventQueueProvider;\nimport com.netflix.conductor.core.events.queue.ObservableQueue;\nimport com.netflix.conductor.model.TaskModel.Status;\nimport com.netflix.conductor.sqs.eventqueue.SQSObservableQueue.Builder;\n\nimport com.amazonaws.auth.AWSCredentialsProvider;\nimport com.amazonaws.auth.DefaultAWSCredentialsProviderChain;\nimport com.amazonaws.client.builder.AwsClientBuilder;\nimport com.amazonaws.services.sqs.AmazonSQS;\nimport com.amazonaws.services.sqs.AmazonSQSClientBuilder;\nimport rx.Scheduler;\n\n@Configuration\n@EnableConfigurationProperties(SQSEventQueueProperties.class)\n@ConditionalOnProperty(name = \"conductor.event-queues.sqs.enabled\", havingValue = \"true\")\npublic class SQSEventQueueConfiguration {\n\n @Autowired private SQSEventQueueProperties sqsProperties;\n\n private static final Logger LOGGER = LoggerFactory.getLogger(SQSEventQueueConfiguration.class);\n\n @Bean\n AWSCredentialsProvider createAWSCredentialsProvider() {\n return new DefaultAWSCredentialsProviderChain();\n }\n\n @ConditionalOnMissingBean\n @Bean\n public AmazonSQS getSQSClient(AWSCredentialsProvider credentialsProvider) {\n AmazonSQSClientBuilder builder =\n AmazonSQSClientBuilder.standard().withCredentials(credentialsProvider);\n if (!sqsProperties.getEndpoint().isEmpty()) {\n LOGGER.info(\"Setting custom SQS endpoint to {}\", sqsProperties.getEndpoint());\n builder.withEndpointConfiguration(\n new AwsClientBuilder.EndpointConfiguration(\n sqsProperties.getEndpoint(), System.getenv(\"AWS_REGION\")));\n }\n return builder.build();\n }\n\n @Bean\n public EventQueueProvider sqsEventQueueProvider(\n AmazonSQS sqsClient, SQSEventQueueProperties properties, Scheduler scheduler) {\n return new SQSEventQueueProvider(sqsClient, properties, scheduler);\n }\n\n @ConditionalOnProperty(\n name = \"conductor.default-event-queue.type\",\n havingValue = \"sqs\",\n matchIfMissing = true)\n @Bean\n public Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.config;\n\nimport java.time.Duration;\nimport java.time.temporal.ChronoUnit;\n\nimport org.springframework.boot.context.properties.ConfigurationProperties;\nimport org.springframework.boot.convert.DurationUnit;\n\n@ConfigurationProperties(\"conductor.event-queues.sqs\")\npublic class SQSEventQueueProperties {\n\n /** The maximum number of messages to be fetched from the queue in a single request */\n private int batchSize = 1;\n\n /** The polling interval (in milliseconds) */\n private Duration pollTimeDuration = Duration.ofMillis(100);\n\n /** The visibility timeout (in seconds) for the message on the queue */\n @DurationUnit(ChronoUnit.SECONDS)\n private Duration visibilityTimeout = Duration.ofSeconds(60);\n\n /** The prefix to be used for the default listener queues */\n private String listenerQueuePrefix = \"\";\n\n /** The AWS account Ids authorized to send messages to the queues */\n private String authorizedAccounts = \"\";\n\n /** The endpoint to use to connect to a local SQS server for testing */\n private String endpoint = \"\";\n\n public int getBatchSize() {\n return batchSize;\n }\n\n public void setBatchSize(int batchSize) {\n this.batchSize = batchSize;\n }\n\n public Duration getPollTimeDuration() {\n return pollTimeDuration;\n }\n\n public void setPollTimeDuration(Duration pollTimeDuration) {\n this.pollTimeDuration = pollTimeDuration;\n }\n\n public Duration getVisibilityTimeout() {\n return visibilityTimeout;\n }\n\n public void setVisibilityTimeout(Duration visibilityTimeout) {\n this.visibilityTimeout = visibilityTimeout;\n }\n\n public String getListenerQueuePrefix() {\n return listenerQueuePrefix;\n }\n\n public void setListenerQueuePrefix(String listenerQueuePrefix) {\n this.listenerQueuePrefix = listenerQueuePrefix;\n }\n\n public String getAuthorizedAccounts() {\n return authorizedAccounts;\n }\n\n public void setAuthorizedAccounts(String authorizedAccounts) {\n this.authorizedAccounts = authorizedAccounts;\n }\n\n public String getEndpoint() {\n return endpoint;\n }\n\n public void setEndpoint(String endpoint) {\n this.endpoint = endpoint;\n }\n}\n"
},
{
"path": "awssqs-event-queue/src/main/java/com/netflix/conductor/sqs/config/SQSEventQueueProvider.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.config;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport org.springframework.lang.NonNull;\n\nimport com.netflix.conductor.core.events.EventQueueProvider;\nimport com.netflix.conductor.core.events.queue.ObservableQueue;\nimport com.netflix.conductor.sqs.eventqueue.SQSObservableQueue;\n\nimport com.amazonaws.services.sqs.AmazonSQS;\nimport rx.Scheduler;\n\npublic class SQSEventQueueProvider implements EventQueueProvider {\n\n private final Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.eventqueue;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.core.events.queue.Message;\nimport com.netflix.conductor.core.events.queue.ObservableQueue;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport com.amazonaws.auth.policy.Action;\nimport com.amazonaws.auth.policy.Policy;\nimport com.amazonaws.auth.policy.Principal;\nimport com.amazonaws.auth.policy.Resource;\nimport com.amazonaws.auth.policy.Statement;\nimport com.amazonaws.auth.policy.Statement.Effect;\nimport com.amazonaws.auth.policy.actions.SQSActions;\nimport com.amazonaws.services.sqs.AmazonSQS;\nimport com.amazonaws.services.sqs.model.BatchResultErrorEntry;\nimport com.amazonaws.services.sqs.model.ChangeMessageVisibilityRequest;\nimport com.amazonaws.services.sqs.model.CreateQueueRequest;\nimport com.amazonaws.services.sqs.model.CreateQueueResult;\nimport com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;\nimport com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;\nimport com.amazonaws.services.sqs.model.DeleteMessageBatchResult;\nimport com.amazonaws.services.sqs.model.GetQueueAttributesResult;\nimport com.amazonaws.services.sqs.model.ListQueuesRequest;\nimport com.amazonaws.services.sqs.model.ListQueuesResult;\nimport com.amazonaws.services.sqs.model.ReceiveMessageRequest;\nimport com.amazonaws.services.sqs.model.ReceiveMessageResult;\nimport com.amazonaws.services.sqs.model.SendMessageBatchRequest;\nimport com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;\nimport com.amazonaws.services.sqs.model.SendMessageBatchResult;\nimport com.amazonaws.services.sqs.model.SetQueueAttributesResult;\nimport rx.Observable;\nimport rx.Observable.OnSubscribe;\nimport rx.Scheduler;\n\npublic class SQSObservableQueue implements ObservableQueue {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(SQSObservableQueue.class);\n private static final String QUEUE_TYPE = \"sqs\";\n\n private final String queueName;\n private final int visibilityTimeoutInSeconds;\n private final int batchSize;\n private final AmazonSQS client;\n private final long pollTimeInMS;\n private final String queueURL;\n private final Scheduler scheduler;\n private volatile boolean running;\n\n private SQSObservableQueue(\n String queueName,\n AmazonSQS client,\n int visibilityTimeoutInSeconds,\n int batchSize,\n long pollTimeInMS,\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.eventqueue;\n\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.stubbing.Answer;\nimport org.springframework.beans.factory.annotation.Autowired;\nimport org.springframework.test.context.ContextConfiguration;\nimport org.springframework.test.context.junit4.SpringRunner;\n\nimport com.netflix.conductor.common.config.TestObjectMapperConfiguration;\nimport com.netflix.conductor.common.metadata.tasks.Task;\nimport com.netflix.conductor.common.metadata.tasks.TaskResult;\nimport com.netflix.conductor.core.events.queue.DefaultEventQueueProcessor;\nimport com.netflix.conductor.core.events.queue.Message;\nimport com.netflix.conductor.core.events.queue.ObservableQueue;\nimport com.netflix.conductor.core.execution.WorkflowExecutor;\nimport com.netflix.conductor.model.TaskModel;\nimport com.netflix.conductor.model.TaskModel.Status;\nimport com.netflix.conductor.model.WorkflowModel;\n\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.google.common.util.concurrent.Uninterruptibles;\n\nimport static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.*;\nimport static org.mockito.Mockito.*;\n\n@SuppressWarnings(\"unchecked\")\n@ContextConfiguration(classes = {TestObjectMapperConfiguration.class})\n@RunWith(SpringRunner.class)\npublic class DefaultEventQueueProcessorTest {\n\n private static SQSObservableQueue queue;\n private static WorkflowExecutor workflowExecutor;\n private DefaultEventQueueProcessor defaultEventQueueProcessor;\n\n @Autowired private ObjectMapper objectMapper;\n\n private static final List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.sqs.eventqueue;\n\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\n\nimport org.junit.Test;\nimport org.mockito.stubbing.Answer;\n\nimport com.netflix.conductor.core.events.queue.Message;\n\nimport com.amazonaws.services.sqs.AmazonSQS;\nimport com.amazonaws.services.sqs.model.ListQueuesRequest;\nimport com.amazonaws.services.sqs.model.ListQueuesResult;\nimport com.amazonaws.services.sqs.model.ReceiveMessageRequest;\nimport com.amazonaws.services.sqs.model.ReceiveMessageResult;\nimport com.google.common.util.concurrent.Uninterruptibles;\nimport rx.Observable;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class SQSObservableQueueTest {\n\n @Test\n public void test() {\n\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\n\n// Establish version and status\next.githubProjectName = rootProject.name // Change if github project name is not the same as the root project's name\n\nsubprojects {\n tasks.withType(Javadoc).all { enabled = false }\n}\n\napply from: \"$rootDir/dependencies.gradle\"\napply from: \"$rootDir/springboot-bom-overrides.gradle\"\n\nallprojects {\n apply plugin: 'com.netflix.nebula.netflixoss'\n apply plugin: 'io.spring.dependency-management'\n apply plugin: 'java-library'\n apply plugin: 'project-report'\n\n java {\n toolchain {\n languageVersion = JavaLanguageVersion.of(17)\n }\n }\n\n group = 'com.netflix.conductor'\n\n configurations.all {\n exclude group: 'ch.qos.logback', module: 'logback-classic'\n exclude group: 'ch.qos.logback', module: 'logback-core'\n exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'\n exclude group: 'org.slf4j', module: 'slf4j-log4j12'\n resolutionStrategy {\n force 'org.codehaus.jettison:jettison:1.5.4'\n force \"org.apache.commons:commons-compress:${revCommonsCompress}\"\n }\n }\n\n repositories {\n mavenCentral()\n\n // oss-candidate for -rc.* verions:\n maven {\n url \"https://artifactory-oss.prod.netflix.net/artifactory/maven-oss-candidates\"\n }\n\n /**\n * This repository locates artifacts that don't exist in maven central but we had to backup from jcenter\n * The exclusiveContent\n */\n exclusiveContent {\n forRepository {\n maven {\n url \"https://artifactory-oss.prod.netflix.net/artifactory/required-jcenter-modules-backup\"\n }\n }\n filter {\n includeGroupByRegex \"com\\\\.github\\\\.vmg.*\"\n }\n }\n }\n\n dependencyManagement {\n imports {\n // dependency versions for the BOM can be found at https://docs.spring.io/spring-boot/docs/2.7.3/reference/htmlsingle/#appendix.dependency-versions\n mavenBom(SpringBootPlugin.BOM_COORDINATES)\n }\n }\n\n dependencies {\n implementation('org.apache.logging.log4j:log4j-core') {\n version {\n // this is the preferred version this library will use\n prefer '2.17.2'\n // the strict bounds, effectively allowing any 2.x version greater than 2.17.2\n // could also remove the upper bound entirely if we wanted too\n strictly '[2.17.2,3.0)'\n }\n }\n implementation('org.apache.logging.log4j:log4j-api') {\n version {\n // this is the preferred version this library will use\n prefer '2.17.2'\n // the strict bounds, effectively allowing any 2.x version greater than 2.17.2\n // could also remove the upper bound entirely if we wanted too\n strictly '[2.17.2,3.0)'\n }\n }\n implementation('org.apache.logging.log4j:log4j-slf4j-impl') {\n version {\n // this is the preferred version this library will use\n prefer '2.17.2'\n // the strict bounds, effectively allowing any 2.x version greater than 2.17.2\n // could also remove the upper bound entirely if we wanted too\n strictly '[2.17.2,3.0)'\n }\n }\n implementation('org.apache.logging.log4j:log4j-jul') {\n version {\n // this is the preferred version this library will use\n prefer '2.17.2'\n // the strict bounds, effectively allowing any 2.x version greater than 2.17.2\n // could also remove the upper bound entirely if we wanted too\n strictly '[2.17.2,3.0)'\n }\n }\n implementation('org.apache.logging.log4j:log4j-web') {\n version {\n // this is the preferred version this library will use\n prefer '2.17.2'\n // the strict bounds, effectively allowing any 2.x version greater than 2.17.2\n // could also remove the upper bound entirely if we wanted too\n strictly '[2.17.2,3.0)'\n }\n }\n implementation('org.yaml:snakeyaml') {\n version {\n // this is the preferred version this library will use\n prefer '2.0'\n // the strict bounds, effectively allowing any 2.x version between 2.0 and 2.1\n strictly '[2.0,2.1)'\n }\n }\n implementation('com.fasterxml.jackson.core:jackson-core') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.core:jackson-databind') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-yaml') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.core:jackson-annotations') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-smile') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-cbor') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.datatype:jackson-datatype-jdk8') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.datatype:jackson-datatype-joda') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.datatype:jackson-datatype-jsr310') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('com.fasterxml.jackson.module:jackson-module-afterburner') {\n version {\n // this is the preferred version this library will use\n prefer '2.15.0'\n // the strict bounds, effectively allowing any 2.15.x version between 2.15.0 and 2.15.2\n strictly '[2.15.0,2.15.2)'\n }\n }\n implementation('org.apache.logging.log4j:log4j-core')\n implementation('org.apache.logging.log4j:log4j-api')\n implementation('org.apache.logging.log4j:log4j-slf4j-impl')\n implementation('org.apache.logging.log4j:log4j-jul')\n implementation('org.apache.logging.log4j:log4j-web') \n annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'\n\n testImplementation('org.springframework.boot:spring-boot-starter-test') {\n exclude group: 'org.yaml', module: 'snakeyaml'\n }\n testImplementation('org.springframework.boot:spring-boot-starter-log4j2')\n testImplementation 'junit:junit'\n testImplementation \"org.junit.vintage:junit-vintage-engine\"\n\n // Needed for build to work on m1/m2 macs\n testImplementation 'net.java.dev.jna:jna:5.13.0'\n }\n\n // processes additional configuration metadata json file as described here\n // https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/appendix-configuration-metadata.html#configuration-metadata-additional-metadata\n compileJava.inputs.files(processResources)\n\n test {\n useJUnitPlatform()\n testLogging {\n events = [\"SKIPPED\", \"FAILED\"]\n exceptionFormat = \"full\"\n displayGranularity = 1\n showStandardStreams = false\n }\n }\n}\n\n// all client and their related modules are published with Java 17 compatibility\n[\"annotations\", \"common\", \"client\", \"client-spring\", \"grpc\", \"grpc-client\"].each {\n project(\":conductor-$it\") {\n compileJava {\n options.release = 17\n }\n }\n}\n\njacocoTestReport {\n reports {\n html.required = true\n xml.required = true\n csv.required = false\n }\n}\n\ntask server {\n dependsOn ':conductor-server:bootRun'\n}\n\nsonarqube {\n properties {\n property \"sonar.projectKey\", \"com.netflix.conductor:conductor\"\n property \"sonar.organization\", \"netflix\"\n property \"sonar.host.url\", \"https://sonarcloud.io\"\n }\n}\n\nconfigure(allprojects - project(':conductor-grpc')) {\n apply plugin: 'com.diffplug.spotless'\n\n spotless {\n java {\n googleJavaFormat().aosp()\n removeUnusedImports()\n importOrder('java', 'javax', 'org', 'com.netflix', '', '\\\\#com.netflix', '\\\\#')\n licenseHeaderFile(\"$rootDir/licenseheader.txt\")\n }\n }\n}\n\n['cassandra-persistence', 'core', 'redis-concurrency-limit', 'test-harness', 'client'].each {\n configure(project(\":conductor-$it\")) {\n spotless {\n groovy {\n importOrder('java', 'javax', 'org', 'com.netflix', '', '\\\\#com.netflix', '\\\\#')\n licenseHeaderFile(\"$rootDir/licenseheader.txt\")\n }\n }\n }\n}\n"
},
{
"path": "cassandra-persistence/build.gradle",
"content": "/*\n * Copyright 2021 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\napply plugin: 'groovy'\n\ndependencies {\n compileOnly 'org.springframework.boot:spring-boot-starter'\n\n implementation project(':conductor-common')\n implementation project(':conductor-core')\n implementation \"com.datastax.cassandra:cassandra-driver-core:${revCassandra}\"\n implementation \"org.apache.commons:commons-lang3\"\n\n testImplementation project(':conductor-core').sourceSets.test.output\n testImplementation project(':conductor-common').sourceSets.test.output\n\n testImplementation \"org.codehaus.groovy:groovy-all:${revGroovy}\"\n testImplementation \"org.spockframework:spock-core:${revSpock}\"\n testImplementation \"org.spockframework:spock-spring:${revSpock}\"\n testImplementation \"org.testcontainers:spock:${revTestContainer}\"\n testImplementation \"org.testcontainers:cassandra:${revTestContainer}\"\n testImplementation \"com.google.protobuf:protobuf-java:${revProtoBuf}\"\n}\n"
},
{
"path": "cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/CassandraConfiguration.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.config;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;\nimport org.springframework.boot.context.properties.EnableConfigurationProperties;\nimport org.springframework.cache.CacheManager;\nimport org.springframework.context.annotation.Bean;\nimport org.springframework.context.annotation.Configuration;\n\nimport com.netflix.conductor.cassandra.config.cache.CacheableEventHandlerDAO;\nimport com.netflix.conductor.cassandra.config.cache.CacheableMetadataDAO;\nimport com.netflix.conductor.cassandra.dao.CassandraEventHandlerDAO;\nimport com.netflix.conductor.cassandra.dao.CassandraExecutionDAO;\nimport com.netflix.conductor.cassandra.dao.CassandraMetadataDAO;\nimport com.netflix.conductor.cassandra.dao.CassandraPollDataDAO;\nimport com.netflix.conductor.cassandra.util.Statements;\nimport com.netflix.conductor.dao.EventHandlerDAO;\nimport com.netflix.conductor.dao.ExecutionDAO;\nimport com.netflix.conductor.dao.MetadataDAO;\n\nimport com.datastax.driver.core.Cluster;\nimport com.datastax.driver.core.Metadata;\nimport com.datastax.driver.core.Session;\nimport com.fasterxml.jackson.databind.ObjectMapper;\n\n@Configuration(proxyBeanMethods = false)\n@EnableConfigurationProperties(CassandraProperties.class)\n@ConditionalOnProperty(name = \"conductor.db.type\", havingValue = \"cassandra\")\npublic class CassandraConfiguration {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CassandraConfiguration.class);\n\n @Bean\n public Cluster cluster(CassandraProperties properties) {\n String host = properties.getHostAddress();\n int port = properties.getPort();\n\n LOGGER.info(\"Connecting to cassandra cluster with host:{}, port:{}\", host, port);\n\n Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build();\n\n Metadata metadata = cluster.getMetadata();\n LOGGER.info(\"Connected to cluster: {}\", metadata.getClusterName());\n metadata.getAllHosts()\n .forEach(\n h ->\n LOGGER.info(\n \"Datacenter:{}, host:{}, rack: {}\",\n h.getDatacenter(),\n h.getEndPoint().resolve().getHostName(),\n h.getRack()));\n return cluster;\n }\n\n @Bean\n public Session session(Cluster cluster) {\n LOGGER.info(\"Initializing cassandra session\");\n return cluster.connect();\n }\n\n @Bean\n public MetadataDAO cassandraMetadataDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements,\n CacheManager cacheManager) {\n CassandraMetadataDAO cassandraMetadataDAO =\n new CassandraMetadataDAO(session, objectMapper, properties, statements);\n return new CacheableMetadataDAO(cassandraMetadataDAO, properties, cacheManager);\n }\n\n @Bean\n public ExecutionDAO cassandraExecutionDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements) {\n return new CassandraExecutionDAO(session, objectMapper, properties, statements);\n }\n\n @Bean\n public EventHandlerDAO cassandraEventHandlerDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements,\n CacheManager cacheManager) {\n CassandraEventHandlerDAO cassandraEventHandlerDAO =\n new CassandraEventHandlerDAO(session, objectMapper, properties, statements);\n return new CacheableEventHandlerDAO(cassandraEventHandlerDAO, properties, cacheManager);\n }\n\n @Bean\n public CassandraPollDataDAO cassandraPollDataDAO() {\n return new CassandraPollDataDAO();\n }\n\n @Bean\n public Statements statements(CassandraProperties cassandraProperties) {\n return new Statements(cassandraProperties.getKeyspace());\n }\n}\n"
},
{
"path": "cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/CassandraProperties.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.config;\n\nimport java.time.Duration;\nimport java.time.temporal.ChronoUnit;\n\nimport org.springframework.boot.context.properties.ConfigurationProperties;\nimport org.springframework.boot.convert.DurationUnit;\n\nimport com.datastax.driver.core.ConsistencyLevel;\n\n@ConfigurationProperties(\"conductor.cassandra\")\npublic class CassandraProperties {\n\n /** The address for the cassandra database host */\n private String hostAddress = \"127.0.0.1\";\n\n /** The port to be used to connect to the cassandra database instance */\n private int port = 9142;\n\n /** The name of the cassandra cluster */\n private String cluster = \"\";\n\n /** The keyspace to be used in the cassandra datastore */\n private String keyspace = \"conductor\";\n\n /**\n * The number of tasks to be stored in a single partition which will be used for sharding\n * workflows in the datastore\n */\n private int shardSize = 100;\n\n /** The replication strategy with which to configure the keyspace */\n private String replicationStrategy = \"SimpleStrategy\";\n\n /** The key to be used while configuring the replication factor */\n private String replicationFactorKey = \"replication_factor\";\n\n /** The replication factor value with which the keyspace is configured */\n private int replicationFactorValue = 3;\n\n /** The consistency level to be used for read operations */\n private ConsistencyLevel readConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM;\n\n /** The consistency level to be used for write operations */\n private ConsistencyLevel writeConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM;\n\n /** The time in seconds after which the in-memory task definitions cache will be refreshed */\n @DurationUnit(ChronoUnit.SECONDS)\n private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60);\n\n /** The time in seconds after which the in-memory event handler cache will be refreshed */\n @DurationUnit(ChronoUnit.SECONDS)\n private Duration eventHandlerCacheRefreshInterval = Duration.ofSeconds(60);\n\n /** The time to live in seconds for which the event execution will be persisted */\n @DurationUnit(ChronoUnit.SECONDS)\n private Duration eventExecutionPersistenceTtl = Duration.ZERO;\n\n public String getHostAddress() {\n return hostAddress;\n }\n\n public void setHostAddress(String hostAddress) {\n this.hostAddress = hostAddress;\n }\n\n public int getPort() {\n return port;\n }\n\n public void setPort(int port) {\n this.port = port;\n }\n\n public String getCluster() {\n return cluster;\n }\n\n public void setCluster(String cluster) {\n this.cluster = cluster;\n }\n\n public String getKeyspace() {\n return keyspace;\n }\n\n public void setKeyspace(String keyspace) {\n this.keyspace = keyspace;\n }\n\n public int getShardSize() {\n return shardSize;\n }\n\n public void setShardSize(int shardSize) {\n this.shardSize = shardSize;\n }\n\n public String getReplicationStrategy() {\n return replicationStrategy;\n }\n\n public void setReplicationStrategy(String replicationStrategy) {\n this.replicationStrategy = replicationStrategy;\n }\n\n public String getReplicationFactorKey() {\n return replicationFactorKey;\n }\n\n public void setReplicationFactorKey(String replicationFactorKey) {\n this.replicationFactorKey = replicationFactorKey;\n }\n\n public int getReplicationFactorValue() {\n return replicationFactorValue;\n }\n\n public void setReplicationFactorValue(int replicationFactorValue) {\n this.replicationFactorValue = replicationFactorValue;\n }\n\n public ConsistencyLevel getReadConsistencyLevel() {\n return readConsistencyLevel;\n }\n\n public void setReadConsistencyLevel(ConsistencyLevel readConsistencyLevel) {\n this.readConsistencyLevel = readConsistencyLevel;\n }\n\n public ConsistencyLevel getWriteConsistencyLevel() {\n return writeConsistencyLevel;\n }\n\n public void setWriteConsistencyLevel(ConsistencyLevel writeConsistencyLevel) {\n this.writeConsistencyLevel = writeConsistencyLevel;\n }\n\n public Duration getTaskDefCacheRefreshInterval() {\n return taskDefCacheRefreshInterval;\n }\n\n public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) {\n this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval;\n }\n\n public Duration getEventHandlerCacheRefreshInterval() {\n return eventHandlerCacheRefreshInterval;\n }\n\n public void setEventHandlerCacheRefreshInterval(Duration eventHandlerCacheRefreshInterval) {\n this.eventHandlerCacheRefreshInterval = eventHandlerCacheRefreshInterval;\n }\n\n public Duration getEventExecutionPersistenceTtl() {\n return eventExecutionPersistenceTtl;\n }\n\n public void setEventExecutionPersistenceTtl(Duration eventExecutionPersistenceTtl) {\n this.eventExecutionPersistenceTtl = eventExecutionPersistenceTtl;\n }\n}\n"
},
{
"path": "cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/config/cache/CacheableEventHandlerDAO.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.config.cache;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\n\nimport javax.annotation.PostConstruct;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.springframework.cache.Cache;\nimport org.springframework.cache.CacheManager;\nimport org.springframework.cache.annotation.CacheEvict;\nimport org.springframework.cache.annotation.CachePut;\n\nimport com.netflix.conductor.annotations.Trace;\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.cassandra.dao.CassandraEventHandlerDAO;\nimport com.netflix.conductor.common.metadata.events.EventHandler;\nimport com.netflix.conductor.dao.EventHandlerDAO;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport static com.netflix.conductor.cassandra.config.cache.CachingConfig.EVENT_HANDLER_CACHE;\n\n@Trace\npublic class CacheableEventHandlerDAO implements EventHandlerDAO {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CacheableEventHandlerDAO.class);\n\n private static final String CLASS_NAME = CacheableEventHandlerDAO.class.getSimpleName();\n\n private final CassandraEventHandlerDAO cassandraEventHandlerDAO;\n private final CassandraProperties properties;\n\n private final CacheManager cacheManager;\n\n public CacheableEventHandlerDAO(\n CassandraEventHandlerDAO cassandraEventHandlerDAO,\n CassandraProperties properties,\n CacheManager cacheManager) {\n this.cassandraEventHandlerDAO = cassandraEventHandlerDAO;\n this.properties = properties;\n this.cacheManager = cacheManager;\n }\n\n @PostConstruct\n public void scheduleEventHandlerRefresh() {\n long cacheRefreshTime = properties.getEventHandlerCacheRefreshInterval().getSeconds();\n Executors.newSingleThreadScheduledExecutor()\n .scheduleWithFixedDelay(\n this::refreshEventHandlersCache, 0, cacheRefreshTime, TimeUnit.SECONDS);\n }\n\n @Override\n @CachePut(value = EVENT_HANDLER_CACHE, key = \"#eventHandler.name\")\n public void addEventHandler(EventHandler eventHandler) {\n cassandraEventHandlerDAO.addEventHandler(eventHandler);\n }\n\n @Override\n @CachePut(value = EVENT_HANDLER_CACHE, key = \"#eventHandler.name\")\n public void updateEventHandler(EventHandler eventHandler) {\n cassandraEventHandlerDAO.updateEventHandler(eventHandler);\n }\n\n @Override\n @CacheEvict(EVENT_HANDLER_CACHE)\n public void removeEventHandler(String name) {\n cassandraEventHandlerDAO.removeEventHandler(name);\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.config.cache;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\n\nimport javax.annotation.PostConstruct;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.springframework.cache.Cache;\nimport org.springframework.cache.CacheManager;\nimport org.springframework.cache.annotation.CacheEvict;\nimport org.springframework.cache.annotation.CachePut;\nimport org.springframework.cache.annotation.Cacheable;\n\nimport com.netflix.conductor.annotations.Trace;\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.cassandra.dao.CassandraMetadataDAO;\nimport com.netflix.conductor.common.metadata.tasks.TaskDef;\nimport com.netflix.conductor.common.metadata.workflow.WorkflowDef;\nimport com.netflix.conductor.dao.MetadataDAO;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport static com.netflix.conductor.cassandra.config.cache.CachingConfig.TASK_DEF_CACHE;\n\n@Trace\npublic class CacheableMetadataDAO implements MetadataDAO {\n\n private static final String CLASS_NAME = CacheableMetadataDAO.class.getSimpleName();\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CacheableMetadataDAO.class);\n\n private final CassandraMetadataDAO cassandraMetadataDAO;\n private final CassandraProperties properties;\n\n private final CacheManager cacheManager;\n\n public CacheableMetadataDAO(\n CassandraMetadataDAO cassandraMetadataDAO,\n CassandraProperties properties,\n CacheManager cacheManager) {\n this.cassandraMetadataDAO = cassandraMetadataDAO;\n this.properties = properties;\n this.cacheManager = cacheManager;\n }\n\n @PostConstruct\n public void scheduleCacheRefresh() {\n long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds();\n Executors.newSingleThreadScheduledExecutor()\n .scheduleWithFixedDelay(\n this::refreshTaskDefsCache, 0, cacheRefreshTime, TimeUnit.SECONDS);\n LOGGER.info(\n \"Scheduled cache refresh for Task Definitions, every {} seconds\", cacheRefreshTime);\n }\n\n @Override\n @CachePut(value = TASK_DEF_CACHE, key = \"#taskDef.name\")\n public TaskDef createTaskDef(TaskDef taskDef) {\n cassandraMetadataDAO.createTaskDef(taskDef);\n return taskDef;\n }\n\n @Override\n @CachePut(value = TASK_DEF_CACHE, key = \"#taskDef.name\")\n public TaskDef updateTaskDef(TaskDef taskDef) {\n return cassandraMetadataDAO.updateTaskDef(taskDef);\n }\n\n @Override\n @Cacheable(TASK_DEF_CACHE)\n public TaskDef getTaskDef(String name) {\n return cassandraMetadataDAO.getTaskDef(name);\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.config.cache;\n\nimport org.springframework.cache.CacheManager;\nimport org.springframework.cache.annotation.EnableCaching;\nimport org.springframework.cache.concurrent.ConcurrentMapCacheManager;\nimport org.springframework.context.annotation.Bean;\nimport org.springframework.context.annotation.Configuration;\n\n@Configuration\n@EnableCaching\npublic class CachingConfig {\n public static final String TASK_DEF_CACHE = \"taskDefCache\";\n public static final String EVENT_HANDLER_CACHE = \"eventHandlerCache\";\n\n @Bean\n public CacheManager cacheManager() {\n return new ConcurrentMapCacheManager(TASK_DEF_CACHE, EVENT_HANDLER_CACHE);\n }\n}\n"
},
{
"path": "cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraBaseDAO.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao;\n\nimport java.io.IOException;\nimport java.util.UUID;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.core.exception.NonTransientException;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport com.datastax.driver.core.DataType;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.schemabuilder.SchemaBuilder;\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.google.common.collect.ImmutableMap;\n\nimport static com.netflix.conductor.cassandra.util.Constants.DAO_NAME;\nimport static com.netflix.conductor.cassandra.util.Constants.ENTITY_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_EXECUTION_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.MESSAGE_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.PAYLOAD_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.SHARD_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_EXECUTIONS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_HANDLERS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEFS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEF_LIMIT;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_LOOKUP;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOWS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS_INDEX;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEF_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TOTAL_PARTITIONS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TOTAL_TASKS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_VALUE;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_VERSION_KEY;\n\n/**\n * Creates the keyspace and tables.\n *\n * CREATE KEYSPACE IF NOT EXISTS conductor WITH replication = { 'class' :\n * 'NetworkTopologyStrategy', 'us-east': '3'};\n *\n * CREATE TABLE IF NOT EXISTS conductor.workflows ( workflow_id uuid, shard_id int, task_id text,\n * entity text, payload text, total_tasks int STATIC, total_partitions int STATIC, PRIMARY\n * KEY((workflow_id, shard_id), entity, task_id) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.task_lookup( task_id uuid, workflow_id uuid, PRIMARY KEY\n * (task_id) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.task_def_limit( task_def_name text, task_id uuid,\n * workflow_id uuid, PRIMARY KEY ((task_def_name), task_id_key) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.workflow_definitions( workflow_def_name text, version\n * int, workflow_definition text, PRIMARY KEY ((workflow_def_name), version) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.workflow_defs_index( workflow_def_version_index text,\n * workflow_def_name_version text, workflow_def_index_value text,PRIMARY KEY\n * ((workflow_def_version_index), workflow_def_name_version) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.task_definitions( task_defs text, task_def_name text,\n * task_definition text, PRIMARY KEY ((task_defs), task_def_name) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.event_handlers( handlers text, event_handler_name text,\n * event_handler text, PRIMARY KEY ((handlers), event_handler_name) );\n *\n * CREATE TABLE IF NOT EXISTS conductor.event_executions( message_id text, event_handler_name\n * text, event_execution_id text, payload text, PRIMARY KEY ((message_id, event_handler_name),\n * event_execution_id) );\n */\npublic abstract class CassandraBaseDAO {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CassandraBaseDAO.class);\n\n private final ObjectMapper objectMapper;\n protected final Session session;\n protected final CassandraProperties properties;\n\n private boolean initialized = false;\n\n public CassandraBaseDAO(\n Session session, ObjectMapper objectMapper, CassandraProperties properties) {\n this.session = session;\n this.objectMapper = objectMapper;\n this.properties = properties;\n\n init();\n }\n\n protected static UUID toUUID(String uuidString, String message) {\n try {\n return UUID.fromString(uuidString);\n } catch (IllegalArgumentException iae) {\n throw new IllegalArgumentException(message + \" \" + uuidString, iae);\n }\n }\n\n private void init() {\n try {\n if (!initialized) {\n session.execute(getCreateKeyspaceStatement());\n session.execute(getCreateWorkflowsTableStatement());\n session.execute(getCreateTaskLookupTableStatement());\n session.execute(getCreateTaskDefLimitTableStatement());\n session.execute(getCreateWorkflowDefsTableStatement());\n session.execute(getCreateWorkflowDefsIndexTableStatement());\n session.execute(getCreateTaskDefsTableStatement());\n session.execute(getCreateEventHandlersTableStatement());\n session.execute(getCreateEventExecutionsTableStatement());\n LOGGER.info(\n \"{} initialization complete! Tables created!\", getClass().getSimpleName());\n initialized = true;\n }\n } catch (Exception e) {\n LOGGER.error(\"Error initializing and setting up keyspace and table in cassandra\", e);\n throw e;\n }\n }\n\n private String getCreateKeyspaceStatement() {\n return SchemaBuilder.createKeyspace(properties.getKeyspace())\n .ifNotExists()\n .with()\n .replication(\n ImmutableMap.of(\n \"class\",\n properties.getReplicationStrategy(),\n properties.getReplicationFactorKey(),\n properties.getReplicationFactorValue()))\n .durableWrites(true)\n .getQueryString();\n }\n\n private String getCreateWorkflowsTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOWS)\n .ifNotExists()\n .addPartitionKey(WORKFLOW_ID_KEY, DataType.uuid())\n .addPartitionKey(SHARD_ID_KEY, DataType.cint())\n .addClusteringColumn(ENTITY_KEY, DataType.text())\n .addClusteringColumn(TASK_ID_KEY, DataType.text())\n .addColumn(PAYLOAD_KEY, DataType.text())\n .addStaticColumn(TOTAL_TASKS_KEY, DataType.cint())\n .addStaticColumn(TOTAL_PARTITIONS_KEY, DataType.cint())\n .getQueryString();\n }\n\n private String getCreateTaskLookupTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_LOOKUP)\n .ifNotExists()\n .addPartitionKey(TASK_ID_KEY, DataType.uuid())\n .addColumn(WORKFLOW_ID_KEY, DataType.uuid())\n .getQueryString();\n }\n\n private String getCreateTaskDefLimitTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_DEF_LIMIT)\n .ifNotExists()\n .addPartitionKey(TASK_DEF_NAME_KEY, DataType.text())\n .addClusteringColumn(TASK_ID_KEY, DataType.uuid())\n .addColumn(WORKFLOW_ID_KEY, DataType.uuid())\n .getQueryString();\n }\n\n private String getCreateWorkflowDefsTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOW_DEFS)\n .ifNotExists()\n .addPartitionKey(WORKFLOW_DEF_NAME_KEY, DataType.text())\n .addClusteringColumn(WORKFLOW_VERSION_KEY, DataType.cint())\n .addColumn(WORKFLOW_DEFINITION_KEY, DataType.text())\n .getQueryString();\n }\n\n private String getCreateWorkflowDefsIndexTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_WORKFLOW_DEFS_INDEX)\n .ifNotExists()\n .addPartitionKey(WORKFLOW_DEF_INDEX_KEY, DataType.text())\n .addClusteringColumn(WORKFLOW_DEF_NAME_VERSION_KEY, DataType.text())\n .addColumn(WORKFLOW_DEF_INDEX_VALUE, DataType.text())\n .getQueryString();\n }\n\n private String getCreateTaskDefsTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_TASK_DEFS)\n .ifNotExists()\n .addPartitionKey(TASK_DEFS_KEY, DataType.text())\n .addClusteringColumn(TASK_DEF_NAME_KEY, DataType.text())\n .addColumn(TASK_DEFINITION_KEY, DataType.text())\n .getQueryString();\n }\n\n private String getCreateEventHandlersTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_EVENT_HANDLERS)\n .ifNotExists()\n .addPartitionKey(HANDLERS_KEY, DataType.text())\n .addClusteringColumn(EVENT_HANDLER_NAME_KEY, DataType.text())\n .addColumn(EVENT_HANDLER_KEY, DataType.text())\n .getQueryString();\n }\n\n private String getCreateEventExecutionsTableStatement() {\n return SchemaBuilder.createTable(properties.getKeyspace(), TABLE_EVENT_EXECUTIONS)\n .ifNotExists()\n .addPartitionKey(MESSAGE_ID_KEY, DataType.text())\n .addPartitionKey(EVENT_HANDLER_NAME_KEY, DataType.text())\n .addClusteringColumn(EVENT_EXECUTION_ID_KEY, DataType.text())\n .addColumn(PAYLOAD_KEY, DataType.text())\n .getQueryString();\n }\n\n String toJson(Object value) {\n try {\n return objectMapper.writeValueAsString(value);\n } catch (JsonProcessingException e) {\n throw new NonTransientException(\"Error serializing to json\", e);\n }\n }\n\n \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.stream.Collectors;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.annotations.Trace;\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.cassandra.util.Statements;\nimport com.netflix.conductor.common.metadata.events.EventHandler;\nimport com.netflix.conductor.core.exception.TransientException;\nimport com.netflix.conductor.dao.EventHandlerDAO;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport com.datastax.driver.core.PreparedStatement;\nimport com.datastax.driver.core.ResultSet;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.exceptions.DriverException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\n\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY;\n\n@Trace\npublic class CassandraEventHandlerDAO extends CassandraBaseDAO implements EventHandlerDAO {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CassandraEventHandlerDAO.class);\n private static final String CLASS_NAME = CassandraEventHandlerDAO.class.getSimpleName();\n\n private final PreparedStatement insertEventHandlerStatement;\n private final PreparedStatement selectAllEventHandlersStatement;\n private final PreparedStatement deleteEventHandlerStatement;\n\n public CassandraEventHandlerDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements) {\n super(session, objectMapper, properties);\n\n insertEventHandlerStatement =\n session.prepare(statements.getInsertEventHandlerStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n selectAllEventHandlersStatement =\n session.prepare(statements.getSelectAllEventHandlersStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n deleteEventHandlerStatement =\n session.prepare(statements.getDeleteEventHandlerStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n }\n\n @Override\n public void addEventHandler(EventHandler eventHandler) {\n insertOrUpdateEventHandler(eventHandler);\n }\n\n @Override\n public void updateEventHandler(EventHandler eventHandler) {\n insertOrUpdateEventHandler(eventHandler);\n }\n\n @Override\n public void removeEventHandler(String name) {\n try {\n recordCassandraDaoRequests(\"removeEventHandler\");\n session.execute(deleteEventHandlerStatement.bind(name));\n } catch (Exception e) {\n Monitors.error(CLASS_NAME, \"removeEventHandler\");\n String errorMsg = String.format(\"Failed to remove event handler: %s\", name);\n LOGGER.error(errorMsg, e);\n throw new TransientException(errorMsg, e);\n }\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao;\n\nimport java.util.*;\nimport java.util.stream.Collectors;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.annotations.Trace;\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.cassandra.util.Statements;\nimport com.netflix.conductor.common.metadata.events.EventExecution;\nimport com.netflix.conductor.common.metadata.tasks.TaskDef;\nimport com.netflix.conductor.core.exception.NonTransientException;\nimport com.netflix.conductor.core.exception.NotFoundException;\nimport com.netflix.conductor.core.exception.TransientException;\nimport com.netflix.conductor.dao.ConcurrentExecutionLimitDAO;\nimport com.netflix.conductor.dao.ExecutionDAO;\nimport com.netflix.conductor.metrics.Monitors;\nimport com.netflix.conductor.model.TaskModel;\nimport com.netflix.conductor.model.WorkflowModel;\n\nimport com.datastax.driver.core.*;\nimport com.datastax.driver.core.exceptions.DriverException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\n\nimport static com.netflix.conductor.cassandra.util.Constants.*;\n\n@Trace\npublic class CassandraExecutionDAO extends CassandraBaseDAO\n implements ExecutionDAO, ConcurrentExecutionLimitDAO {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CassandraExecutionDAO.class);\n private static final String CLASS_NAME = CassandraExecutionDAO.class.getSimpleName();\n\n protected final PreparedStatement insertWorkflowStatement;\n protected final PreparedStatement insertTaskStatement;\n protected final PreparedStatement insertEventExecutionStatement;\n\n protected final PreparedStatement selectTotalStatement;\n protected final PreparedStatement selectTaskStatement;\n protected final PreparedStatement selectWorkflowStatement;\n protected final PreparedStatement selectWorkflowWithTasksStatement;\n protected final PreparedStatement selectTaskLookupStatement;\n protected final PreparedStatement selectTasksFromTaskDefLimitStatement;\n protected final PreparedStatement selectEventExecutionsStatement;\n\n protected final PreparedStatement updateWorkflowStatement;\n protected final PreparedStatement updateTotalTasksStatement;\n protected final PreparedStatement updateTotalPartitionsStatement;\n protected final PreparedStatement updateTaskLookupStatement;\n protected final PreparedStatement updateTaskDefLimitStatement;\n protected final PreparedStatement updateEventExecutionStatement;\n\n protected final PreparedStatement deleteWorkflowStatement;\n protected final PreparedStatement deleteTaskStatement;\n protected final PreparedStatement deleteTaskLookupStatement;\n protected final PreparedStatement deleteTaskDefLimitStatement;\n protected final PreparedStatement deleteEventExecutionStatement;\n\n protected final int eventExecutionsTTL;\n\n public CassandraExecutionDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements) {\n super(session, objectMapper, properties);\n\n eventExecutionsTTL = (int) properties.getEventExecutionPersistenceTtl().getSeconds();\n\n this.insertWorkflowStatement =\n session.prepare(statements.getInsertWorkflowStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.insertTaskStatement =\n session.prepare(statements.getInsertTaskStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.insertEventExecutionStatement =\n session.prepare(statements.getInsertEventExecutionStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n\n this.selectTotalStatement =\n session.prepare(statements.getSelectTotalStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectTaskStatement =\n session.prepare(statements.getSelectTaskStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectWorkflowStatement =\n session.prepare(statements.getSelectWorkflowStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectWorkflowWithTasksStatement =\n session.prepare(statements.getSelectWorkflowWithTasksStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectTaskLookupStatement =\n session.prepare(statements.getSelectTaskFromLookupTableStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectTasksFromTaskDefLimitStatement =\n session.prepare(statements.getSelectTasksFromTaskDefLimitStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectEventExecutionsStatement =\n session.prepare(\n statements\n .getSelectAllEventExecutionsForMessageFromEventExecutionsStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n\n this.updateWorkflowStatement =\n session.prepare(statements.getUpdateWorkflowStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.updateTotalTasksStatement =\n session.prepare(statements.getUpdateTotalTasksStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.updateTotalPartitionsStatement =\n session.prepare(statements.getUpdateTotalPartitionsStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.updateTaskLookupStatement =\n session.prepare(statements.getUpdateTaskLookupStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.updateTaskDefLimitStatement =\n session.prepare(statements.getUpdateTaskDefLimitStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.updateEventExecutionStatement =\n session.prepare(statements.getUpdateEventExecutionStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n\n this.deleteWorkflowStatement =\n session.prepare(statements.getDeleteWorkflowStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteTaskStatement =\n session.prepare(statements.getDeleteTaskStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteTaskLookupStatement =\n session.prepare(statements.getDeleteTaskLookupStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteTaskDefLimitStatement =\n session.prepare(statements.getDeleteTaskDefLimitStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteEventExecutionStatement =\n session.prepare(statements.getDeleteEventExecutionsStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao;\n\nimport java.util.Collections;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.PriorityQueue;\nimport java.util.stream.Collectors;\n\nimport org.apache.commons.lang3.tuple.ImmutablePair;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.annotations.Trace;\nimport com.netflix.conductor.annotations.VisibleForTesting;\nimport com.netflix.conductor.cassandra.config.CassandraProperties;\nimport com.netflix.conductor.cassandra.util.Statements;\nimport com.netflix.conductor.common.metadata.tasks.TaskDef;\nimport com.netflix.conductor.common.metadata.workflow.WorkflowDef;\nimport com.netflix.conductor.core.exception.ConflictException;\nimport com.netflix.conductor.core.exception.TransientException;\nimport com.netflix.conductor.dao.MetadataDAO;\nimport com.netflix.conductor.metrics.Monitors;\n\nimport com.datastax.driver.core.PreparedStatement;\nimport com.datastax.driver.core.ResultSet;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.exceptions.DriverException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\n\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY;\nimport static com.netflix.conductor.common.metadata.tasks.TaskDef.ONE_HOUR;\n\n@Trace\npublic class CassandraMetadataDAO extends CassandraBaseDAO implements MetadataDAO {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(CassandraMetadataDAO.class);\n private static final String CLASS_NAME = CassandraMetadataDAO.class.getSimpleName();\n private static final String INDEX_DELIMITER = \"/\";\n\n private final PreparedStatement insertWorkflowDefStatement;\n private final PreparedStatement insertWorkflowDefVersionIndexStatement;\n private final PreparedStatement insertTaskDefStatement;\n\n private final PreparedStatement selectWorkflowDefStatement;\n\n private final PreparedStatement selectAllWorkflowDefVersionsByNameStatement;\n private final PreparedStatement selectAllWorkflowDefsStatement;\n private final PreparedStatement selectAllWorkflowDefsLatestVersionsStatement;\n private final PreparedStatement selectTaskDefStatement;\n private final PreparedStatement selectAllTaskDefsStatement;\n\n private final PreparedStatement updateWorkflowDefStatement;\n\n private final PreparedStatement deleteWorkflowDefStatement;\n private final PreparedStatement deleteWorkflowDefIndexStatement;\n private final PreparedStatement deleteTaskDefStatement;\n\n public CassandraMetadataDAO(\n Session session,\n ObjectMapper objectMapper,\n CassandraProperties properties,\n Statements statements) {\n super(session, objectMapper, properties);\n\n this.insertWorkflowDefStatement =\n session.prepare(statements.getInsertWorkflowDefStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.insertWorkflowDefVersionIndexStatement =\n session.prepare(statements.getInsertWorkflowDefVersionIndexStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.insertTaskDefStatement =\n session.prepare(statements.getInsertTaskDefStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n\n this.selectWorkflowDefStatement =\n session.prepare(statements.getSelectWorkflowDefStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectAllWorkflowDefVersionsByNameStatement =\n session.prepare(statements.getSelectAllWorkflowDefVersionsByNameStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectAllWorkflowDefsStatement =\n session.prepare(statements.getSelectAllWorkflowDefsStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectAllWorkflowDefsLatestVersionsStatement =\n session.prepare(statements.getSelectAllWorkflowDefsLatestVersionsStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectTaskDefStatement =\n session.prepare(statements.getSelectTaskDefStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n this.selectAllTaskDefsStatement =\n session.prepare(statements.getSelectAllTaskDefsStatement())\n .setConsistencyLevel(properties.getReadConsistencyLevel());\n\n this.updateWorkflowDefStatement =\n session.prepare(statements.getUpdateWorkflowDefStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n\n this.deleteWorkflowDefStatement =\n session.prepare(statements.getDeleteWorkflowDefStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteWorkflowDefIndexStatement =\n session.prepare(statements.getDeleteWorkflowDefIndexStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n this.deleteTaskDefStatement =\n session.prepare(statements.getDeleteTaskDefStatement())\n .setConsistencyLevel(properties.getWriteConsistencyLevel());\n }\n\n @Override\n public TaskDef createTaskDef(TaskDef taskDef) {\n return insertOrUpdateTaskDef(taskDef);\n }\n\n @Override\n public TaskDef updateTaskDef(TaskDef taskDef) {\n return insertOrUpdateTaskDef(taskDef);\n }\n\n @Override\n public TaskDef getTaskDef(String name) {\n return getTaskDefFromDB(name);\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao;\n\nimport java.util.List;\n\nimport com.netflix.conductor.common.metadata.tasks.PollData;\nimport com.netflix.conductor.dao.PollDataDAO;\n\n/**\n * This is a dummy implementation and this feature is not implemented for Cassandra backed\n * Conductor.\n */\npublic class CassandraPollDataDAO implements PollDataDAO {\n\n @Override\n public void updateLastPollData(String taskDefName, String domain, String workerId) {\n throw new UnsupportedOperationException(\n \"This method is not implemented in CassandraPollDataDAO. Please use ExecutionDAOFacade instead.\");\n }\n\n @Override\n public PollData getPollData(String taskDefName, String domain) {\n throw new UnsupportedOperationException(\n \"This method is not implemented in CassandraPollDataDAO. Please use ExecutionDAOFacade instead.\");\n }\n\n @Override\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.util;\n\npublic interface Constants {\n\n String DAO_NAME = \"cassandra\";\n\n String TABLE_WORKFLOWS = \"workflows\";\n String TABLE_TASK_LOOKUP = \"task_lookup\";\n String TABLE_TASK_DEF_LIMIT = \"task_def_limit\";\n String TABLE_WORKFLOW_DEFS = \"workflow_definitions\";\n String TABLE_WORKFLOW_DEFS_INDEX = \"workflow_defs_index\";\n String TABLE_TASK_DEFS = \"task_definitions\";\n String TABLE_EVENT_HANDLERS = \"event_handlers\";\n String TABLE_EVENT_EXECUTIONS = \"event_executions\";\n\n String WORKFLOW_ID_KEY = \"workflow_id\";\n String SHARD_ID_KEY = \"shard_id\";\n String TASK_ID_KEY = \"task_id\";\n String ENTITY_KEY = \"entity\";\n String PAYLOAD_KEY = \"payload\";\n String TOTAL_TASKS_KEY = \"total_tasks\";\n String TOTAL_PARTITIONS_KEY = \"total_partitions\";\n String TASK_DEF_NAME_KEY = \"task_def_name\";\n String WORKFLOW_DEF_NAME_KEY = \"workflow_def_name\";\n String WORKFLOW_VERSION_KEY = \"version\";\n String WORKFLOW_DEFINITION_KEY = \"workflow_definition\";\n String WORKFLOW_DEF_INDEX_KEY = \"workflow_def_version_index\";\n String WORKFLOW_DEF_INDEX_VALUE = \"workflow_def_index_value\";\n String WORKFLOW_DEF_NAME_VERSION_KEY = \"workflow_def_name_version\";\n String TASK_DEFS_KEY = \"task_defs\";\n String TASK_DEFINITION_KEY = \"task_definition\";\n String HANDLERS_KEY = \"handlers\";\n String EVENT_HANDLER_NAME_KEY = \"event_handler_name\";\n String EVENT_HANDLER_KEY = \"event_handler\";\n String MESSAGE_ID_KEY = \"message_id\";\n String EVENT_EXECUTION_ID_KEY = \"event_execution_id\";\n\n String ENTITY_TYPE_WORKFLOW = \"workflow\";\n String ENTITY_TYPE_TASK = \"task\";\n\n int DEFAULT_SHARD_ID = 1;\n int DEFAULT_TOTAL_PARTITIONS = 1;\n}\n"
},
{
"path": "cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/util/Statements.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.util;\n\nimport com.datastax.driver.core.querybuilder.QueryBuilder;\n\nimport static com.netflix.conductor.cassandra.util.Constants.ENTITY_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_TASK;\nimport static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_WORKFLOW;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_EXECUTION_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.EVENT_HANDLER_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.HANDLERS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.MESSAGE_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.PAYLOAD_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.SHARD_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_EXECUTIONS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_EVENT_HANDLERS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEFS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_DEF_LIMIT;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_TASK_LOOKUP;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOWS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS;\nimport static com.netflix.conductor.cassandra.util.Constants.TABLE_WORKFLOW_DEFS_INDEX;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEFS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_DEF_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TASK_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TOTAL_PARTITIONS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.TOTAL_TASKS_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEFINITION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_INDEX_VALUE;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_ID_KEY;\nimport static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_VERSION_KEY;\n\nimport static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;\nimport static com.datastax.driver.core.querybuilder.QueryBuilder.eq;\nimport static com.datastax.driver.core.querybuilder.QueryBuilder.set;\n\n/**\n * DML statements\n *\n * MetadataDAO\n *\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao\n\nimport com.netflix.conductor.common.metadata.events.EventExecution\nimport com.netflix.conductor.common.metadata.events.EventHandler\n\nimport spock.lang.Subject\n\nclass CassandraEventHandlerDAOSpec extends CassandraSpec {\n\n @Subject\n CassandraEventHandlerDAO eventHandlerDAO\n\n CassandraExecutionDAO executionDAO\n\n def setup() {\n eventHandlerDAO = new CassandraEventHandlerDAO(session, objectMapper, cassandraProperties, statements)\n executionDAO = new CassandraExecutionDAO(session, objectMapper, cassandraProperties, statements)\n }\n\n def testEventHandlerCRUD() {\n given:\n String event = \"event\"\n String eventHandlerName1 = \"event_handler1\"\n String eventHandlerName2 = \"event_handler2\"\n\n EventHandler eventHandler = new EventHandler()\n eventHandler.setName(eventHandlerName1)\n eventHandler.setEvent(event)\n\n when: // create event handler\n eventHandlerDAO.addEventHandler(eventHandler)\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao\n\nimport com.netflix.conductor.common.metadata.events.EventExecution\nimport com.netflix.conductor.common.metadata.tasks.TaskDef\nimport com.netflix.conductor.common.metadata.workflow.WorkflowDef\nimport com.netflix.conductor.common.metadata.workflow.WorkflowTask\nimport com.netflix.conductor.core.exception.NonTransientException\nimport com.netflix.conductor.core.utils.IDGenerator\nimport com.netflix.conductor.model.TaskModel\nimport com.netflix.conductor.model.WorkflowModel\n\nimport spock.lang.Subject\n\nimport static com.netflix.conductor.common.metadata.events.EventExecution.Status.COMPLETED\nimport static com.netflix.conductor.common.metadata.events.EventExecution.Status.IN_PROGRESS\n\nclass CassandraExecutionDAOSpec extends CassandraSpec {\n\n @Subject\n CassandraExecutionDAO executionDAO\n\n def setup() {\n executionDAO = new CassandraExecutionDAO(session, objectMapper, cassandraProperties, statements)\n }\n\n def \"verify if tasks are validated\"() {\n given:\n def tasks = []\n\n // create tasks for a workflow and add to list\n TaskModel task1 = new TaskModel(workflowInstanceId: 'uuid', taskId: 'task1id', referenceTaskName: 'task1')\n TaskModel task2 = new TaskModel(workflowInstanceId: 'uuid', taskId: 'task2id', referenceTaskName: 'task2')\n tasks << task1 << task2\n\n when:\n executionDAO.validateTasks(tasks)\n\n then:\n noExceptionThrown()\n\n and:\n // add a task from a different workflow to the list\n TaskModel task3 = new TaskModel(workflowInstanceId: 'other-uuid', taskId: 'task3id', referenceTaskName: 'task3')\n tasks << task3\n\n when:\n executionDAO.validateTasks(tasks)\n\n then:\n def ex = thrown(NonTransientException.class)\n ex.message == \"Tasks of multiple workflows cannot be created/updated simultaneously\"\n }\n\n def \"workflow CRUD\"() {\n given:\n String workflowId = new IDGenerator().generate()\n WorkflowDef workflowDef = new WorkflowDef()\n workflowDef.name = \"def1\"\n workflowDef.setVersion(1)\n WorkflowModel workflow = new WorkflowModel()\n workflow.setWorkflowDefinition(workflowDef)\n workflow.setWorkflowId(workflowId)\n workflow.setInput(new HashMap<>())\n workflow.setStatus(WorkflowModel.Status.RUNNING)\n workflow.setCreateTime(System.currentTimeMillis())\n\n when:\n // create a new workflow in the datastore\n String id = executionDAO.createWorkflow(workflow)\n\n then:\n workflowId == id\n\n when:\n // read the workflow from the datastore\n WorkflowModel found = executionDAO.getWorkflow(workflowId)\n\n then:\n workflow == found\n\n and:\n // update the workflow\n workflow.setStatus(WorkflowModel.Status.COMPLETED)\n executionDAO.updateWorkflow(workflow)\n\n when:\n found = executionDAO.getWorkflow(workflowId)\n\n then:\n workflow == found\n\n when:\n // remove the workflow from datastore\n boolean removed = executionDAO.removeWorkflow(workflowId)\n\n then:\n removed\n\n when:\n // read workflow again\n workflow = executionDAO.getWorkflow(workflowId, true)\n\n then:\n workflow == null\n }\n\n def \"create tasks and verify methods that read tasks and workflow\"() {\n given: 'we create a workflow'\n String workflowId = new IDGenerator().generate()\n WorkflowDef workflowDef = new WorkflowDef(name: 'def1', version: 1)\n WorkflowModel workflow = new WorkflowModel(workflowDefinition: workflowDef, workflowId: workflowId, input: new HashMap(), status: WorkflowModel.Status.RUNNING, createTime: System.currentTimeMillis())\n executionDAO.createWorkflow(workflow)\n\n and: 'create tasks for this workflow'\n TaskModel task1 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task1', referenceTaskName: 'task1', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate())\n TaskModel task2 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task2', referenceTaskName: 'task2', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate())\n TaskModel task3 = new TaskModel(workflowInstanceId: workflowId, taskType: 'task3', referenceTaskName: 'task3', status: TaskModel.Status.SCHEDULED, taskId: new IDGenerator().generate())\n\n def taskList = [task1, task2, task3]\n\n when: 'add the tasks to the datastore'\n List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao\n\nimport com.netflix.conductor.common.metadata.tasks.TaskDef\nimport com.netflix.conductor.common.metadata.workflow.WorkflowDef\n\nimport spock.lang.Subject\n\nclass CassandraMetadataDAOSpec extends CassandraSpec {\n\n @Subject\n CassandraMetadataDAO metadataDAO\n\n def setup() {\n metadataDAO = new CassandraMetadataDAO(session, objectMapper, cassandraProperties, statements)\n }\n\n def cleanup() {\n\n }\n\n def \"CRUD on WorkflowDef\"() throws Exception {\n given:\n String name = \"workflow_def_1\"\n int version = 1\n\n WorkflowDef workflowDef = new WorkflowDef()\n workflowDef.setName(name)\n workflowDef.setVersion(version)\n workflowDef.setOwnerEmail(\"test@junit.com\")\n\n when: 'create workflow definition'\n metadataDAO.createWorkflowDef(workflowDef)\n\n then: // fetch the workflow definition\n def defOptional = metadataDAO.getWorkflowDef(name, version)\n defOptional.present\n defOptional.get() == workflowDef\n\n and: // register a higher version\n int higherVersion = 2\n workflowDef.setVersion(higherVersion)\n workflowDef.setDescription(\"higher version\")\n\n when: // register the higher version definition\n metadataDAO.createWorkflowDef(workflowDef)\n defOptional = metadataDAO.getWorkflowDef(name, higherVersion)\n\n then: // fetch the higher version\n defOptional.present\n defOptional.get() == workflowDef\n\n when: // fetch latest version\n defOptional = metadataDAO.getLatestWorkflowDef(name)\n\n then:\n defOptional && defOptional.present\n defOptional.get() == workflowDef\n\n when: // modify the definition\n workflowDef.setOwnerEmail(\"test@junit.com\")\n metadataDAO.updateWorkflowDef(workflowDef)\n defOptional = metadataDAO.getWorkflowDef(name, higherVersion)\n\n then: // fetch the workflow definition\n defOptional.present\n defOptional.get() == workflowDef\n\n when: // delete workflow def\n metadataDAO.removeWorkflowDef(name, higherVersion)\n defOptional = metadataDAO.getWorkflowDef(name, higherVersion)\n\n then:\n defOptional.empty\n }\n\n def \"CRUD on TaskDef\"() {\n given:\n String task1Name = \"task1\"\n String task2Name = \"task2\"\n\n when: // fetch all task defs\n def taskDefList = metadataDAO.getAllTaskDefs()\n\n then:\n taskDefList.empty\n\n when: // register a task definition\n TaskDef taskDef = new TaskDef()\n taskDef.setName(task1Name)\n metadataDAO.createTaskDef(taskDef)\n taskDefList = metadataDAO.getAllTaskDefs()\n\n then: // fetch all task defs\n taskDefList && taskDefList.size() == 1\n\n when: // fetch the task def\n def returnTaskDef = metadataDAO.getTaskDef(task1Name)\n\n then:\n returnTaskDef == taskDef\n\n when: // register another task definition\n TaskDef taskDef1 = new TaskDef()\n taskDef1.setName(task2Name)\n metadataDAO.createTaskDef(taskDef1)\n // fetch all task defs\n taskDefList = metadataDAO.getAllTaskDefs()\n\n then:\n taskDefList && taskDefList.size() == 2\n\n when: // update task def\n taskDef.setOwnerEmail(\"juni@test.com\")\n metadataDAO.updateTaskDef(taskDef)\n returnTaskDef = metadataDAO.getTaskDef(task1Name)\n\n then:\n returnTaskDef == taskDef\n\n when: // delete task def\n metadataDAO.removeTaskDef(task2Name)\n taskDefList = metadataDAO.getAllTaskDefs()\n\n then:\n taskDefList && taskDefList.size() == 1\n // fetch deleted task def\n metadataDAO.getTaskDef(task2Name) == null\n }\n\n def \"set default response timeout when not set\"() {\n given:\n String task1Name = \"task1\"\n\n when: // register a task definition\n TaskDef taskDef = new TaskDef()\n taskDef.setName(task1Name)\n taskDef.setResponseTimeoutSeconds(0)\n metadataDAO.createTaskDef(taskDef)\n def returnTaskDef = metadataDAO.getTaskDef(task1Name)\n\n then:\n returnTaskDef.getResponseTimeoutSeconds() == 3600\n\n when: // register another task definition\n taskDef.setTimeoutSeconds(200)\n taskDef.setResponseTimeoutSeconds(0)\n metadataDAO.updateTaskDef(taskDef)\n // fetch all task defs\n def taskDefList = metadataDAO.getAllTaskDefs()\n\n then:\n taskDefList && taskDefList.size() == 1\n taskDefList.get(0).getResponseTimeoutSeconds() == 199\n\n }\n\n def \"Get All WorkflowDef\"() {\n when:\n metadataDAO.removeWorkflowDef(\"workflow_def_1\", 1)\n WorkflowDef workflowDef = new WorkflowDef()\n workflowDef.setName(\"workflow_def_1\")\n workflowDef.setVersion(1)\n workflowDef.setOwnerEmail(\"test@junit.com\")\n metadataDAO.createWorkflowDef(workflowDef)\n\n workflowDef.setName(\"workflow_def_2\")\n metadataDAO.createWorkflowDef(workflowDef)\n workflowDef.setVersion(2)\n metadataDAO.createWorkflowDef(workflowDef)\n\n workflowDef.setName(\"workflow_def_3\")\n workflowDef.setVersion(1)\n metadataDAO.createWorkflowDef(workflowDef)\n workflowDef.setVersion(2)\n metadataDAO.createWorkflowDef(workflowDef)\n workflowDef.setVersion(3)\n metadataDAO.createWorkflowDef(workflowDef)\n\n then: // fetch the workflow definition\n def allDefsLatestVersions = metadataDAO.getAllWorkflowDefsLatestVersions()\n Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.dao\n\nimport java.time.Duration\n\nimport org.springframework.beans.factory.annotation.Autowired\nimport org.springframework.test.context.ContextConfiguration\nimport org.testcontainers.containers.CassandraContainer\nimport org.testcontainers.spock.Testcontainers\n\nimport com.netflix.conductor.cassandra.config.CassandraProperties\nimport com.netflix.conductor.cassandra.util.Statements\nimport com.netflix.conductor.common.config.TestObjectMapperConfiguration\n\nimport com.datastax.driver.core.ConsistencyLevel\nimport com.datastax.driver.core.Session\nimport com.fasterxml.jackson.databind.ObjectMapper\nimport groovy.transform.PackageScope\nimport spock.lang.Shared\nimport spock.lang.Specification\n\n@ContextConfiguration(classes = [TestObjectMapperConfiguration.class])\n@Testcontainers\n@PackageScope\nabstract class CassandraSpec extends Specification {\n\n @Shared\n CassandraContainer cassandra = new CassandraContainer()\n\n @Shared\n Session session\n\n @Autowired\n ObjectMapper objectMapper\n\n CassandraProperties cassandraProperties\n Statements statements\n\n def setupSpec() {\n session = cassandra.cluster.newSession()\n }\n\n def setup() {\n String keyspaceName = \"junit\"\n cassandraProperties = Mock(CassandraProperties.class) {\n getKeyspace() >> keyspaceName\n getReplicationStrategy() >> \"SimpleStrategy\"\n getReplicationFactorKey() >> \"replication_factor\"\n getReplicationFactorValue() >> 1\n getReadConsistencyLevel() >> ConsistencyLevel.LOCAL_ONE\n getWriteConsistencyLevel() >> ConsistencyLevel.LOCAL_ONE\n getTaskDefCacheRefreshInterval() >> Duration.ofSeconds(60)\n getEventHandlerCacheRefreshInterval() >> Duration.ofSeconds(60)\n getEventExecutionPersistenceTtl() >> Duration.ofSeconds(5)\n }\n\n statements = new Statements(keyspaceName)\n }\n}\n"
},
{
"path": "cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/util/StatementsSpec.groovy",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.cassandra.util\n\nimport spock.lang.Specification\nimport spock.lang.Subject\n\nclass StatementsSpec extends Specification {\n\n @Subject\n Statements subject\n\n def setup() {\n subject = new Statements('test')\n }\n\n def \"verify statements\"() {\n when:\n subject\n\n then:\n with(subject) {\n insertWorkflowDefStatement == \"INSERT INTO test.workflow_definitions (workflow_def_name,version,workflow_definition) VALUES (?,?,?) IF NOT EXISTS;\"\n insertTaskDefStatement == \"INSERT INTO test.task_definitions (task_defs,task_def_name,task_definition) VALUES ('task_defs',?,?);\"\n selectWorkflowDefStatement == \"SELECT workflow_definition FROM test.workflow_definitions WHERE workflow_def_name=? AND version=?;\"\n selectAllWorkflowDefVersionsByNameStatement == \"SELECT * FROM test.workflow_definitions WHERE workflow_def_name=?;\"\n selectAllWorkflowDefsStatement == \"SELECT * FROM test.workflow_defs_index WHERE workflow_def_version_index=?;\"\n selectTaskDefStatement == \"SELECT task_definition FROM test.task_definitions WHERE task_defs='task_defs' AND task_def_name=?;\"\n selectAllTaskDefsStatement == \"SELECT * FROM test.task_definitions WHERE task_defs=?;\"\n updateWorkflowDefStatement == \"UPDATE test.workflow_definitions SET workflow_definition=? WHERE workflow_def_name=? AND version=?;\"\n deleteWorkflowDefStatement == \"DELETE FROM test.workflow_definitions WHERE workflow_def_name=? AND version=?;\"\n deleteWorkflowDefIndexStatement == \"DELETE FROM test.workflow_defs_index WHERE workflow_def_version_index=? AND workflow_def_name_version=?;\"\n deleteTaskDefStatement == \"DELETE FROM test.task_definitions WHERE task_defs='task_defs' AND task_def_name=?;\"\n insertWorkflowStatement == \"INSERT INTO test.workflows (workflow_id,shard_id,task_id,entity,payload,total_tasks,total_partitions) VALUES (?,?,?,'workflow',?,?,?);\"\n insertTaskStatement == \"INSERT INTO test.workflows (workflow_id,shard_id,task_id,entity,payload) VALUES (?,?,?,'task',?);\"\n insertEventExecutionStatement == \"INSERT INTO test.event_executions (message_id,event_handler_name,event_execution_id,payload) VALUES (?,?,?,?) IF NOT EXISTS;\"\n selectTotalStatement == \"SELECT total_tasks,total_partitions FROM test.workflows WHERE workflow_id=? AND shard_id=1;\"\n selectTaskStatement == \"SELECT payload FROM test.workflows WHERE workflow_id=? AND shard_id=? AND entity='task' AND task_id=?;\"\n selectWorkflowStatement == \"SELECT payload FROM test.workflows WHERE workflow_id=? AND shard_id=1 AND entity='workflow';\"\n selectWorkflowWithTasksStatement == \"SELECT * FROM test.workflows WHERE workflow_id=? AND shard_id=?;\"\n selectTaskFromLookupTableStatement == \"SELECT workflow_id FROM test.task_lookup WHERE task_id=?;\"\n selectTasksFromTaskDefLimitStatement == \"SELECT * FROM test.task_def_limit WHERE task_def_name=?;\"\n selectAllEventExecutionsForMessageFromEventExecutionsStatement == \"SELECT * FROM test.event_executions WHERE message_id=? AND event_handler_name=?;\"\n updateWorkflowStatement == \"UPDATE test.workflows SET payload=? WHERE workflow_id=? AND shard_id=1 AND entity='workflow' AND task_id='';\"\n updateTotalTasksStatement == \"UPDATE test.workflows SET total_tasks=? WHERE workflow_id=? AND shard_id=?;\"\n updateTotalPartitionsStatement == \"UPDATE test.workflows SET total_partitions=?,total_tasks=? WHERE workflow_id=? AND shard_id=1;\"\n updateTaskLookupStatement == \"UPDATE test.task_lookup SET workflow_id=? WHERE task_id=?;\"\n updateTaskDefLimitStatement == \"UPDATE test.task_def_limit SET workflow_id=? WHERE task_def_name=? AND task_id=?;\"\n updateEventExecutionStatement == \"UPDATE test.event_executions USING TTL ? SET payload=? WHERE message_id=? AND event_handler_name=? AND event_execution_id=?;\"\n deleteWorkflowStatement == \"DELETE FROM test.workflows WHERE workflow_id=? AND shard_id=?;\"\n deleteTaskLookupStatement == \"DELETE FROM test.task_lookup WHERE task_id=?;\"\n deleteTaskStatement == \"DELETE FROM test.workflows WHERE workflow_id=? AND shard_id=? AND entity='task' AND task_id=?;\"\n deleteTaskDefLimitStatement == \"DELETE FROM test.task_def_limit WHERE task_def_name=? AND task_id=?;\"\n deleteEventExecutionsStatement == \"DELETE FROM test.event_executions WHERE message_id=? AND event_handler_name=? AND event_execution_id=?;\"\n insertEventHandlerStatement == \"INSERT INTO test.event_handlers (handlers,event_handler_name,event_handler) VALUES ('handlers',?,?);\"\n selectAllEventHandlersStatement == \"SELECT * FROM test.event_handlers WHERE handlers=?;\"\n deleteEventHandlerStatement == \"DELETE FROM test.event_handlers WHERE handlers='handlers' AND event_handler_name=?;\"\n }\n }\n}\n"
},
{
"path": "client/build.gradle",
"content": "buildscript {\n repositories {\n maven {\n url \"https://plugins.gradle.org/m2/\"\n }\n }\n dependencies {\n classpath \"gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.7.5\"\n }\n}\n\napply plugin: 'groovy'\n\nconfigurations.all {\n exclude group: 'amazon', module: 'aws-java-sdk'\n}\n\ndependencies {\n compileOnly 'org.jetbrains:annotations:23.0.0'\n\n implementation project(':conductor-common')\n implementation \"com.sun.jersey:jersey-client:${revJersey}\"\n implementation \"javax.ws.rs:javax.ws.rs-api:${revJAXRS}\"\n implementation \"org.glassfish.jersey.core:jersey-common:${revJerseyCommon}\"\n\n implementation \"com.netflix.spectator:spectator-api:${revSpectator}\"\n implementation (\"com.netflix.eureka:eureka-client:${revEurekaClient}\") {\n exclude group: 'com.google.guava', module: 'guava'\n }\n implementation \"com.amazonaws:aws-java-sdk-core:${revAwsSdk}\"\n\n implementation \"com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider\"\n implementation \"com.fasterxml.jackson.datatype:jackson-datatype-jsr310\"\n\n implementation \"org.apache.commons:commons-lang3\"\n implementation \"commons-io:commons-io:${revCommonsIo}\"\n\n implementation \"org.slf4j:slf4j-api\"\n\n testImplementation \"org.powermock:powermock-module-junit4:${revPowerMock}\"\n testImplementation \"org.powermock:powermock-api-mockito2:${revPowerMock}\"\n\n testImplementation \"org.codehaus.groovy:groovy-all:${revGroovy}\"\n testImplementation \"org.spockframework:spock-core:${revSpock}\"\n testImplementation \"org.spockframework:spock-spring:${revSpock}\"\n}\n"
},
{
"path": "client/spotbugsExclude.xml",
"content": "\n \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.automator;\n\nimport java.util.concurrent.Semaphore;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n * A class wrapping a semaphore which holds the number of permits available for polling and\n * executing tasks.\n */\nclass PollingSemaphore {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(PollingSemaphore.class);\n private final Semaphore semaphore;\n\n PollingSemaphore(int numSlots) {\n LOGGER.debug(\"Polling semaphore initialized with {} permits\", numSlots);\n semaphore = new Semaphore(numSlots);\n }\n\n /** Signals that processing is complete and the specified number of permits can be released. */\n void complete(int numSlots) {\n LOGGER.debug(\"Completed execution; releasing permit\");\n semaphore.release(numSlots);\n }\n\n /**\n * Gets the number of threads available for processing.\n *\n * @return number of available permits\n */\n int availableSlots() {\n int available = semaphore.availablePermits();\n LOGGER.debug(\"Number of available permits: {}\", available);\n return available;\n }\n\n /**\n * Signals if processing is allowed based on whether specified number of permits can be\n * acquired.\n *\n * @param numSlots the number of permits to acquire\n * @return {@code true} - if permit is acquired {@code false} - if permit could not be acquired\n */\n public boolean acquireSlots(int numSlots) {\n boolean acquired = semaphore.tryAcquire(numSlots);\n LOGGER.debug(\"Trying to acquire {} permit: {}\", numSlots, acquired);\n return acquired;\n }\n}\n"
},
{
"path": "client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.automator;\n\nimport java.io.PrintWriter;\nimport java.io.StringWriter;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.concurrent.*;\nimport java.util.function.Function;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.commons.lang3.time.StopWatch;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.appinfo.InstanceInfo.InstanceStatus;\nimport com.netflix.conductor.client.config.PropertyFactory;\nimport com.netflix.conductor.client.http.TaskClient;\nimport com.netflix.conductor.client.telemetry.MetricsContainer;\nimport com.netflix.conductor.client.worker.Worker;\nimport com.netflix.conductor.common.metadata.tasks.Task;\nimport com.netflix.conductor.common.metadata.tasks.TaskResult;\nimport com.netflix.discovery.EurekaClient;\nimport com.netflix.spectator.api.Registry;\nimport com.netflix.spectator.api.Spectator;\nimport com.netflix.spectator.api.patterns.ThreadPoolMonitor;\n\n/**\n * Manages the threadpool used by the workers for execution and server communication (polling and\n * task update).\n */\nclass TaskPollExecutor {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(TaskPollExecutor.class);\n\n private static final Registry REGISTRY = Spectator.globalRegistry();\n\n private final EurekaClient eurekaClient;\n private final TaskClient taskClient;\n private final int updateRetryCount;\n private final ExecutorService executorService;\n private final Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.automator;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\n\nimport org.apache.commons.lang3.Validate;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.client.exception.ConductorClientException;\nimport com.netflix.conductor.client.http.TaskClient;\nimport com.netflix.conductor.client.worker.Worker;\nimport com.netflix.discovery.EurekaClient;\n\n/** Configures automated polling of tasks and execution via the registered {@link Worker}s. */\npublic class TaskRunnerConfigurer {\n private static final Logger LOGGER = LoggerFactory.getLogger(TaskRunnerConfigurer.class);\n private static final String INVALID_THREAD_COUNT =\n \"Invalid worker thread count specified, use either shared thread pool or config thread count per task\";\n private static final String MISSING_TASK_THREAD_COUNT =\n \"Missing task thread count config for %s\";\n\n private ScheduledExecutorService scheduledExecutorService;\n\n private final EurekaClient eurekaClient;\n private final TaskClient taskClient;\n private final List Please see {@link TaskRunnerConfigurer#init()} method. The method must be called after\n * this constructor for the polling to start.\n */\n public TaskRunnerConfigurer build() {\n return new TaskRunnerConfigurer(this);\n }\n }\n\n /**\n * @return Thread Count for the shared executor pool\n */\n @Deprecated\n public int getThreadCount() {\n return threadCount;\n }\n\n /**\n * @return Thread Count for individual task type\n */\n public Map \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.config;\n\npublic interface ConductorClientConfiguration {\n\n /**\n * @return the workflow input payload size threshold in KB, beyond which the payload will be\n * processed based on {@link\n * ConductorClientConfiguration#isExternalPayloadStorageEnabled()}.\n */\n int getWorkflowInputPayloadThresholdKB();\n\n /**\n * @return the max value of workflow input payload size threshold in KB, beyond which the\n * payload will be rejected regardless external payload storage is enabled.\n */\n int getWorkflowInputMaxPayloadThresholdKB();\n\n /**\n * @return the task output payload size threshold in KB, beyond which the payload will be\n * processed based on {@link\n * ConductorClientConfiguration#isExternalPayloadStorageEnabled()}.\n */\n int getTaskOutputPayloadThresholdKB();\n\n /**\n * @return the max value of task output payload size threshold in KB, beyond which the payload\n * will be rejected regardless external payload storage is enabled.\n */\n int getTaskOutputMaxPayloadThresholdKB();\n\n /**\n * @return the flag which controls the use of external storage for storing workflow/task input\n * and output JSON payloads with size greater than threshold. If it is set to true, the\n * payload is stored in external location. If it is set to false, the payload is rejected\n * and the task/workflow execution fails.\n */\n boolean isExternalPayloadStorageEnabled();\n}\n"
},
{
"path": "client/src/main/java/com/netflix/conductor/client/config/DefaultConductorClientConfiguration.java",
"content": "/*\n * Copyright 2020 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.config;\n\n/**\n * A default implementation of {@link ConductorClientConfiguration} where external payload storage\n * is disabled.\n */\npublic class DefaultConductorClientConfiguration implements ConductorClientConfiguration {\n\n @Override\n public int getWorkflowInputPayloadThresholdKB() {\n return 5120;\n }\n\n @Override\n public int getWorkflowInputMaxPayloadThresholdKB() {\n return 10240;\n }\n\n @Override\n public int getTaskOutputPayloadThresholdKB() {\n return 3072;\n }\n\n @Override\n public int getTaskOutputMaxPayloadThresholdKB() {\n return 10240;\n }\n\n @Override\n public boolean isExternalPayloadStorageEnabled() {\n return false;\n }\n}\n"
},
{
"path": "client/src/main/java/com/netflix/conductor/client/config/PropertyFactory.java",
"content": "/*\n * Copyright 2020 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.config;\n\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport com.netflix.config.DynamicProperty;\n\n/** Used to configure the Conductor workers using properties. */\npublic class PropertyFactory {\n\n private final DynamicProperty global;\n private final DynamicProperty local;\n\n private static final String PROPERTY_PREFIX = \"conductor.worker\";\n\n private static final ConcurrentHashMap \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.exception;\n\nimport java.util.List;\n\nimport com.netflix.conductor.common.validation.ErrorResponse;\nimport com.netflix.conductor.common.validation.ValidationError;\n\n/** Client exception thrown from Conductor api clients. */\npublic class ConductorClientException extends RuntimeException {\n\n private int status;\n private String message;\n private String instance;\n private String code;\n private boolean retryable;\n\n public List \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.net.URI;\nimport java.util.Collection;\nimport java.util.Map;\nimport java.util.function.Function;\n\nimport javax.ws.rs.core.UriBuilder;\n\nimport org.apache.commons.lang3.ObjectUtils;\nimport org.apache.commons.lang3.Validate;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.client.config.ConductorClientConfiguration;\nimport com.netflix.conductor.client.config.DefaultConductorClientConfiguration;\nimport com.netflix.conductor.client.exception.ConductorClientException;\nimport com.netflix.conductor.common.config.ObjectMapperProvider;\nimport com.netflix.conductor.common.model.BulkResponse;\nimport com.netflix.conductor.common.run.ExternalStorageLocation;\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage;\nimport com.netflix.conductor.common.validation.ErrorResponse;\n\nimport com.fasterxml.jackson.core.Version;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;\nimport com.sun.jersey.api.client.ClientHandlerException;\nimport com.sun.jersey.api.client.ClientResponse;\nimport com.sun.jersey.api.client.GenericType;\nimport com.sun.jersey.api.client.UniformInterfaceException;\nimport com.sun.jersey.api.client.WebResource.Builder;\n\n/** Abstract client for the REST server */\npublic abstract class ClientBase {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(ClientBase.class);\n\n protected ClientRequestHandler requestHandler;\n\n protected String root = \"\";\n\n protected ObjectMapper objectMapper;\n\n protected PayloadStorage payloadStorage;\n\n protected ConductorClientConfiguration conductorClientConfiguration;\n\n protected ClientBase(\n ClientRequestHandler requestHandler, ConductorClientConfiguration clientConfiguration) {\n this.objectMapper = new ObjectMapperProvider().getObjectMapper();\n\n // https://github.com/FasterXML/jackson-databind/issues/2683\n if (isNewerJacksonVersion()) {\n objectMapper.registerModule(new JavaTimeModule());\n }\n\n this.requestHandler = requestHandler;\n this.conductorClientConfiguration =\n ObjectUtils.defaultIfNull(\n clientConfiguration, new DefaultConductorClientConfiguration());\n this.payloadStorage = new PayloadStorage(this);\n }\n\n public void setRootURI(String root) {\n this.root = root;\n }\n\n protected void delete(String url, Object... uriVariables) {\n deleteWithUriVariables(null, url, uriVariables);\n }\n\n protected void deleteWithUriVariables(\n Object[] queryParams, String url, Object... uriVariables) {\n delete(queryParams, url, uriVariables, null);\n }\n\n protected BulkResponse deleteWithRequestBody(Object[] queryParams, String url, Object body) {\n return delete(queryParams, url, null, body);\n }\n\n private BulkResponse delete(\n Object[] queryParams, String url, Object[] uriVariables, Object body) {\n URI uri = null;\n BulkResponse response = null;\n try {\n uri = getURIBuilder(root + url, queryParams).build(uriVariables);\n response = requestHandler.delete(uri, body);\n } catch (UniformInterfaceException e) {\n handleUniformInterfaceException(e, uri);\n } catch (RuntimeException e) {\n handleRuntimeException(e, uri);\n }\n return response;\n }\n\n protected void put(String url, Object[] queryParams, Object request, Object... uriVariables) {\n URI uri = null;\n try {\n uri = getURIBuilder(root + url, queryParams).build(uriVariables);\n requestHandler.getWebResourceBuilder(uri, request).put();\n } catch (RuntimeException e) {\n handleException(uri, e);\n }\n }\n\n protected void postForEntityWithRequestOnly(String url, Object request) {\n Class type = null;\n postForEntity(url, request, null, type);\n }\n\n protected void postForEntityWithUriVariablesOnly(String url, Object... uriVariables) {\n Class type = null;\n postForEntity(url, null, null, type, uriVariables);\n }\n\n protected \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.net.URI;\n\nimport javax.ws.rs.core.MediaType;\n\nimport com.netflix.conductor.common.config.ObjectMapperProvider;\nimport com.netflix.conductor.common.model.BulkResponse;\n\nimport com.fasterxml.jackson.core.Version;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;\nimport com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;\nimport com.sun.jersey.api.client.Client;\nimport com.sun.jersey.api.client.ClientHandler;\nimport com.sun.jersey.api.client.ClientResponse;\nimport com.sun.jersey.api.client.WebResource;\nimport com.sun.jersey.api.client.config.ClientConfig;\nimport com.sun.jersey.api.client.filter.ClientFilter;\n\npublic class ClientRequestHandler {\n private final Client client;\n\n public ClientRequestHandler(\n ClientConfig config, ClientHandler handler, ClientFilter... filters) {\n ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper();\n\n // https://github.com/FasterXML/jackson-databind/issues/2683\n if (isNewerJacksonVersion()) {\n objectMapper.registerModule(new JavaTimeModule());\n }\n\n JacksonJsonProvider provider = new JacksonJsonProvider(objectMapper);\n config.getSingletons().add(provider);\n\n if (handler == null) {\n this.client = Client.create(config);\n } else {\n this.client = new Client(handler, config);\n }\n\n for (ClientFilter filter : filters) {\n this.client.addFilter(filter);\n }\n }\n\n public BulkResponse delete(URI uri, Object body) {\n if (body != null) {\n return client.resource(uri)\n .type(MediaType.APPLICATION_JSON_TYPE)\n .delete(BulkResponse.class, body);\n } else {\n client.resource(uri).delete();\n }\n return null;\n }\n\n public ClientResponse get(URI uri) {\n return client.resource(uri)\n .accept(MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN)\n .get(ClientResponse.class);\n }\n\n public WebResource.Builder getWebResourceBuilder(URI URI, Object entity) {\n return client.resource(URI)\n .type(MediaType.APPLICATION_JSON)\n .entity(entity)\n .accept(MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON);\n }\n\n private boolean isNewerJacksonVersion() {\n Version version = com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION;\n return version.getMajorVersion() == 2 && version.getMinorVersion() >= 12;\n }\n}\n"
},
{
"path": "client/src/main/java/com/netflix/conductor/client/http/EventClient.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.util.List;\n\nimport org.apache.commons.lang3.Validate;\n\nimport com.netflix.conductor.client.config.ConductorClientConfiguration;\nimport com.netflix.conductor.client.config.DefaultConductorClientConfiguration;\nimport com.netflix.conductor.common.metadata.events.EventHandler;\n\nimport com.sun.jersey.api.client.ClientHandler;\nimport com.sun.jersey.api.client.GenericType;\nimport com.sun.jersey.api.client.config.ClientConfig;\nimport com.sun.jersey.api.client.config.DefaultClientConfig;\nimport com.sun.jersey.api.client.filter.ClientFilter;\n\n// Client class for all Event Handler operations\npublic class EventClient extends ClientBase {\n private static final GenericType \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.util.List;\n\nimport org.apache.commons.lang3.Validate;\n\nimport com.netflix.conductor.client.config.ConductorClientConfiguration;\nimport com.netflix.conductor.client.config.DefaultConductorClientConfiguration;\nimport com.netflix.conductor.common.metadata.tasks.TaskDef;\nimport com.netflix.conductor.common.metadata.workflow.WorkflowDef;\n\nimport com.sun.jersey.api.client.ClientHandler;\nimport com.sun.jersey.api.client.GenericType;\nimport com.sun.jersey.api.client.config.ClientConfig;\nimport com.sun.jersey.api.client.config.DefaultClientConfig;\nimport com.sun.jersey.api.client.filter.ClientFilter;\n\npublic class MetadataClient extends ClientBase {\n\n private static final GenericType \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.net.HttpURLConnection;\nimport java.net.MalformedURLException;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.URL;\n\nimport javax.ws.rs.core.Response;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.client.exception.ConductorClientException;\nimport com.netflix.conductor.common.run.ExternalStorageLocation;\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage;\n\nimport com.amazonaws.util.IOUtils;\n\n/** An implementation of {@link ExternalPayloadStorage} for storing large JSON payload data. */\nclass PayloadStorage implements ExternalPayloadStorage {\n\n private static final Logger LOGGER = LoggerFactory.getLogger(PayloadStorage.class);\n\n private final ClientBase clientBase;\n\n PayloadStorage(ClientBase clientBase) {\n this.clientBase = clientBase;\n }\n\n /**\n * This method is not intended to be used in the client. The client makes a request to the\n * server to get the {@link ExternalStorageLocation}\n */\n @Override\n public ExternalStorageLocation getLocation(\n Operation operation, PayloadType payloadType, String path) {\n String uri;\n switch (payloadType) {\n case WORKFLOW_INPUT:\n case WORKFLOW_OUTPUT:\n uri = \"workflow\";\n break;\n case TASK_INPUT:\n case TASK_OUTPUT:\n uri = \"tasks\";\n break;\n default:\n throw new ConductorClientException(\n String.format(\n \"Invalid payload type: %s for operation: %s\",\n payloadType.toString(), operation.toString()));\n }\n return clientBase.getForEntity(\n String.format(\"%s/externalstoragelocation\", uri),\n new Object[] {\n \"path\",\n path,\n \"operation\",\n operation.toString(),\n \"payloadType\",\n payloadType.toString()\n },\n ExternalStorageLocation.class);\n }\n\n /**\n * Uploads the payload to the uri specified.\n *\n * @param uri the location to which the object is to be uploaded\n * @param payload an {@link InputStream} containing the json payload which is to be uploaded\n * @param payloadSize the size of the json payload in bytes\n * @throws ConductorClientException if the upload fails due to an invalid path or an error from\n * external storage\n */\n @Override\n public void upload(String uri, InputStream payload, long payloadSize) {\n HttpURLConnection connection = null;\n try {\n URL url = new URI(uri).toURL();\n\n connection = (HttpURLConnection) url.openConnection();\n connection.setDoOutput(true);\n connection.setRequestMethod(\"PUT\");\n\n try (BufferedOutputStream bufferedOutputStream =\n new BufferedOutputStream(connection.getOutputStream())) {\n long count = IOUtils.copy(payload, bufferedOutputStream);\n bufferedOutputStream.flush();\n // Check the HTTP response code\n int responseCode = connection.getResponseCode();\n if (Response.Status.fromStatusCode(responseCode).getFamily()\n != Response.Status.Family.SUCCESSFUL) {\n String errorMsg =\n String.format(\"Unable to upload. Response code: %d\", responseCode);\n LOGGER.error(errorMsg);\n throw new ConductorClientException(errorMsg);\n }\n LOGGER.debug(\n \"Uploaded {} bytes to uri: {}, with HTTP response code: {}\",\n count,\n uri,\n responseCode);\n }\n } catch (URISyntaxException | MalformedURLException e) {\n String errorMsg = String.format(\"Invalid path specified: %s\", uri);\n LOGGER.error(errorMsg, e);\n throw new ConductorClientException(errorMsg, e);\n } catch (IOException e) {\n String errorMsg = String.format(\"Error uploading to path: %s\", uri);\n LOGGER.error(errorMsg, e);\n throw new ConductorClientException(errorMsg, e);\n } finally {\n if (connection != null) {\n connection.disconnect();\n }\n try {\n if (payload != null) {\n payload.close();\n }\n } catch (IOException e) {\n LOGGER.warn(\"Unable to close inputstream when uploading to uri: {}\", uri);\n }\n }\n }\n\n /**\n * Downloads the payload from the given uri.\n *\n * @param uri the location from where the object is to be downloaded\n * @return an inputstream of the payload in the external storage\n * @throws ConductorClientException if the download fails due to an invalid path or an error\n * from external storage\n */\n @Override\n public InputStream download(String uri) {\n HttpURLConnection connection = null;\n String errorMsg;\n try {\n URL url = new URI(uri).toURL();\n connection = (HttpURLConnection) url.openConnection();\n connection.setDoOutput(false);\n\n // Check the HTTP response code\n int responseCode = connection.getResponseCode();\n if (responseCode == HttpURLConnection.HTTP_OK) {\n LOGGER.debug(\n \"Download completed with HTTP response code: {}\",\n connection.getResponseCode());\n return org.apache.commons.io.IOUtils.toBufferedInputStream(\n connection.getInputStream());\n }\n errorMsg = String.format(\"Unable to download. Response code: %d\", responseCode);\n LOGGER.error(errorMsg);\n throw new ConductorClientException(errorMsg);\n } catch (URISyntaxException | MalformedURLException e) {\n errorMsg = String.format(\"Invalid uri specified: %s\", uri);\n LOGGER.error(errorMsg, e);\n throw new ConductorClientException(errorMsg, e);\n } catch (IOException e) {\n errorMsg = String.format(\"Error downloading from uri: %s\", uri);\n LOGGER.error(errorMsg, e);\n throw new ConductorClientException(errorMsg, e);\n } finally {\n if (connection != null) {\n connection.disconnect();\n }\n }\n }\n}\n"
},
{
"path": "client/src/main/java/com/netflix/conductor/client/http/TaskClient.java",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n * \n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage com.netflix.conductor.client.http;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\n\nimport org.apache.commons.lang3.ObjectUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.Validate;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport com.netflix.conductor.client.config.ConductorClientConfiguration;\nimport com.netflix.conductor.client.config.DefaultConductorClientConfiguration;\nimport com.netflix.conductor.client.exception.ConductorClientException;\nimport com.netflix.conductor.client.telemetry.MetricsContainer;\nimport com.netflix.conductor.common.metadata.tasks.PollData;\nimport com.netflix.conductor.common.metadata.tasks.Task;\nimport com.netflix.conductor.common.metadata.tasks.TaskExecLog;\nimport com.netflix.conductor.common.metadata.tasks.TaskResult;\nimport com.netflix.conductor.common.run.SearchResult;\nimport com.netflix.conductor.common.run.TaskSummary;\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage;\nimport com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType;\n\nimport com.sun.jersey.api.client.ClientHandler;\nimport com.sun.jersey.api.client.GenericType;\nimport com.sun.jersey.api.client.config.ClientConfig;\nimport com.sun.jersey.api.client.config.DefaultClientConfig;\nimport com.sun.jersey.api.client.filter.ClientFilter;\n\n/** Client for conductor task management including polling for task, updating task status etc. */\npublic class TaskClient extends ClientBase {\n\n private static final GenericType\n *
\n *\n * ExecutionDAO\n *\n * \n *
\n *\n * EventHandlerDAO\n *\n * \n *
\n */\npublic class Statements {\n\n private final String keyspace;\n\n public Statements(String keyspace) {\n this.keyspace = keyspace;\n }\n\n // MetadataDAO\n // Insert Statements\n\n /**\n * @return cql query statement to insert a new workflow definition into the\n * \"workflow_definitions\" table\n */\n public String getInsertWorkflowDefStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOW_DEFS)\n .value(WORKFLOW_DEF_NAME_KEY, bindMarker())\n .value(WORKFLOW_VERSION_KEY, bindMarker())\n .value(WORKFLOW_DEFINITION_KEY, bindMarker())\n .ifNotExists()\n .getQueryString();\n }\n\n /**\n * @return cql query statement to insert a workflow def name version index into the\n * \"workflow_defs_index\" table\n */\n public String getInsertWorkflowDefVersionIndexStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOW_DEFS_INDEX)\n .value(WORKFLOW_DEF_INDEX_KEY, WORKFLOW_DEF_INDEX_KEY)\n .value(WORKFLOW_DEF_NAME_VERSION_KEY, bindMarker())\n .value(WORKFLOW_DEF_INDEX_VALUE, bindMarker())\n .getQueryString();\n }\n\n /**\n * @return cql query statement to insert a new task definition into the \"task_definitions\" table\n */\n public String getInsertTaskDefStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_TASK_DEFS)\n .value(TASK_DEFS_KEY, TASK_DEFS_KEY)\n .value(TASK_DEF_NAME_KEY, bindMarker())\n .value(TASK_DEFINITION_KEY, bindMarker())\n .getQueryString();\n }\n\n // Select Statements\n\n /**\n * @return cql query statement to fetch a workflow definition by name and version from the\n * \"workflow_definitions\" table\n */\n public String getSelectWorkflowDefStatement() {\n return QueryBuilder.select(WORKFLOW_DEFINITION_KEY)\n .from(keyspace, TABLE_WORKFLOW_DEFS)\n .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker()))\n .and(eq(WORKFLOW_VERSION_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve all versions of a workflow definition by name from\n * the \"workflow_definitions\" table\n */\n public String getSelectAllWorkflowDefVersionsByNameStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_WORKFLOW_DEFS)\n .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to fetch all workflow def names and version from the\n * \"workflow_defs_index\" table\n */\n public String getSelectAllWorkflowDefsStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX)\n .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker()))\n .getQueryString();\n }\n\n public String getSelectAllWorkflowDefsLatestVersionsStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX)\n .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to fetch a task definition by name from the \"task_definitions\"\n * table\n */\n public String getSelectTaskDefStatement() {\n return QueryBuilder.select(TASK_DEFINITION_KEY)\n .from(keyspace, TABLE_TASK_DEFS)\n .where(eq(TASK_DEFS_KEY, TASK_DEFS_KEY))\n .and(eq(TASK_DEF_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve all task definitions from the \"task_definitions\"\n * table\n */\n public String getSelectAllTaskDefsStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_TASK_DEFS)\n .where(eq(TASK_DEFS_KEY, bindMarker()))\n .getQueryString();\n }\n\n // Update Statement\n\n /**\n * @return cql query statement to update a workflow definitinos in the \"workflow_definitions\"\n * table\n */\n public String getUpdateWorkflowDefStatement() {\n return QueryBuilder.update(keyspace, TABLE_WORKFLOW_DEFS)\n .with(set(WORKFLOW_DEFINITION_KEY, bindMarker()))\n .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker()))\n .and(eq(WORKFLOW_VERSION_KEY, bindMarker()))\n .getQueryString();\n }\n\n // Delete Statements\n\n /**\n * @return cql query statement to delete a workflow definition by name and version from the\n * \"workflow_definitions\" table\n */\n public String getDeleteWorkflowDefStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_WORKFLOW_DEFS)\n .where(eq(WORKFLOW_DEF_NAME_KEY, bindMarker()))\n .and(eq(WORKFLOW_VERSION_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete a workflow def name/version from the\n * \"workflow_defs_index\" table\n */\n public String getDeleteWorkflowDefIndexStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_WORKFLOW_DEFS_INDEX)\n .where(eq(WORKFLOW_DEF_INDEX_KEY, bindMarker()))\n .and(eq(WORKFLOW_DEF_NAME_VERSION_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete a task definition by name from the \"task_definitions\"\n * table\n */\n public String getDeleteTaskDefStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_TASK_DEFS)\n .where(eq(TASK_DEFS_KEY, TASK_DEFS_KEY))\n .and(eq(TASK_DEF_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n\n // ExecutionDAO\n // Insert Statements\n\n /**\n * @return cql query statement to insert a new workflow into the \"workflows\" table\n */\n public String getInsertWorkflowStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOWS)\n .value(WORKFLOW_ID_KEY, bindMarker())\n .value(SHARD_ID_KEY, bindMarker())\n .value(TASK_ID_KEY, bindMarker())\n .value(ENTITY_KEY, ENTITY_TYPE_WORKFLOW)\n .value(PAYLOAD_KEY, bindMarker())\n .value(TOTAL_TASKS_KEY, bindMarker())\n .value(TOTAL_PARTITIONS_KEY, bindMarker())\n .getQueryString();\n }\n\n /**\n * @return cql query statement to insert a new task into the \"workflows\" table\n */\n public String getInsertTaskStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_WORKFLOWS)\n .value(WORKFLOW_ID_KEY, bindMarker())\n .value(SHARD_ID_KEY, bindMarker())\n .value(TASK_ID_KEY, bindMarker())\n .value(ENTITY_KEY, ENTITY_TYPE_TASK)\n .value(PAYLOAD_KEY, bindMarker())\n .getQueryString();\n }\n\n /**\n * @return cql query statement to insert a new event execution into the \"event_executions\" table\n */\n public String getInsertEventExecutionStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_EVENT_EXECUTIONS)\n .value(MESSAGE_ID_KEY, bindMarker())\n .value(EVENT_HANDLER_NAME_KEY, bindMarker())\n .value(EVENT_EXECUTION_ID_KEY, bindMarker())\n .value(PAYLOAD_KEY, bindMarker())\n .ifNotExists()\n .getQueryString();\n }\n\n // Select Statements\n\n /**\n * @return cql query statement to retrieve the total_tasks and total_partitions for a workflow\n * from the \"workflows\" table\n */\n public String getSelectTotalStatement() {\n return QueryBuilder.select(TOTAL_TASKS_KEY, TOTAL_PARTITIONS_KEY)\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, 1))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve a task from the \"workflows\" table\n */\n public String getSelectTaskStatement() {\n return QueryBuilder.select(PAYLOAD_KEY)\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, bindMarker()))\n .and(eq(ENTITY_KEY, ENTITY_TYPE_TASK))\n .and(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve a workflow (without its tasks) from the \"workflows\"\n * table\n */\n public String getSelectWorkflowStatement() {\n return QueryBuilder.select(PAYLOAD_KEY)\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, 1))\n .and(eq(ENTITY_KEY, ENTITY_TYPE_WORKFLOW))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve a workflow with its tasks from the \"workflows\" table\n */\n public String getSelectWorkflowWithTasksStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve the workflow_id for a particular task_id from the\n * \"task_lookup\" table\n */\n public String getSelectTaskFromLookupTableStatement() {\n return QueryBuilder.select(WORKFLOW_ID_KEY)\n .from(keyspace, TABLE_TASK_LOOKUP)\n .where(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve all task ids for a given taskDefName with concurrent\n * execution limit configured from the \"task_def_limit\" table\n */\n public String getSelectTasksFromTaskDefLimitStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_TASK_DEF_LIMIT)\n .where(eq(TASK_DEF_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to retrieve all event executions for a given message and event\n * handler from the \"event_executions\" table\n */\n public String getSelectAllEventExecutionsForMessageFromEventExecutionsStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_EVENT_EXECUTIONS)\n .where(eq(MESSAGE_ID_KEY, bindMarker()))\n .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n\n // Update Statements\n\n /**\n * @return cql query statement to update a workflow in the \"workflows\" table\n */\n public String getUpdateWorkflowStatement() {\n return QueryBuilder.update(keyspace, TABLE_WORKFLOWS)\n .with(set(PAYLOAD_KEY, bindMarker()))\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, 1))\n .and(eq(ENTITY_KEY, ENTITY_TYPE_WORKFLOW))\n .and(eq(TASK_ID_KEY, \"\"))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to update the total_tasks in a shard for a workflow in the\n * \"workflows\" table\n */\n public String getUpdateTotalTasksStatement() {\n return QueryBuilder.update(keyspace, TABLE_WORKFLOWS)\n .with(set(TOTAL_TASKS_KEY, bindMarker()))\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to update the total_partitions for a workflow in the \"workflows\"\n * table\n */\n public String getUpdateTotalPartitionsStatement() {\n return QueryBuilder.update(keyspace, TABLE_WORKFLOWS)\n .with(set(TOTAL_PARTITIONS_KEY, bindMarker()))\n .and(set(TOTAL_TASKS_KEY, bindMarker()))\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, 1))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to add a new task_id to workflow_id mapping to the \"task_lookup\"\n * table\n */\n public String getUpdateTaskLookupStatement() {\n return QueryBuilder.update(keyspace, TABLE_TASK_LOOKUP)\n .with(set(WORKFLOW_ID_KEY, bindMarker()))\n .where(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to add a new task_id to the \"task_def_limit\" table\n */\n public String getUpdateTaskDefLimitStatement() {\n return QueryBuilder.update(keyspace, TABLE_TASK_DEF_LIMIT)\n .with(set(WORKFLOW_ID_KEY, bindMarker()))\n .where(eq(TASK_DEF_NAME_KEY, bindMarker()))\n .and(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to update an event execution in the \"event_executions\" table\n */\n public String getUpdateEventExecutionStatement() {\n return QueryBuilder.update(keyspace, TABLE_EVENT_EXECUTIONS)\n .using(QueryBuilder.ttl(bindMarker()))\n .with(set(PAYLOAD_KEY, bindMarker()))\n .where(eq(MESSAGE_ID_KEY, bindMarker()))\n .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker()))\n .and(eq(EVENT_EXECUTION_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n // Delete statements\n\n /**\n * @return cql query statement to delete a workflow from the \"workflows\" table\n */\n public String getDeleteWorkflowStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete a task_id to workflow_id mapping from the \"task_lookup\"\n * table\n */\n public String getDeleteTaskLookupStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_TASK_LOOKUP)\n .where(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete a task from the \"workflows\" table\n */\n public String getDeleteTaskStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_WORKFLOWS)\n .where(eq(WORKFLOW_ID_KEY, bindMarker()))\n .and(eq(SHARD_ID_KEY, bindMarker()))\n .and(eq(ENTITY_KEY, ENTITY_TYPE_TASK))\n .and(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete a task_id from the \"task_def_limit\" table\n */\n public String getDeleteTaskDefLimitStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_TASK_DEF_LIMIT)\n .where(eq(TASK_DEF_NAME_KEY, bindMarker()))\n .and(eq(TASK_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n /**\n * @return cql query statement to delete an event execution from the \"event_execution\" table\n */\n public String getDeleteEventExecutionsStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_EVENT_EXECUTIONS)\n .where(eq(MESSAGE_ID_KEY, bindMarker()))\n .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker()))\n .and(eq(EVENT_EXECUTION_ID_KEY, bindMarker()))\n .getQueryString();\n }\n\n // EventHandlerDAO\n // Insert Statements\n\n /**\n * @return cql query statement to insert an event handler into the \"event_handlers\" table\n */\n public String getInsertEventHandlerStatement() {\n return QueryBuilder.insertInto(keyspace, TABLE_EVENT_HANDLERS)\n .value(HANDLERS_KEY, HANDLERS_KEY)\n .value(EVENT_HANDLER_NAME_KEY, bindMarker())\n .value(EVENT_HANDLER_KEY, bindMarker())\n .getQueryString();\n }\n\n // Select Statements\n\n /**\n * @return cql query statement to retrieve all event handlers from the \"event_handlers\" table\n */\n public String getSelectAllEventHandlersStatement() {\n return QueryBuilder.select()\n .all()\n .from(keyspace, TABLE_EVENT_HANDLERS)\n .where(eq(HANDLERS_KEY, bindMarker()))\n .getQueryString();\n }\n\n // Delete Statements\n\n /**\n * @return cql query statement to delete an event handler by name from the \"event_handlers\"\n * table\n */\n public String getDeleteEventHandlerStatement() {\n return QueryBuilder.delete()\n .from(keyspace, TABLE_EVENT_HANDLERS)\n .where(eq(HANDLERS_KEY, HANDLERS_KEY))\n .and(eq(EVENT_HANDLER_NAME_KEY, bindMarker()))\n .getQueryString();\n }\n}\n"
},
{
"path": "cassandra-persistence/src/main/resources/META-INF/additional-spring-configuration-metadata.json",
"content": "{\n \"properties\": [\n {\n \"name\": \"conductor.cassandra.write-consistency-level\",\n \"defaultValue\": \"LOCAL_QUORUM\"\n },\n {\n \"name\": \"conductor.cassandra.read-consistency-level\",\n \"defaultValue\": \"LOCAL_QUORUM\"\n }\n ],\n \"hints\": [\n {\n \"name\": \"conductor.cassandra.write-consistency-level\",\n \"providers\": [\n {\n \"name\": \"handle-as\",\n \"parameters\": {\n \"target\": \"java.lang.Enum\"\n }\n }\n ]\n },\n {\n \"name\": \"conductor.cassandra.read-consistency-level\",\n \"providers\": [\n {\n \"name\": \"handle-as\",\n \"parameters\": {\n \"target\": \"java.lang.Enum\"\n }\n }\n ]\n }\n ]\n}\n"
},
{
"path": "cassandra-persistence/src/test/groovy/com/netflix/conductor/cassandra/dao/CassandraEventHandlerDAOSpec.groovy",
"content": "/*\n * Copyright 2022 Netflix, Inc.\n *