podNames) {\n for (int i = 0; i < podNames.size() + 1; i++) {\n String podName = podName(resourceName, i);\n\n if (!podNames.contains(podName)) {\n return podName;\n }\n }\n throw new RuntimeException(\"Can't generate pod name for '\" + resourceName + \"', current pods: \" + podNames);\n }\n\n private static String configMapName(String resourceName) {\n return resourceName + \"-cfg\";\n }\n\n private static String podName(String resourceName, int i) {\n return String.format(\"%s-%05d\", resourceName, i);\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/planner/TagForRemovalChange.java",
"content": "package com.walmartlabs.concord.agentoperator.planner;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.agent.AgentClient;\nimport com.walmartlabs.concord.agentoperator.PodLabels;\nimport com.walmartlabs.concord.agentoperator.resources.AgentPod;\nimport io.fabric8.kubernetes.client.KubernetesClient;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class TagForRemovalChange implements Change {\n\n private static final Logger log = LoggerFactory.getLogger(TagForRemovalChange.class);\n\n private final String podName;\n private final AgentClient agentClient;\n\n public TagForRemovalChange(String podName, AgentClient agentClient) {\n this.podName = podName;\n this.agentClient = agentClient;\n }\n\n @Override\n public void apply(KubernetesClient client) {\n try {\n agentClient.enableMaintenanceMode();\n } catch (Exception e) {\n log.error(\"Error enabling maintenance mode for pod '{}'\", podName, e);\n return;\n }\n\n PodLabels.applyTag(client, podName, AgentPod.TAGGED_FOR_REMOVAL_LABEL, \"true\");\n }\n\n @Override\n public String toString() {\n return \"TagForRemovalChange{\" +\n \"podName='\" + podName + '\\'' +\n '}';\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/planner/TryToDeletePodChange.java",
"content": "package com.walmartlabs.concord.agentoperator.planner;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.agent.AgentClient;\nimport com.walmartlabs.concord.agentoperator.PodLabels;\nimport com.walmartlabs.concord.agentoperator.resources.AgentPod;\nimport io.fabric8.kubernetes.api.model.Pod;\nimport io.fabric8.kubernetes.client.KubernetesClient;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.util.Map;\n\npublic class TryToDeletePodChange implements Change {\n\n private static final Logger log = LoggerFactory.getLogger(TryToDeletePodChange.class);\n\n private final String podName;\n private final AgentClient agentClient;\n\n public TryToDeletePodChange(String podName, AgentClient agentClient) {\n this.podName = podName;\n this.agentClient = agentClient;\n }\n\n /**\n * When the agent pod is being deleted, Kubernetes calls the prestop hook configured.\n * The prestop hook script configured for agent container enables maintenance mode on the agent,\n * and waits for the number of workers in use to go to 0, before the pod gets terminated.\n * \n * Whenever the scheduler calls this `apply` method, the following conditions are checked, and\n * corresponding actions are executed.\n *
\n * - If the pod has a label `preStopHookTermination: true`, do nothing and exit, as the pod is being\n * terminated, waiting for the prestop hook script to complete (that is, last running process on the agent\n * container to complete).\n *
\n * - Otherwise if the pod is in `RUNNING` phase, call the kubernetes client `delete` method on the agent pod,\n * which will put the pod in `Terminating` state and start executing the prestop hook script on the\n * agent container. Add the label `preStopHookTermination: true` (this will be checked on\n * subsequent executions).\n *\n * @param client instance of Kubernetes client\n */\n @Override\n public void apply(KubernetesClient client) {\n Pod pod = client.pods().withName(podName).get();\n\n if (pod == null) {\n log.warn(\"apply ['{}'] -> pod doesn't exist, nothing to do\", podName);\n return;\n }\n\n Map labels = pod.getMetadata().getLabels();\n\n if (\"true\".equals(labels.getOrDefault(AgentPod.PRE_STOP_HOOK_TERMINATION_LABEL, \"false\"))) {\n log.debug(\"['{}'] -> has already been marked for termination\", podName);\n return;\n }\n\n try {\n if (agentClient.hasBusyWorkers()) {\n return;\n }\n } catch (Exception e) {\n log.error(\"Error while checking agent workers count for pod '{}'\", podName, e);\n return;\n }\n\n // agent pod in maintenance mode and all workers done\n client.pods().withName(podName).delete();\n PodLabels.applyTag(client, podName, AgentPod.PRE_STOP_HOOK_TERMINATION_LABEL, \"true\");\n log.info(\"apply ['{}'] -> Marked for termination (former phase: {})\", podName, pod.getStatus().getPhase());\n }\n\n @Override\n public String toString() {\n return \"TryToDeletePodChange{\" +\n \"podName='\" + podName + '\\'' +\n '}';\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueClient.java",
"content": "package com.walmartlabs.concord.agentoperator.processqueue;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2024 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.fasterxml.jackson.core.type.TypeReference;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.escape.Escaper;\nimport com.google.common.net.UrlEscapers;\nimport com.walmartlabs.concord.agentoperator.scheduler.QueueSelector;\nimport com.walmartlabs.concord.sdk.Constants;\n\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.TrustManager;\nimport javax.net.ssl.X509TrustManager;\nimport java.io.IOException;\nimport java.net.CookieManager;\nimport java.net.CookiePolicy;\nimport java.net.URI;\nimport java.net.http.HttpClient;\nimport java.net.http.HttpRequest;\nimport java.net.http.HttpResponse;\nimport java.security.KeyManagementException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.SecureRandom;\nimport java.security.cert.X509Certificate;\nimport java.util.List;\n\npublic class ProcessQueueClient {\n\n private static final TypeReference> LIST_OF_PROCESS_QUEUE_ENTRIES = new TypeReference<>() {\n };\n\n private final String baseUrl;\n private final String apiToken;\n private final ObjectMapper objectMapper;\n private final HttpClient client;\n\n public ProcessQueueClient(String baseUrl, String apiToken) {\n this.baseUrl = baseUrl;\n this.apiToken = apiToken;\n this.objectMapper = new ObjectMapper();\n this.client = initClient();\n }\n\n public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {\n StringBuilder queryUrl = new StringBuilder(baseUrl + \"/api/v2/process/requirements?status=\" + processStatus + \"&limit=\" + limit + \"&startAt.len=\");\n String flavor = queueSelector.getFlavor();\n if (flavor != null) {\n queryUrl.append(\"&requirements.agent.flavor.eq=\").append(flavor);\n }\n List queryParams = queueSelector.getQueryParams();\n if (queryParams != null) {\n for (String queryParam : queryParams) {\n queryUrl.append(\"&\").append(escapeQueryParam(queryParam));\n }\n }\n\n HttpRequest request = HttpRequest.newBuilder()\n .uri(URI.create(queryUrl.toString()))\n .header(\"Authorization\", apiToken)\n .header(\"User-Agent\", \"k8s-agent-operator\")\n .header(Constants.Headers.ENABLE_HTTP_SESSION, \"true\")\n .GET()\n .build();\n\n HttpResponse response;\n try {\n response = client.send(request, HttpResponse.BodyHandlers.ofString());\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n throw new IOException(\"Interrupted while fetching the queue data\", e);\n }\n\n if (response.statusCode() != 200) {\n throw new IOException(\"Error while fetching the process queue data: \" + response.statusCode());\n }\n\n return objectMapper.readValue(response.body(), LIST_OF_PROCESS_QUEUE_ENTRIES);\n }\n\n private static HttpClient initClient() {\n try {\n TrustManager[] trustAllCerts = new TrustManager[]{\n new X509TrustManager() {\n public X509Certificate[] getAcceptedIssuers() {\n return new X509Certificate[0];\n }\n\n public void checkClientTrusted(X509Certificate[] certs, String authType) {\n }\n\n public void checkServerTrusted(X509Certificate[] certs, String authType) {\n }\n }\n };\n\n SSLContext sslContext = SSLContext.getInstance(\"SSL\");\n sslContext.init(null, trustAllCerts, new SecureRandom());\n\n CookieManager cookieManager = new CookieManager();\n cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL);\n\n return HttpClient.newBuilder()\n .sslContext(sslContext)\n .cookieHandler(cookieManager)\n .build();\n } catch (NoSuchAlgorithmException | KeyManagementException e) {\n throw new RuntimeException(\"Error while initializing the HTTP client\", e);\n }\n }\n\n @VisibleForTesting\n static String escapeQueryParam(String s) {\n Escaper escaper = UrlEscapers.urlPathSegmentEscaper();\n int i = s.indexOf(\"=\");\n if (i < 0) {\n return escaper.escape(s);\n }\n String key = s.substring(0, i);\n String value = s.substring(i + 1);\n return escaper.escape(key) + \"=\" + escaper.escape(value);\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueEntry.java",
"content": "package com.walmartlabs.concord.agentoperator.processqueue;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.fasterxml.jackson.annotation.JsonCreator;\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\n\nimport java.io.Serializable;\nimport java.util.Map;\n\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic class ProcessQueueEntry implements Serializable {\n\n private static final long serialVersionUID = 1L;\n\n private final Map requirements;\n\n @JsonCreator\n public ProcessQueueEntry(@JsonProperty(\"requirements\") Map requirements) {\n this.requirements = requirements;\n }\n\n public Map getRequirements() {\n return requirements;\n }\n\n @Override\n public String toString() {\n return \"ProcessQueueEntry{\" +\n \"requirements=\" + requirements +\n '}';\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/AgentConfigMap.java",
"content": "package com.walmartlabs.concord.agentoperator.resources;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.walmartlabs.concord.agentoperator.HashUtils;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.scheduler.AgentPoolInstance;\nimport io.fabric8.kubernetes.api.model.ConfigMap;\nimport io.fabric8.kubernetes.client.KubernetesClient;\nimport io.fabric8.kubernetes.client.KubernetesClientException;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\n\npublic final class AgentConfigMap {\n\n private static final Logger log = LoggerFactory.getLogger(AgentConfigMap.class);\n\n private static final ObjectMapper objectMapper = new ObjectMapper();\n\n public static ConfigMap get(KubernetesClient client, String configMapName) {\n try {\n return client.configMaps().withName(configMapName).get();\n } catch (KubernetesClientException e) {\n log.warn(\"get ['{}'] -> error while getting a configmap: {}\", configMapName, e.getMessage());\n throw e;\n }\n }\n\n public static void create(KubernetesClient client, AgentPoolInstance poolInstance, String configMapName) throws IOException {\n try {\n ConfigMap m = prepare(client, poolInstance, configMapName);\n client.configMaps().resource(m).create();\n } catch (KubernetesClientException e) {\n log.warn(\"create ['{}', '{}'] -> error while creating a configmap: {}\", poolInstance.getName(), configMapName, e.getMessage());\n throw e;\n }\n }\n\n public static void delete(KubernetesClient client, String configMapName) {\n try {\n client.configMaps().withName(configMapName).delete();\n\n // wait till it's actually removed\n while (client.configMaps().withName(configMapName).get() != null) {\n try {\n Thread.sleep(1000);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n return;\n }\n }\n } catch (KubernetesClientException e) {\n log.warn(\"delete ['{}'] -> error while deleting a configmap: {}\", configMapName, e.getMessage());\n throw e;\n }\n }\n\n public static boolean hasChanges(KubernetesClient client, AgentPoolInstance poolInstance, ConfigMap a) throws IOException {\n ConfigMap b = prepare(client, poolInstance, a.getMetadata().getName());\n\n String hashA = HashUtils.hashAsHexString(a.getData());\n String hashB = HashUtils.hashAsHexString(b.getData());\n\n return !hashA.equals(hashB);\n }\n\n private static ConfigMap prepare(KubernetesClient client, AgentPoolInstance poolInstance, String configMapName) throws IOException {\n try {\n AgentPoolConfiguration spec = poolInstance.getResource().getSpec();\n\n String configMapYaml = objectMapper.writeValueAsString(spec.getConfigMap())\n .replaceAll(\"%%configMapName%%\", configMapName)\n .replace(\"%%preStopHook%%\", escape(Resources.get(\"/prestop-hook.sh\")));\n\n return client.configMaps().load(new ByteArrayInputStream(configMapYaml.getBytes())).item();\n } catch (KubernetesClientException e) {\n log.warn(\"prepare ['{}', '{}'] -> error while preparing a configmap: {}\", poolInstance.getName(), configMapName, e.getMessage());\n throw e;\n }\n }\n\n private static String escape(String str) throws JsonProcessingException {\n String result = objectMapper.writeValueAsString(str);\n return result.substring(1, result.length() - 1);\n }\n\n private AgentConfigMap() {\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/AgentPod.java",
"content": "package com.walmartlabs.concord.agentoperator.resources;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.scheduler.AgentPoolInstance;\nimport io.fabric8.kubernetes.api.model.Pod;\nimport io.fabric8.kubernetes.client.KubernetesClient;\nimport io.fabric8.kubernetes.client.KubernetesClientException;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.util.List;\n\npublic final class AgentPod {\n\n private static final Logger log = LoggerFactory.getLogger(AgentPod.class);\n\n public static final String TAGGED_FOR_REMOVAL_LABEL = \"concordTaggedForRemoval\";\n public static final String PRE_STOP_HOOK_TERMINATION_LABEL = \"preStopHookTermination\";\n public static final String POOL_NAME_LABEL = \"poolName\";\n public static final String CONFIG_HASH_LABEL = \"concordCfgHash\";\n\n private static final ObjectMapper objectMapper = new ObjectMapper();\n\n public static List listMarkedForRemoval(KubernetesClient client, String resourceName) {\n try {\n return client.pods()\n .withLabel(AgentPod.TAGGED_FOR_REMOVAL_LABEL)\n .withLabel(AgentPod.POOL_NAME_LABEL, resourceName)\n .list()\n .getItems();\n } catch (KubernetesClientException e) {\n log.warn(\"listMarkedForRemoval ['{}'] -> error while listing marked for removal pods: {}\", resourceName, e.getMessage());\n throw e;\n }\n }\n\n public static List list(KubernetesClient client, String resourceName) {\n try {\n return client.pods()\n .withLabel(POOL_NAME_LABEL, resourceName)\n .list()\n .getItems();\n } catch (KubernetesClientException e) {\n log.warn(\"list ['{}'] -> error while listing pool pods: {}\", resourceName, e.getMessage());\n throw e;\n }\n }\n\n public static void create(KubernetesClient client,\n AgentPoolInstance poolInstance,\n String podName,\n String configMapName,\n String hash) throws IOException {\n\n try {\n AgentPoolConfiguration spec = poolInstance.getResource().getSpec();\n\n String podYaml = objectMapper.writeValueAsString(spec.getPod())\n .replaceAll(\"%%podName%%\", podName)\n .replaceAll(\"%%app%%\", AgentPool.SERVICE_FULL_NAME)\n .replaceAll(\"%%\" + POOL_NAME_LABEL + \"%%\", poolInstance.getName())\n .replaceAll(\"%%configMapName%%\", configMapName)\n .replaceAll(\"%%\" + CONFIG_HASH_LABEL + \"%%\", hash);\n\n client.pods().load(new ByteArrayInputStream(podYaml.getBytes())).create();\n } catch (KubernetesClientException e) {\n log.warn(\"create ['{}', '{}', '{}', '{}'] -> error while creating a pod: {}\", poolInstance.getName(), podName, configMapName, hash, e.getMessage());\n throw e;\n }\n }\n\n private AgentPod() {\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/Resources.java",
"content": "package com.walmartlabs.concord.agentoperator.resources;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2020 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.google.common.base.Charsets;\nimport com.google.common.io.CharStreams;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic final class Resources {\n\n private static final Map resources = new ConcurrentHashMap<>();\n\n public static String get(String name) {\n return resources.computeIfAbsent(name, Resources::load);\n }\n\n private static String load(String name) {\n try (InputStream in = Resources.class.getResourceAsStream(name)) {\n if (in == null) {\n throw new RuntimeException(\"Resource '\" + name + \"' not found\");\n }\n return CharStreams.toString(new InputStreamReader(in, Charsets.UTF_8));\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n }\n\n private Resources() {\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AgentPoolInstance.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\n\npublic class AgentPoolInstance {\n\n public static AgentPoolInstance updateStatus(AgentPoolInstance i, Status status) {\n return new AgentPoolInstance(i.name, i.resource, status, i.targetSize, System.currentTimeMillis(), i.getLastScaleUpTimestamp(), i.getLastScaleDownTimeStamp());\n }\n\n public static AgentPoolInstance updateTargetSize(AgentPoolInstance i, int targetSize, long scaleUptimeStamp, long scaleDownTimeStamp) {\n return new AgentPoolInstance(i.name, i.resource, i.status, targetSize, System.currentTimeMillis(), scaleUptimeStamp, scaleDownTimeStamp);\n }\n\n private final String name;\n private final AgentPool resource;\n private final Status status;\n private final int targetSize;\n private final long lastUpdateTimestamp;\n private final long lastScaleUpTimestamp;\n private final long lastScaleDownTimeStamp;\n\n public AgentPoolInstance(String name, AgentPool resource, Status status, int targetSize, long lastUpdateTimestamp,\n long lastScaleUpTimestamp, long lastScaleDownTimeStamp) {\n this.name = name;\n this.resource = resource;\n this.status = status;\n this.targetSize = targetSize;\n this.lastUpdateTimestamp = lastUpdateTimestamp;\n this.lastScaleUpTimestamp = lastScaleUpTimestamp;\n this.lastScaleDownTimeStamp = lastScaleDownTimeStamp;\n }\n\n public String getName() {\n return name;\n }\n\n public AgentPool getResource() {\n return resource;\n }\n\n public Status getStatus() {\n return status;\n }\n\n public int getTargetSize() {\n return targetSize;\n }\n\n public long getLastUpdateTimestamp() {\n return lastUpdateTimestamp;\n }\n\n public long getLastScaleUpTimestamp() {\n return lastScaleUpTimestamp;\n }\n\n public long getLastScaleDownTimeStamp() {\n return lastScaleDownTimeStamp;\n }\n\n public enum Status {\n ACTIVE,\n DELETED\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AutoScaler.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2023 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport java.io.IOException;\n\npublic interface AutoScaler {\n\n AgentPoolInstance apply(AgentPoolInstance i) throws IOException;\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AutoScalerFactory.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2023 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;\nimport com.walmartlabs.concord.agentoperator.resources.AgentPod;\nimport io.fabric8.kubernetes.client.KubernetesClient;\n\nimport java.util.function.Function;\n\npublic class AutoScalerFactory {\n\n private final Function podCounter;\n private final ProcessQueueClient processQueueClient;\n\n public AutoScalerFactory(String concordBaseUrl, String concordApiToken, KubernetesClient k8sClient) {\n this.podCounter = n -> AgentPod.list(k8sClient, n).size();\n this.processQueueClient = new ProcessQueueClient(concordBaseUrl, concordApiToken);\n }\n\n public AutoScaler create(AgentPoolInstance poolInstance) {\n AgentPoolConfiguration cfg = poolInstance.getResource().getSpec();\n if (LinearAutoScaler.NAME.equals(cfg.getAutoScaleStrategy())) {\n return new LinearAutoScaler(processQueueClient, podCounter);\n }\n\n return new DefaultAutoScaler(processQueueClient, podCounter);\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/DefaultAutoScaler.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;\nimport com.walmartlabs.concord.common.Matcher;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Function;\n\npublic class DefaultAutoScaler implements AutoScaler {\n\n public static final String NAME = \"default\";\n\n private static final Logger log = LoggerFactory.getLogger(DefaultAutoScaler.class);\n\n private final ProcessQueueClient processQueueClient;\n private final Function podCounter;\n private final Function canBeScaledUp;\n private final Function canBeScaledDown;\n private long scaleUpTimeStamp;\n private long scaleDownTimeStamp;\n\n public DefaultAutoScaler(ProcessQueueClient processQueueClient, Function podCounter) {\n this(processQueueClient, podCounter, i -> {\n long t = System.currentTimeMillis();\n return t - i.getLastScaleUpTimestamp() > i.getResource().getSpec().getScaleUpDelayMs();\n }, i -> {\n long t = System.currentTimeMillis();\n return t - i.getLastScaleDownTimeStamp() > i.getResource().getSpec().getScaleDownDelayMs();\n });\n }\n\n public DefaultAutoScaler(ProcessQueueClient processQueueClient,\n Function podCounter, Function canBeScaledUp,\n Function canBeScaledDown) {\n this.processQueueClient = processQueueClient;\n this.podCounter = podCounter;\n this.canBeScaledUp = canBeScaledUp;\n this.canBeScaledDown = canBeScaledDown;\n this.scaleUpTimeStamp = System.currentTimeMillis();\n this.scaleDownTimeStamp = System.currentTimeMillis();\n }\n\n /**\n * Scale up or Scale down the number of agent pods depending on various conditions\n * \n * If enqueued process count is greater than the threshold defined for incrementing to max\n * pool size, increase the pool size to the maximum size.\n * Otherwise, if the enqueued process count is greater than the threshold defined for incrementing\n * the pool size, increase the pool size by the increment percentage defined\n *
\n * Decrease the pool size by the decrement percentage defined only if,\n * - enqueued process count is lesser than the minimum pool size threshold defined\n * - running process count is lesser than the threshold defined (which depends on current size of the pool\n * running the processes, and a constant factor specified - default to 1. Simplified to\n * runningCount < podsCount)\n *\n * @param i Agent pool on which the scaling activity is to be performed\n */\n public AgentPoolInstance apply(AgentPoolInstance i) throws IOException {\n int queueQueryLimit = i.getResource().getSpec().getQueueQueryLimit();\n QueueSelector queueSelector = QueueSelector.parse(i.getResource().getSpec().getQueueSelector());\n List queueEntries = processQueueClient.query(\"ENQUEUED\", queueQueryLimit, queueSelector);\n\n scaleUpTimeStamp = i.getLastScaleUpTimestamp();\n scaleDownTimeStamp = i.getLastScaleDownTimeStamp();\n\n AgentPoolConfiguration cfg = i.getResource().getSpec();\n\n boolean canBeScaled = canBeScaledUp.apply(i) || canBeScaledDown.apply(i);\n if (!canBeScaled) {\n // was updated recently, skipping\n return i;\n }\n\n // count the currently running pods\n int podsCount = podCounter.apply(i.getName());\n log.info(\"['{}']: Current pool size: {}\", i.getName(), podsCount);\n\n // the number of processes waiting for an agent in the current pool\n int enqueuedCount = getProcessCount(cfg, queueEntries);\n log.info(\"['{}']: Enqueued process count: {}\", i.getName(), enqueuedCount);\n\n if (podsCount < cfg.getMinSize()) {\n return AgentPoolInstance.updateTargetSize(i, cfg.getMinSize(), System.currentTimeMillis(), System.currentTimeMillis());\n }\n\n // The threshold above which the operator can scale up the agent pods to the defined maximum pool size\n double maxPoolSizeThreshold = cfg.getMaxSize() * cfg.getIncrementThresholdFactor();\n\n // The threshold above which the pool size can be increased by the increment percentage defined\n double incrementThreshold = cfg.getIncrementThresholdFactor() * podsCount;\n\n // The threshold, combined with threshold for running processes determine if the pool size can be\n // reduced by the decrement percentage defined\n double minPoolSizeThreshold = cfg.getDecrementThresholdFactor() * cfg.getMinSize();\n\n // Initial target size of the agent pool before updation\n int targetSize = i.getTargetSize();\n\n // Try scaling up if the time elapsed after last scale up operation\n // is greater than the scale up delay defined (default: 15s)\n if (canBeScaledUp.apply(i)) {\n targetSize = tryScaleUp(cfg, i, podsCount, enqueuedCount, targetSize, maxPoolSizeThreshold, incrementThreshold);\n\n // Reset scaledown delay counter if enqueued count is greater than min threshold.\n // Scale down should happen only if enqueued count is less than\n // min threshold consistently for scaledown delay defined (default: 180s)\n if (enqueuedCount >= minPoolSizeThreshold) {\n log.info(\"['{}']: Resetting scale down delay counter - (enqueued count({}) >= minimum threshold({}))...\",\n i.getName(), enqueuedCount, minPoolSizeThreshold);\n scaleDownTimeStamp = System.currentTimeMillis();\n }\n }\n\n // Try scaling down if the time elapsed after last scale down operation\n // is greater than the scale down delay defined (default: 180s)\n if (canBeScaledDown.apply(i)) {\n targetSize = tryScaleDown(cfg, i, podsCount, enqueuedCount, targetSize, minPoolSizeThreshold);\n }\n\n if (targetSize == i.getTargetSize()) {\n log.info(\"['{}']: Not changing the pool size.\", i.getName());\n } else {\n log.info(\"apply ['{}'] -> updated to {}\", i.getName(), targetSize);\n }\n return AgentPoolInstance.updateTargetSize(i, targetSize, scaleUpTimeStamp, scaleDownTimeStamp);\n }\n\n private int tryScaleUp(AgentPoolConfiguration cfg, AgentPoolInstance i, int podsCount, int enqueuedCount, int poolSize,\n double maxPoolSizeThreshold, double incrementThreshold) {\n\n // To prevent scale up before previous scale down action is completed\n podsCount = Math.min(podsCount, i.getTargetSize());\n\n // Reset scaleup delay counter for every attempt to scale up\n scaleUpTimeStamp = System.currentTimeMillis();\n\n if (podsCount < cfg.getMaxSize()) {\n if (enqueuedCount >= maxPoolSizeThreshold) {\n poolSize = cfg.getMaxSize();\n log.info(\"['{}']: Incrementing to max size - {}\", i.getName(), poolSize);\n } else if (enqueuedCount >= incrementThreshold) {\n poolSize = (int) Math.round(podsCount * (1 + cfg.getPercentIncrement() / 100));\n\n // Limit to maximum pool size if the computed target size is more than the max size\n if (poolSize > cfg.getMaxSize()) {\n log.warn(\"['{}']: Target pool size exceeds the allowed maximum: {} > {}. Updating to maximum size - {}\",\n i.getName(), poolSize, cfg.getMaxSize(), cfg.getMaxSize());\n poolSize = cfg.getMaxSize();\n }\n\n log.info(\"['{}']: Scaling up to {}...\", i.getName(), poolSize);\n }\n } else {\n log.warn(\"['{}']: Target pool size already the allowed maximum size: {}. Not updating.\",\n i.getName(), cfg.getMaxSize());\n }\n\n return poolSize;\n }\n\n private int tryScaleDown(AgentPoolConfiguration cfg, AgentPoolInstance i, int podsCount, int enqueuedCount,\n int poolSize, double minPoolSizeThreshold) {\n\n // To prevent scale down before previous scale up action is completed\n podsCount = Math.max(podsCount, i.getTargetSize());\n\n // Reset scaledown delay counter for every attempt to scale down\n scaleDownTimeStamp = System.currentTimeMillis();\n\n if (podsCount > cfg.getMinSize()) {\n if (enqueuedCount < minPoolSizeThreshold) {\n poolSize = (int) Math.floor(podsCount * (1 - cfg.getPercentDecrement() / 100));\n\n log.info(\"['{}']: Scaling down - (enqueued count({}) < minimum threshold({})) for more than {} seconds...\",\n i.getName(), enqueuedCount, minPoolSizeThreshold,\n (i.getResource().getSpec().getScaleDownDelayMs() / 1000));\n\n // Limit to minimum pool size if the computed target size is less than the min size\n if (poolSize < cfg.getMinSize()) {\n log.warn(\"['{}']: Target pool size lesser than the allowed minimum: {} < {}. Updating to minimum size - {}\",\n i.getName(), poolSize, cfg.getMinSize(), cfg.getMinSize());\n poolSize = cfg.getMinSize();\n } else {\n log.info(\"['{}']: Scaling down to {}...\", i.getName(), poolSize);\n }\n }\n } else {\n log.warn(\"['{}']: Target pool size already the allowed minimum size: {}. Not updating.\",\n i.getName(), cfg.getMinSize());\n }\n\n return poolSize;\n }\n\n private static int getProcessCount(AgentPoolConfiguration cfg, List processQueueEntries) {\n return (int) processQueueEntries.stream()\n .map(ProcessQueueEntry::getRequirements)\n .filter(a -> isEmpty(cfg.getQueueSelector()) || Matcher.matches(a, cfg.getQueueSelector()))\n .count();\n }\n\n private static boolean isEmpty(Map m) {\n return m == null || m.isEmpty();\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/Event.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\n\npublic class Event {\n\n private final Type type;\n private final AgentPool resource;\n\n public Event(Type type, AgentPool resource) {\n this.type = type;\n this.resource = resource;\n }\n\n public Type getType() {\n return type;\n }\n\n public AgentPool getResource() {\n return resource;\n }\n\n public enum Type {\n\n MODIFIED,\n DELETED\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/LinearAutoScaler.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2023 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.util.List;\nimport java.util.function.Function;\n\npublic class LinearAutoScaler implements AutoScaler {\n\n private static final Logger log = LoggerFactory.getLogger(LinearAutoScaler.class);\n\n public static final String NAME = \"linear\";\n\n private final ProcessQueueClient processQueueClient;\n private final Function podCounter;\n private final Function canBeScaledUp;\n private final Function canBeScaledDown;\n\n public LinearAutoScaler(ProcessQueueClient processQueueClient, Function podCounter) {\n this(processQueueClient, podCounter, i -> {\n long t = System.currentTimeMillis();\n return t - i.getLastScaleUpTimestamp() > i.getResource().getSpec().getScaleUpDelayMs();\n }, i -> {\n long t = System.currentTimeMillis();\n return t - i.getLastScaleDownTimeStamp() > i.getResource().getSpec().getScaleDownDelayMs();\n });\n }\n\n public LinearAutoScaler(ProcessQueueClient processQueueClient,\n Function podCounter, Function canBeScaledUp,\n Function canBeScaledDown) {\n this.processQueueClient = processQueueClient;\n this.podCounter = podCounter;\n this.canBeScaledUp = canBeScaledUp;\n this.canBeScaledDown = canBeScaledDown;\n }\n\n @Override\n public AgentPoolInstance apply(AgentPoolInstance i) throws IOException {\n if (!canBeScaledUp.apply(i) && !canBeScaledDown.apply(i)) {\n log.info(\"apply [{}] -> not a time. up: {}, down: {}, delay up: {}, delay down: {}\", i.getName(), (System.currentTimeMillis() - i.getLastScaleUpTimestamp()), (System.currentTimeMillis() - i.getLastScaleDownTimeStamp()), i.getResource().getSpec().getScaleUpDelayMs(), i.getResource().getSpec().getScaleDownDelayMs());\n // was updated recently, skipping\n return i;\n }\n\n long scaleUpTimeStamp = i.getLastScaleUpTimestamp();\n long scaleDownTimeStamp = i.getLastScaleDownTimeStamp();\n\n AgentPoolConfiguration cfg = i.getResource().getSpec();\n\n QueueSelector queueSelector = QueueSelector.parse(cfg.getQueueSelector());\n List queueEntries = processQueueClient.query(\"ENQUEUED\", cfg.getMaxSize(), queueSelector);\n\n // count the currently running pods\n int podsCount = podCounter.apply(i.getName());\n\n // the number of processes waiting for an agent in the current pool\n int enqueuedCount = queueEntries.size();\n int runningCount = processQueueClient.query(\"RUNNING\", cfg.getMaxSize(), queueSelector).size();\n int freePodsCount = Math.max(podsCount - runningCount, 0);\n\n int increment = 0;\n if (enqueuedCount > freePodsCount) {\n increment = cfg.getSizeIncrement();\n scaleUpTimeStamp = System.currentTimeMillis();\n scaleDownTimeStamp = System.currentTimeMillis();\n } else if (enqueuedCount < freePodsCount) {\n increment = -cfg.getSizeIncrement();\n scaleDownTimeStamp = System.currentTimeMillis();\n }\n\n int targetSize = Math.max(cfg.getMinSize(), podsCount + increment);\n if (i.getTargetSize() == targetSize) {\n log.info(\"apply ['{}'] -> targetSize = {}, enqueuedCount = {}, increment = {}, podsCount = {}\", i.getName(), targetSize, enqueuedCount, increment, podsCount);\n // no changes needed\n return i;\n }\n\n if (increment > 0 && !canBeScaledUp.apply(i)) {\n log.info(\"apply ['{}'] -> not a time to scale up to {}\", i.getName(), targetSize);\n return i;\n }\n if (increment < 0 && !canBeScaledDown.apply(i)) {\n log.info(\"apply ['{}'] -> not a time to scale down to {}\", i.getName(), targetSize);\n return i;\n }\n\n if (targetSize > cfg.getMaxSize()) {\n log.warn(\"apply ['{}'] -> target pool size exceeds the allowed maximum: {} > {}\", i.getName(), enqueuedCount, cfg.getMaxSize());\n }\n\n targetSize = Math.min(targetSize, cfg.getMaxSize());\n log.info(\"apply ['{}'] -> updated to {}, pods: {}, free: {}, enqueued: {}, running: {}\", i.getName(), targetSize, podsCount, freePodsCount, enqueuedCount, runningCount);\n return AgentPoolInstance.updateTargetSize(i, targetSize, scaleUpTimeStamp, scaleDownTimeStamp);\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/QueueSelector.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2024 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.common.ConfigurationUtils;\n\nimport java.util.List;\nimport java.util.Map;\n\npublic class QueueSelector {\n\n public static QueueSelector parse(Map queueSelector) {\n String flavor;\n Object maybeFlavor = ConfigurationUtils.get(queueSelector, \"agent\", \"flavor\");\n if (maybeFlavor != null && !(maybeFlavor instanceof String)) {\n throw new IllegalArgumentException(\"Expected a string value as 'agent.flavor', got: \" + maybeFlavor);\n }\n flavor = (String) maybeFlavor;\n\n List queryParams;\n Object maybeQueryParams = ConfigurationUtils.get(queueSelector, \"queryParams\");\n if (maybeQueryParams != null) {\n if (!(maybeQueryParams instanceof List)) {\n throw new IllegalArgumentException(\"Expected a list value as 'queryParams', got: \" + maybeQueryParams);\n }\n\n ((List) maybeQueryParams).forEach(qp -> {\n if (!(qp instanceof String)) {\n throw new IllegalArgumentException(\"Expected a string value as 'queryParams' item, got: \" + qp);\n }\n });\n }\n //noinspection unchecked\n queryParams = (List) maybeQueryParams;\n return new QueueSelector(flavor, queryParams);\n }\n\n private final String flavor;\n private final List queryParams;\n\n private QueueSelector(String flavor, List queryParams) {\n this.flavor = flavor;\n this.queryParams = queryParams;\n }\n\n /**\n * \"Flavor\" of the current agent. Translates to the \"requirements.agent.flavor.eq\"\n * query parameter when fetching the process queue.\n */\n public String getFlavor() {\n return flavor;\n }\n\n /**\n * Additional query parameters to be used when fetching the process queue. Appended\n * as-is to the query URL.\n */\n public List getQueryParams() {\n return queryParams;\n }\n}\n"
},
{
"path": "agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/Scheduler.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.agent.AgentClientFactory;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\nimport com.walmartlabs.concord.agentoperator.planner.Change;\nimport com.walmartlabs.concord.agentoperator.planner.Planner;\nimport com.walmartlabs.concord.agentoperator.resources.AgentPod;\nimport io.fabric8.kubernetes.api.model.Pod;\nimport io.fabric8.kubernetes.client.KubernetesClient;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.util.*;\n\npublic class Scheduler {\n\n private static final Logger log = LoggerFactory.getLogger(Scheduler.class);\n\n private static final long POLL_DELAY = 10000;\n private static final long ERROR_DELAY = 30000;\n\n private final AutoScalerFactory autoScalerFactory;\n private final KubernetesClient k8sClient;\n private final Planner planner;\n private final Map pools;\n private final List events;\n\n public Scheduler(AutoScalerFactory autoScalerFactory, KubernetesClient k8sClient, boolean useMaintenanceMode) {\n this.autoScalerFactory = autoScalerFactory;\n this.k8sClient = k8sClient;\n this.planner = new Planner(k8sClient, new AgentClientFactory(useMaintenanceMode));\n this.pools = new HashMap<>();\n this.events = new LinkedList<>();\n }\n\n public void onEvent(Event.Type type, AgentPool resource) {\n log.info(\"onEvent -> handling {} for {}/{}\", type, resource.getMetadata().getNamespace(), resource.getMetadata().getName());\n synchronized (events) {\n events.add(new Event(type, resource));\n }\n }\n\n public void start() {\n new Thread(new Worker(), \"scheduler-worker\").start();\n }\n\n /**\n * Process the recent events and update the cluster state.\n */\n private void doRun() {\n // drain the event queue\n List evs;\n synchronized (events) {\n evs = new ArrayList<>(events);\n events.clear();\n }\n\n for (Event e : evs) {\n AgentPool resource = e.getResource();\n String resourceName = resource.getMetadata().getName();\n\n switch (e.getType()) {\n case MODIFIED: {\n onAdd(resourceName, resource);\n break;\n }\n case DELETED: {\n onDelete(resourceName);\n break;\n }\n default:\n throw new IllegalArgumentException(\"Unknown event type: \" + e.getType());\n }\n }\n\n // process the pool\n List todo;\n synchronized (pools) {\n todo = new ArrayList<>(pools.values());\n }\n\n if (todo.isEmpty()) {\n return;\n }\n\n // fetch the process queue status\n\n todo.parallelStream().forEach(i -> {\n try {\n switch (i.getStatus()) {\n case ACTIVE: {\n AgentPoolInstance updated = updateTargetSize(i);\n processActive(updated);\n break;\n }\n case DELETED: {\n processDeleted(i);\n break;\n }\n default:\n throw new IllegalArgumentException(\"Unknown pool status: \" + i.getStatus());\n }\n } catch (IOException e) {\n log.error(\"doRun -> error while processing a registered pool {} ({}): {}\", i.getName(), i.getStatus(), e.getMessage());\n }\n });\n }\n\n private void onAdd(String resourceName, AgentPool resource) {\n int targetSize = resource.getSpec().getSize();\n synchronized (pools) {\n long currentTimeStamp = System.currentTimeMillis();\n pools.put(resourceName, new AgentPoolInstance(resourceName, resource, AgentPoolInstance.Status.ACTIVE,\n targetSize, currentTimeStamp, currentTimeStamp, currentTimeStamp));\n }\n }\n\n private void onDelete(String resourceName) {\n synchronized (pools) {\n AgentPoolInstance i = pools.get(resourceName);\n if (i == null) {\n return;\n }\n\n pools.put(resourceName, AgentPoolInstance.updateStatus(i, AgentPoolInstance.Status.DELETED));\n }\n }\n\n private AgentPoolInstance updateTargetSize(AgentPoolInstance i) throws IOException {\n if (!i.getResource().getSpec().isAutoScale()) {\n return i;\n }\n\n AgentPoolInstance result = autoScalerFactory.create(i).apply(i);\n\n synchronized (pools) {\n pools.put(i.getName(), result);\n }\n\n return result;\n }\n\n private void processActive(AgentPoolInstance i) throws IOException {\n log.info(\"processActive ['{}']\", i.getName());\n List changes = planner.plan(i);\n apply(changes);\n }\n\n private void processDeleted(AgentPoolInstance i) throws IOException {\n log.info(\"processDeleted ['{}']\", i.getName());\n String resourceName = i.getName();\n\n // remove all pool's pods\n List changes = planner.plan(i);\n apply(changes);\n\n // if no pods left - remove the pool\n List pods = AgentPod.list(k8sClient, resourceName);\n if (pods.isEmpty()) {\n synchronized (pools) {\n pools.remove(resourceName);\n log.info(\"processDeleted ['{}'] -> no pods left, the pool was removed\", resourceName);\n }\n } else {\n log.info(\"processDeleted ['{}'] -> {} pod(s) left, will be deleted on the next iteration\", resourceName, pods.size());\n }\n }\n\n private void apply(List changes) {\n changes.forEach(c -> c.apply(k8sClient));\n }\n\n private static void sleep(long ms) {\n try {\n Thread.sleep(ms);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n }\n }\n\n private class Worker implements Runnable {\n\n @Override\n public void run() {\n while (!Thread.currentThread().isInterrupted()) {\n try {\n doRun();\n sleep(POLL_DELAY);\n } catch (Exception e) {\n log.error(\"run -> error while running the scheduler: {}\", e.getMessage(), e);\n sleep(ERROR_DELAY);\n }\n }\n }\n }\n}\n"
},
{
"path": "agent-operator/src/main/resources/logback.xml",
"content": "\n \n \n %d{HH:mm:ss.SSS} [%thread] [%-5level] %logger{36} - %msg%n\n \n \n\n \n \n \n\n"
},
{
"path": "agent-operator/src/main/resources/prestop-hook.sh",
"content": "#!/usr/bin/env bash\n\necho \"PreStop hook started at $(date)\"\n\nMAX_RETRIES=5\nRETRY_DELAY=1\n\ncurrent_workers=\"1\"\nnum_retries=0\n\nwhile [ \"$current_workers\" != \"0\" ] && [ \"$num_retries\" -lt \"$MAX_RETRIES\" ]\ndo\n echo \"[$HOSTNAME]: Agent is still executing a process.. enabling maintenance mode and checking the number of current_workers\"\n\n response=$(exec 3<>/dev/tcp/127.0.0.1/8010;\n echo -e \"POST /maintenance-mode HTTP/1.1\\r\\nHost: 127.0.0.1:8010\\r\\nContent-Length: 0\\r\\nConnection: close\\r\\n\\r\\n\" >&3;\n cat <&3;\n exec 3>&-)\n\n mmode_response=$(echo \"$response\" | sed -n '/^\\r*$/,$p' | tail -n +2)\n mmode_enabled=$(echo \"$mmode_response\" | sed -n 's/^.*\\\"maintenanceMode\\\":\\([a-z]*\\).*$/\\1/p')\n\n if [ \"$mmode_enabled\" == \"true\" ]; then\n echo \"[$HOSTNAME]: Maintenance mode enabled: $mmode_enabled\"\n current_workers=$(echo \"$mmode_response\" | sed -n 's/^.*\\\"workersAlive\\\":\\([0-9]*\\).*$/\\1/p')\n echo \"[$HOSTNAME]: Number of current_workers: $current_workers\"\n else\n echo \"[$HOSTNAME]: trouble enabling maintenance mode\"\n num_retries=$((\"$num_retries\" + 1))\n fi\n\n sleep ${RETRY_DELAY}\ndone\n\nif [ \"$num_retries\" -ge \"$MAX_RETRIES\" ]; then\n echo \"[$HOSTNAME]: Number of retries to enable exceeded $MAX_RETRIES times. Exiting ...\"\n exit 1\nfi\n\necho \"[$HOSTNAME]: There are no processes running on this agent. Terminating...\"\n"
},
{
"path": "agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueClientTest.java",
"content": "package com.walmartlabs.concord.agentoperator.processqueue;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2024 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport org.junit.jupiter.api.Test;\n\nimport static com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient.escapeQueryParam;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class ProcessQueueClientTest {\n\n @Test\n public void testEscapeQueryParam() {\n assertEquals(\"foo%20bar\", escapeQueryParam(\"foo bar\"));\n assertEquals(\"foo=bar\", escapeQueryParam(\"foo=bar\"));\n assertEquals(\"foo=bar%20baz\", escapeQueryParam(\"foo=bar baz\"));\n assertEquals(\"foo.bar.baz=.*()%23%2F%2F&++$\", escapeQueryParam(\"foo.bar.baz=.*()#//&++$\"));\n }\n}\n"
},
{
"path": "agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/scheduler/DefaultAutoScalerTest.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class DefaultAutoScalerTest {\n\n @Test\n public void testStill() throws Exception {\n AtomicInteger podCount = new AtomicInteger(1);\n List queue = new ArrayList<>();\n\n DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setPercentIncrement(50);\n spec.setDecrementThresholdFactor(1.0);\n spec.setIncrementThresholdFactor(1.5);\n spec.setPercentDecrement(10);\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n }\n\n @Test\n public void testZeroStart() throws IOException {\n AtomicInteger podCount = new AtomicInteger(0);\n List queue = new ArrayList<>();\n\n DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setPercentIncrement(50);\n spec.setDecrementThresholdFactor(1.0);\n spec.setIncrementThresholdFactor(1.5);\n spec.setPercentDecrement(10);\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n\n podCount.set(2);\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n }\n\n private ProcessQueueClient mockProcessQueueClient(List queue) {\n return new ProcessQueueClient(\"test\", \"test\") {\n @Override\n public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {\n return queue;\n }\n };\n }\n\n @Test\n public void testQueue() throws IOException {\n AtomicInteger podCount = new AtomicInteger(1);\n List queue = new ArrayList<>();\n for (int i = 0; i < 10; i++) {\n queue.add(new ProcessQueueEntry(Collections.singletonMap(\"test\", 123)));\n }\n\n DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setPercentIncrement(50);\n spec.setDecrementThresholdFactor(1.0);\n spec.setIncrementThresholdFactor(1.5);\n spec.setPercentDecrement(10);\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(2, pool.getTargetSize());\n\n podCount.set(2);\n\n pool = as.apply(pool);\n assertEquals(3, pool.getTargetSize());\n\n podCount.set(3);\n\n pool = as.apply(pool);\n assertEquals(5, pool.getTargetSize());\n\n podCount.set(5);\n\n pool = as.apply(pool);\n assertEquals(8, pool.getTargetSize());\n\n podCount.set(8);\n\n // ---\n\n queue.clear();\n\n pool = as.apply(pool);\n assertEquals(7, pool.getTargetSize());\n\n podCount.set(7);\n\n pool = as.apply(pool);\n assertEquals(6, pool.getTargetSize());\n }\n}\n"
},
{
"path": "agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/scheduler/LinearAutoScalerTest.java",
"content": "package com.walmartlabs.concord.agentoperator.scheduler;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.agentoperator.crd.AgentPool;\nimport com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;\nimport com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class LinearAutoScalerTest {\n\n @Test\n public void testStill() throws Exception {\n AtomicInteger podCount = new AtomicInteger(1);\n List queue = new ArrayList<>();\n\n LinearAutoScaler as = new LinearAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n }\n\n @Test\n public void testZeroStart() throws IOException {\n AtomicInteger podCount = new AtomicInteger(0);\n List queue = new ArrayList<>();\n\n AutoScaler as = new LinearAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setMinSize(0);\n spec.setSize(0);\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 0, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(0, pool.getTargetSize());\n\n // 1 enqueued process -> inc pods count\n queue.add(new ProcessQueueEntry(Collections.singletonMap(\"test\", 123)));\n\n pool = as.apply(pool);\n assertEquals(1, pool.getTargetSize());\n\n podCount.set(1);\n\n // no processes -> 0 pods\n queue.clear();\n\n pool = as.apply(pool);\n assertEquals(0, pool.getTargetSize());\n }\n\n private ProcessQueueClient mockProcessQueueClient(List queue) {\n return new ProcessQueueClient(\"test\", \"test\") {\n @Override\n public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {\n return queue;\n }\n };\n }\n\n @Test\n public void testQueue() throws IOException {\n AtomicInteger podCount = new AtomicInteger(1);\n List queue = new ArrayList<>();\n for (int i = 0; i < 10; i++) {\n queue.add(new ProcessQueueEntry(Collections.singletonMap(\"test\", 123)));\n }\n\n DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);\n\n AgentPoolConfiguration spec = new AgentPoolConfiguration();\n spec.setPercentIncrement(50);\n spec.setDecrementThresholdFactor(1.0);\n spec.setIncrementThresholdFactor(1.5);\n spec.setPercentDecrement(10);\n spec.setQueueSelector(Collections.singletonMap(\"test\", 123));\n\n AgentPool resource = new AgentPool();\n resource.setSpec(spec);\n\n AgentPoolInstance pool = new AgentPoolInstance(\"test\", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);\n\n // ---\n\n pool = as.apply(pool);\n assertEquals(2, pool.getTargetSize());\n\n podCount.set(2);\n\n pool = as.apply(pool);\n assertEquals(3, pool.getTargetSize());\n\n podCount.set(3);\n\n pool = as.apply(pool);\n assertEquals(5, pool.getTargetSize());\n\n podCount.set(5);\n\n pool = as.apply(pool);\n assertEquals(8, pool.getTargetSize());\n\n podCount.set(8);\n\n // ---\n\n queue.clear();\n\n pool = as.apply(pool);\n assertEquals(7, pool.getTargetSize());\n\n podCount.set(7);\n\n pool = as.apply(pool);\n assertEquals(6, pool.getTargetSize());\n }\n}\n"
},
{
"path": "checkstyle.xml",
"content": "\n\n\n\n \n\n \n \n \n\n \n \n\n \n \n \n \n\n"
},
{
"path": "cli/pom.xml",
"content": "\n\n\n 4.0.0\n\n \n com.walmartlabs.concord\n parent\n 2.40.1-SNAPSHOT\n ../pom.xml\n \n\n concord-cli\n jar\n\n ${project.groupId}:${project.artifactId}\n\n \n com.walmartlabs.concord.cli.Main\n \n\n \n \n com.walmartlabs.concord.runtime\n concord-runtime-common\n \n\n \n \n com.walmartlabs.concord.runtime.v1\n concord-runtime-model-v1\n \n\n \n \n com.walmartlabs.concord.runtime.v2\n concord-runtime-model-v2\n \n \n com.walmartlabs.concord.runtime.v2\n concord-runner-v2\n \n \n com.walmartlabs.concord.runtime.v2\n concord-runtime-sdk-v2\n \n \n com.walmartlabs.concord.runtime.v2\n concord-runtime-vm-v2\n \n\n \n com.walmartlabs.concord\n concord-common\n \n \n com.walmartlabs.concord.runtime\n concord-runtime-loader\n \n \n com.walmartlabs.concord\n concord-imports\n \n \n com.walmartlabs.concord\n concord-repository\n \n \n com.walmartlabs.concord\n concord-dependency-manager\n \n \n com.walmartlabs.concord\n concord-sdk\n \n \n com.walmartlabs.concord\n concord-client2\n \n\n \n javax.inject\n javax.inject\n \n \n com.google.inject\n guice\n \n \n org.glassfish\n javax.el\n \n \n info.picocli\n picocli\n \n\n \n com.fasterxml.jackson.core\n jackson-core\n \n \n com.fasterxml.jackson.core\n jackson-databind\n \n \n com.fasterxml.jackson.dataformat\n jackson-dataformat-yaml\n \n\n \n org.fusesource.jansi\n jansi\n \n\n \n org.eclipse.jgit\n org.eclipse.jgit\n \n\n \n org.junit.jupiter\n junit-jupiter-api\n test\n \n \n\n \n \n \n false\n ${project.basedir}/src/main/resources\n \n **/*\n \n \n \n true\n ${project.basedir}/src/main/filtered-resources\n \n **/*\n \n \n \n \n \n false\n ${project.basedir}/src/test/resources\n \n **/*\n \n \n \n true\n ${project.basedir}/src/test/filtered-resources\n \n **/*\n \n \n \n\n \n \n org.apache.maven.plugins\n maven-shade-plugin\n \n \n package\n \n shade\n \n \n true\n executable\n \n \n META-INF/sisu/javax.inject.Named\n \n \n META-INF/services/com.oracle.truffle.api.TruffleLanguage$Provider\n \n \n \n ${main.class}\n java.base/java.lang java.base/java.util\n \n \n \n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.RSA\n \n \n \n \n \n \n \n \n org.skife.maven\n really-executable-jar-maven-plugin\n \n \n package\n \n really-executable-jar\n \n \n \n \n -Xmx128m -client\n executable\n true\n concord-cli-${project.version}\n sh\n true\n \n \n \n \n\n"
},
{
"path": "cli/src/main/filtered-resources/defaultCfg.yml",
"content": "configuration:\n dependencies:\n - \"mvn://com.walmartlabs.concord.plugins.basic:http-tasks:${project.version}\"\n - \"mvn://com.walmartlabs.concord.plugins.basic:slack-tasks:${project.version}\"\n - \"mvn://com.walmartlabs.concord.plugins.basic:concord-tasks:${project.version}\"\n"
},
{
"path": "cli/src/main/filtered-resources/project.properties",
"content": "project.version=${project.version}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/AbortException.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2025 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\npublic class AbortException extends RuntimeException {\n\n public AbortException() {\n super(\"Aborted\");\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/App.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport picocli.CommandLine.Command;\nimport picocli.CommandLine.Model.CommandSpec;\nimport picocli.CommandLine.Option;\nimport picocli.CommandLine.Spec;\n\n@Command(name = \"concord\", subcommands = {Lint.class, Run.class, Resume.class, RemoteRun.class, SelfUpdate.class})\npublic class App implements Runnable {\n\n @Spec\n private CommandSpec spec;\n\n @Option(names = {\"-h\", \"--help\"}, usageHelp = true, description = \"display a help message\")\n boolean helpRequested = false;\n\n @Option(names = {\"--version\"}, description = \"display version\")\n boolean versionRequested = false;\n\n @Override\n public void run() {\n if (versionRequested) {\n System.out.println(Version.getVersion());\n return;\n }\n\n spec.commandLine().usage(System.out);\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/CliConfig.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2025 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.fasterxml.jackson.databind.DeserializationFeature;\nimport com.fasterxml.jackson.databind.JsonMappingException;\nimport com.fasterxml.jackson.databind.JsonNode;\nimport com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;\nimport com.fasterxml.jackson.databind.node.ObjectNode;\nimport com.fasterxml.jackson.dataformat.yaml.YAMLMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport javax.annotation.Nullable;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Map;\nimport java.util.Optional;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\nimport static java.util.Objects.requireNonNull;\nimport static org.fusesource.jansi.Ansi.ansi;\n\npublic record CliConfig(Map contexts) {\n\n private static final Logger log = LoggerFactory.getLogger(CliConfig.class);\n\n public static CliConfig.CliConfigContext load(Verbosity verbosity, String context, Overrides overrides) {\n try {\n return loadOrThrow(verbosity, context, overrides);\n } catch (Exception e) {\n handleCliConfigErrorAndBail(resolveCliConfigPath().toAbsolutePath().toString(), e);\n throw new IllegalStateException(\"should be unreachable\");\n }\n }\n\n public static CliConfig.CliConfigContext loadOrThrow(Verbosity verbosity,\n String context,\n Overrides overrides) throws IOException {\n\n var cfgFile = resolveCliConfigPath();\n if (Files.notExists(cfgFile)) {\n var cfg = CliConfig.create();\n return requireCliConfigContext(cfg, context, false).withOverrides(overrides);\n }\n\n if (verbosity.verbose()) {\n log.info(\"Using CLI configuration file: {} (\\\"{}\\\" context)\", cfgFile, context);\n }\n\n var cfg = loadConfigFile(cfgFile);\n return requireCliConfigContext(cfg, context, true).withOverrides(overrides);\n }\n\n private static void handleCliConfigErrorAndBail(String cfgPath, Throwable e) {\n // unwrap runtime exceptions\n if (e instanceof RuntimeException ex) {\n if (ex.getCause() instanceof IllegalArgumentException) {\n e = ex.getCause();\n }\n }\n\n if (e instanceof MissingContextException) {\n System.out.println(ansi().fgRed().a(e.getMessage()));\n System.exit(1);\n }\n\n // handle YAML errors\n if (e instanceof IllegalArgumentException) {\n if (e.getCause() instanceof UnrecognizedPropertyException ex) {\n System.out.println(ansi().fgRed().a(\"Invalid format of the CLI configuration file \").a(cfgPath).a(\". \").a(ex.getMessage()));\n System.exit(1);\n }\n System.out.println(ansi().fgRed().a(\"Invalid format of the CLI configuration file \").a(cfgPath).a(\". \").a(e.getMessage()));\n System.exit(1);\n }\n\n // all other errors\n System.out.println(ansi().fgRed().a(\"Failed to read the CLI configuration file \").a(cfgPath).a(\". \").a(e.getMessage()));\n System.exit(1);\n }\n\n private static CliConfig.CliConfigContext requireCliConfigContext(CliConfig config, String context, boolean userConfigLoaded) {\n var result = config.contexts().get(context);\n if (result == null) {\n throw new MissingContextException(context, userConfigLoaded);\n }\n return result;\n }\n\n static final class MissingContextException extends IllegalArgumentException {\n\n private MissingContextException(String context, boolean userConfigLoaded) {\n super(message(context, userConfigLoaded));\n }\n\n private static String message(String context, boolean userConfigLoaded) {\n if (userConfigLoaded) {\n return \"Configuration context not found: \" + context + \". Check the CLI configuration file.\";\n }\n return \"Configuration context not found: \" + context + \". No CLI configuration file was found in ~/.concord; only the built-in 'default' context is available.\";\n }\n }\n\n @VisibleForTesting\n static CliConfig loadConfigFile(Path path) throws IOException {\n var mapper = new YAMLMapper()\n .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);\n\n JsonNode defaults = mapper.readTree(readDefaultConfig());\n\n JsonNode cfg;\n try (var reader = Files.newBufferedReader(path)) {\n cfg = mapper.readTree(reader);\n }\n\n // merge the loaded config file with the default built-in config\n var cfgWithDefaults = mapper.updateValue(defaults, cfg);\n\n // merge each non-default context with the default context\n var contexts = assertContexts(cfgWithDefaults);\n\n var defaultCtx = contexts.get(\"default\");\n if (defaultCtx == null) {\n throw new IllegalArgumentException(\"Missing 'default' context.\");\n }\n\n contexts.fieldNames().forEachRemaining(ctxName -> {\n if (\"default\".equals(ctxName)) {\n return;\n }\n\n var ctx = contexts.get(ctxName);\n try {\n var mergedCtx = mapper.updateValue(defaultCtx, ctx);\n contexts.set(ctxName, mergedCtx);\n } catch (JsonMappingException e) {\n throw new RuntimeException(e);\n }\n });\n\n return mapper.convertValue(cfgWithDefaults, CliConfig.class);\n }\n\n private static ObjectNode assertContexts(JsonNode cfg) {\n var maybeContexts = cfg.get(\"contexts\");\n if (maybeContexts == null) {\n throw new IllegalArgumentException(\"Missing 'contexts' object.\");\n }\n if (!maybeContexts.isObject()) {\n throw new IllegalArgumentException(\"The 'contexts' field must be an object.\");\n }\n return (ObjectNode) maybeContexts;\n }\n\n public static CliConfig create() {\n var mapper = new YAMLMapper();\n try {\n return mapper.readValue(readDefaultConfig(), CliConfig.class);\n } catch (IOException e) {\n throw new IllegalStateException(\"Can't parse the default CLI config file. \" + e.getMessage());\n }\n }\n\n public record Overrides(@Nullable Path secretStoreDir, @Nullable Path vaultDir, @Nullable String vaultId) {\n }\n\n static boolean hasUserConfig() {\n return Files.exists(Paths.get(System.getProperty(\"user.home\"), \".concord\", \"cli.yaml\"))\n || Files.exists(Paths.get(System.getProperty(\"user.home\"), \".concord\", \"cli.yml\"));\n }\n\n private static Path resolveCliConfigPath() {\n var baseDir = Paths.get(System.getProperty(\"user.home\"), \".concord\");\n var cfgFile = baseDir.resolve(\"cli.yaml\");\n if (Files.exists(cfgFile)) {\n return cfgFile;\n }\n return baseDir.resolve(\"cli.yml\");\n }\n\n public record CliConfigContext(@Nullable RemoteRunConfiguration remoteRun, SecretsConfiguration secrets) {\n\n public CliConfigContext withOverrides(@Nullable Overrides overrides) {\n if (overrides == null) {\n return this;\n }\n var remoteRun = this.remoteRun();\n var secrets = this.secrets().withOverrides(overrides);\n return new CliConfigContext(remoteRun, secrets);\n }\n }\n\n public record SecretRef(String orgName, String secretName) {\n\n public SecretRef(String orgName, String secretName) {\n this.orgName = orgName == null ? \"Default\" : orgName;\n if (this.orgName.isBlank()) {\n throw new IllegalArgumentException(\"'orgName' is required\");\n }\n this.secretName = requireNonNull(secretName);\n if (this.secretName.isBlank()) {\n throw new IllegalArgumentException(\"'secretName' is required\");\n }\n }\n }\n\n public record RemoteRunConfiguration(@Nullable String baseUrl, @Nullable SecretRef apiKeyRef) {\n }\n\n public record SecretsConfiguration(VaultConfiguration vault,\n FileSecretsProviderConfiguration local,\n RemoteSecretsProviderConfiguration remote) {\n\n public SecretsConfiguration withOverrides(@Nullable Overrides overrides) {\n if (overrides == null) {\n return this;\n }\n var vault = this.vault().withOverrides(overrides);\n var localFiles = this.local().withOverrides(overrides);\n return new SecretsConfiguration(vault, localFiles, this.remote);\n }\n\n public record VaultConfiguration(Path dir, String id) {\n\n public VaultConfiguration withOverrides(@Nullable Overrides overrides) {\n if (overrides == null) {\n return this;\n }\n return new VaultConfiguration(\n Optional.ofNullable(overrides.vaultDir()).orElse(this.dir()),\n Optional.ofNullable(overrides.vaultId()).orElse(this.id()));\n }\n }\n\n public record FileSecretsProviderConfiguration(boolean enabled, boolean writable, Path dir) {\n\n public FileSecretsProviderConfiguration withOverrides(@Nullable Overrides overrides) {\n if (overrides == null) {\n return this;\n }\n return new FileSecretsProviderConfiguration(\n this.enabled,\n this.writable,\n Optional.ofNullable(overrides.secretStoreDir()).orElse(this.dir()));\n }\n }\n\n public record RemoteSecretsProviderConfiguration(boolean enabled,\n boolean writable,\n @Nullable String baseUrl,\n @Nullable String apiKey,\n boolean confirmAccess) {\n }\n }\n\n private static String readDefaultConfig() {\n try (var in = CliConfig.class.getResourceAsStream(\"defaultCliConfig.yaml\")) {\n if (in == null) {\n throw new IllegalStateException(\"defaultCliConfig.yaml resource not found\");\n }\n var ab = in.readAllBytes();\n var s = new String(ab, UTF_8);\n\n var dotConcordPath = Paths.get(System.getProperty(\"user.home\")).resolve(\".concord\");\n return s.replace(\"${configDir}\", dotConcordPath.normalize().toAbsolutePath().toString());\n } catch (IOException e) {\n throw new IllegalStateException(\"Can't load the default CLI config file. \" + e.getMessage());\n }\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/CliExitCodes.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2026 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nfinal class CliExitCodes {\n\n static final int SUCCESS = 0;\n static final int ERROR = 1;\n static final int USAGE = 2;\n static final int SUSPENDED = 20;\n static final int INPUT_REQUIRED = 21;\n static final int NON_INTERACTIVE_UNSUPPORTED = 22;\n static final int PROCESS_FAILED = -1;\n\n private CliExitCodes() {\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/CliPaths.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2026 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport java.nio.file.Path;\n\npublic final class CliPaths {\n\n public static final String DEFAULT_TARGET_DIR_NAME = \"target\";\n\n public static Path defaultTargetDir(Path sourceDir) {\n return sourceDir.resolve(DEFAULT_TARGET_DIR_NAME);\n }\n\n public static Path preferredResumeDir(Path sourceDir, Path workDir) {\n var normalizedSourceDir = sourceDir.normalize().toAbsolutePath();\n var normalizedWorkDir = workDir.normalize().toAbsolutePath();\n if (normalizedWorkDir.equals(defaultTargetDir(normalizedSourceDir))) {\n return normalizedSourceDir;\n }\n return normalizedWorkDir;\n }\n\n private CliPaths() {\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/Confirmation.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2025 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport java.io.IOException;\n\nimport static org.fusesource.jansi.Ansi.ansi;\n\npublic final class Confirmation {\n\n public static boolean confirm(String message) throws IOException {\n return confirm(message, false);\n }\n\n public static boolean confirm(String message, boolean defaultValue) throws IOException {\n System.out.println(ansi().fgBrightYellow().bold().a(message).reset());\n\n var response = new StringBuilder();\n while (true) {\n int ch = System.in.read();\n if (ch == -1 || ch == '\\n') {\n break;\n }\n if (ch != '\\r') {\n response.append((char) ch);\n }\n }\n\n var value = response.toString().trim().toLowerCase();\n if (value.isEmpty()) {\n return defaultValue;\n }\n\n return \"y\".equals(value) || \"yes\".equals(value);\n }\n\n private Confirmation() {\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/GitIgnoreFilter.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2025 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport org.eclipse.jgit.ignore.IgnoreNode;\n\nimport java.io.IOException;\nimport java.nio.file.*;\nimport java.nio.file.attribute.BasicFileAttributes;\nimport java.util.*;\n\n/**\n * Utility class for filtering files based on .gitignore patterns.\n * Supports hierarchical .gitignore files in subdirectories per git spec.\n */\npublic class GitIgnoreFilter {\n\n private final Map ignoreNodes;\n\n private GitIgnoreFilter(Map ignoreNodes) {\n this.ignoreNodes = ignoreNodes;\n }\n\n /**\n * Loads all .gitignore files from the given directory and its subdirectories.\n *\n * @param baseDir the base directory to scan for .gitignore files\n * @return a GitIgnoreFilter instance, or null if no .gitignore files exist\n */\n public static GitIgnoreFilter load(Path baseDir) throws IOException {\n var nodes = new HashMap();\n\n Files.walkFileTree(baseDir, new SimpleFileVisitor<>() {\n @Override\n public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {\n var gitignore = dir.resolve(\".gitignore\");\n if (Files.isRegularFile(gitignore)) {\n var node = new IgnoreNode();\n try (var in = Files.newInputStream(gitignore)) {\n node.parse(in);\n }\n if (!node.getRules().isEmpty()) {\n var relDir = baseDir.relativize(dir);\n nodes.put(relDir, node);\n }\n }\n return FileVisitResult.CONTINUE;\n }\n });\n\n if (nodes.isEmpty()) {\n return null;\n }\n\n return new GitIgnoreFilter(nodes);\n }\n\n /**\n * Checks if the given path should be ignored based on .gitignore rules.\n *\n * @param relativePath the path relative to the base directory\n * @param isDirectory true if the path is a directory\n * @return true if the path should be ignored\n */\n public boolean isIgnored(Path relativePath, boolean isDirectory) {\n var pathStr = relativePath.toString().replace('\\\\', '/');\n Boolean ignored = null;\n\n // Check from root down to parent directory\n for (var ancestor : getAncestorPaths(relativePath)) {\n var node = ignoreNodes.get(ancestor);\n if (node != null) {\n // Make path relative to this ignore node's directory\n String relativeToNode;\n if (ancestor.toString().isEmpty()) {\n relativeToNode = pathStr;\n } else {\n var ancestorStr = ancestor.toString().replace('\\\\', '/');\n relativeToNode = pathStr.substring(ancestorStr.length() + 1);\n }\n\n var result = node.isIgnored(relativeToNode, isDirectory);\n if (result == IgnoreNode.MatchResult.IGNORED) {\n ignored = true;\n } else if (result == IgnoreNode.MatchResult.NOT_IGNORED) {\n ignored = false;\n }\n // CHECK_PARENT means no match, keep current value\n }\n }\n\n return Boolean.TRUE.equals(ignored);\n }\n\n /**\n * Returns all ancestor paths from root (empty path) to the parent of the given path.\n */\n private List getAncestorPaths(Path relativePath) {\n var ancestors = new ArrayList();\n ancestors.add(Paths.get(\"\")); // root\n\n var current = Paths.get(\"\");\n for (var i = 0; i < relativePath.getNameCount() - 1; i++) {\n current = current.resolve(relativePath.getName(i));\n ancestors.add(current);\n }\n\n return ancestors;\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/Lint.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2019 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.cli.lint.*;\nimport com.walmartlabs.concord.cli.lint.LintResult.Type;\nimport com.walmartlabs.concord.cli.runner.CliImportsListener;\nimport com.walmartlabs.concord.imports.ImportManager;\nimport com.walmartlabs.concord.imports.NoopImportManager;\nimport com.walmartlabs.concord.process.loader.DelegatingProjectLoader;\nimport com.walmartlabs.concord.runtime.model.ProcessDefinition;\nimport com.walmartlabs.concord.runtime.model.SourceMap;\nimport com.walmartlabs.concord.runtime.v1.ProjectLoaderV1;\nimport com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;\nimport org.fusesource.jansi.Ansi;\nimport picocli.CommandLine.Command;\nimport picocli.CommandLine.Model.CommandSpec;\nimport picocli.CommandLine.Option;\nimport picocli.CommandLine.Parameters;\nimport picocli.CommandLine.Spec;\n\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\n\nimport static org.fusesource.jansi.Ansi.ansi;\n\n@Command(name = \"lint\", description = \"Parse and validate Concord YAML files\")\npublic class Lint implements Callable {\n\n @Spec\n private CommandSpec spec;\n\n @Option(names = {\"-h\", \"--help\"}, usageHelp = true, description = \"display the command's help message\")\n boolean helpRequested = false;\n\n @Option(names = {\"-v\", \"--verbose\"}, description = \"Verbose output\")\n boolean verbose = false;\n\n @Parameters(arity = \"0..1\")\n Path targetDir = Paths.get(System.getProperty(\"user.dir\"));\n\n @Override\n public Integer call() throws Exception {\n targetDir = targetDir.normalize().toAbsolutePath();\n\n if (!Files.isDirectory(targetDir)) {\n throw new IllegalArgumentException(\"Not a directory: \" + targetDir);\n }\n\n ImportManager importManager = new NoopImportManager();\n ProjectLoaderV1 v1 = new ProjectLoaderV1(importManager);\n ProjectLoaderV2 v2 = new ProjectLoaderV2(importManager);\n DelegatingProjectLoader loader = new DelegatingProjectLoader(Set.of(v1, v2));\n ProcessDefinition pd = loader.loadProject(targetDir, new DummyImportsNormalizer(), verbose ? new CliImportsListener() : null).projectDefinition();\n\n List lintResults = new ArrayList<>();\n linters().forEach(l -> lintResults.addAll(l.apply(pd)));\n\n if (!lintResults.isEmpty()) {\n print(lintResults);\n println();\n }\n\n println(\"Found:\");\n println(\" imports: \" + pd.imports().items().size());\n println(\" profiles: \" + pd.profiles().size());\n println(\" flows: \" + pd.flows().size());\n println(\" forms: \" + pd.forms().size());\n println(\" triggers: \" + pd.triggers().size());\n println(\" (not counting dynamically imported resources)\");\n\n println();\n printStats(lintResults);\n\n println();\n boolean hasErrors = hasErrors(lintResults);\n if (hasErrors) {\n System.out.println(ansi().fgBrightRed().bold().a(\"INVALID\").reset());\n } else {\n System.out.println(ansi().fgBrightGreen().bold().a(\"VALID\").reset());\n }\n\n return hasErrors ? 10 : 0;\n }\n\n private List linters() {\n return Arrays.asList(\n new ExpressionLinter(verbose),\n new TaskCallLinter(verbose)\n );\n }\n\n private void print(List results) {\n for (LintResult r : results) {\n Ansi msg = ansi();\n switch (r.getType()) {\n case ERROR: {\n ansi().fgBrightRed().a(\"ERROR:\").reset();\n break;\n }\n case WARNING: {\n ansi().fgBrightYellow().a(\"WARN:\").reset();\n break;\n }\n default:\n throw new IllegalArgumentException(\"Unsupported result type: \" + r.getType());\n }\n\n SourceMap sm = r.getSourceMap();\n if (sm != null) {\n msg.a(\"@ [\").a(sm.source()).a(\"] line: \").a(sm.line()).a(\", col: \").a(sm.column());\n }\n\n msg.append(\"\\n\\t\").append(r.getMessage());\n\n println(msg);\n println(\"------------------------------------------------------------\");\n }\n }\n\n private void printStats(List results) {\n long errors = results.stream().filter(r -> r.getType() == Type.ERROR).count();\n long warns = results.stream().filter(r -> r.getType() == Type.WARNING).count();\n println(\"Result: \" + errors + \" error(s), \" + warns + \" warning(s)\");\n }\n\n private void println(Object o) {\n System.out.println(o);\n }\n\n private void println() {\n System.out.println();\n }\n\n private static boolean hasErrors(List results) {\n return results.stream().anyMatch(l -> l.getType() == Type.ERROR);\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/LocalCliRuntime.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2026 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.google.inject.Injector;\nimport com.walmartlabs.concord.cli.CliConfig.CliConfigContext;\nimport com.walmartlabs.concord.cli.runner.CliServicesModule;\nimport com.walmartlabs.concord.dependencymanager.DependencyManager;\nimport com.walmartlabs.concord.dependencymanager.DependencyManagerConfiguration;\nimport com.walmartlabs.concord.dependencymanager.DependencyManagerRepositories;\nimport com.walmartlabs.concord.imports.ImportsListener;\nimport com.walmartlabs.concord.imports.NoopImportManager;\nimport com.walmartlabs.concord.runtime.common.cfg.RunnerConfiguration;\nimport com.walmartlabs.concord.runtime.v2.NoopImportsNormalizer;\nimport com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;\nimport com.walmartlabs.concord.runtime.v2.runner.InjectorFactory;\nimport com.walmartlabs.concord.runtime.v2.runner.guice.ProcessDependenciesModule;\nimport com.walmartlabs.concord.runtime.v2.sdk.ProcessConfiguration;\nimport com.walmartlabs.concord.runtime.v2.sdk.WorkingDirectory;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\n\nfinal class LocalCliRuntime {\n\n static DependencyManager createDependencyManager(Path depsCacheDir) throws IOException {\n var cfgFile = Paths.get(System.getProperty(\"user.home\"), \".concord\", \"mvn.json\");\n if (Files.exists(cfgFile)) {\n return new DependencyManager(DependencyManagerConfiguration.of(depsCacheDir, DependencyManagerRepositories.get(cfgFile)));\n }\n return new DependencyManager(DependencyManagerConfiguration.of(depsCacheDir));\n }\n\n static Injector createInjector(Path workDir,\n RunnerConfiguration runnerCfg,\n ProcessConfiguration processCfg,\n CliConfigContext cliConfigContext,\n Path defaultTaskVars,\n DependencyManager dependencyManager,\n Verbosity verbosity) {\n\n return new InjectorFactory(new WorkingDirectory(workDir),\n runnerCfg,\n () -> processCfg,\n new ProcessDependenciesModule(workDir, runnerCfg.dependencies(), processCfg.debug()),\n new CliServicesModule(cliConfigContext, workDir, defaultTaskVars, dependencyManager, verbosity))\n .create();\n }\n\n static void notifyProjectLoaded(Path workDir) throws Exception {\n var loader = new ProjectLoaderV2(new NoopImportManager());\n loader.load(workDir, new NoopImportsNormalizer(), ImportsListener.NOP_LISTENER);\n }\n\n private LocalCliRuntime() {\n }\n}\n"
},
{
"path": "cli/src/main/java/com/walmartlabs/concord/cli/LocalFormInputs.java",
"content": "package com.walmartlabs.concord.cli;\n\n/*-\n * *****\n * Concord\n * -----\n * Copyright (C) 2017 - 2026 Walmart Inc.\n * -----\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n * =====\n */\n\nimport com.walmartlabs.concord.forms.DefaultFormValidator;\nimport com.walmartlabs.concord.forms.DefaultFormValidatorLocale;\nimport com.walmartlabs.concord.forms.Form;\nimport com.walmartlabs.concord.forms.FormField;\nimport com.walmartlabs.concord.forms.FormFields;\nimport com.walmartlabs.concord.forms.FormUtils;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.UncheckedIOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static com.walmartlabs.concord.forms.Constants.FORM_FILES;\n\nfinal class LocalFormInputs {\n\n private static final DefaultFormValidatorLocale LOCALE = new DefaultFormValidatorLocale();\n private static final DefaultFormValidator VALIDATOR = new DefaultFormValidator(LOCALE);\n\n static Converted convertAndValidate(Form form, Map rawValues) throws InputException {\n return convertAndValidate(form, rawValues, false);\n }\n\n static Converted convertAndValidate(Form form, Map input, boolean unwrapFormName) throws InputException {\n var rawValues = unwrapFormName ? unwrapFormValues(form, input) : input;\n var tmpFiles = new LinkedHashMap();\n var opened = new ArrayList();\n\n try {\n var convertedInput = prepareInput(form, rawValues, opened);\n var converted = new LinkedHashMap<>(FormUtils.convert(LOCALE, form, convertedInput, tmpFiles));\n var errors = VALIDATOR.validate(form, converted);\n if (!errors.isEmpty()) {\n cleanupTempFiles(tmpFiles);\n throw new InputException(errors.stream().map(e -> e.error()).toList());\n }\n\n return new Converted(converted, tmpFiles);\n } catch (FormUtils.ValidationException e) {\n cleanupTempFiles(tmpFiles);\n throw new InputException(List.of(e.getMessage()), e);\n } catch (IOException e) {\n cleanupTempFiles(tmpFiles);\n throw new InputException(List.of(e.getMessage()), e);\n } finally {\n close(opened);\n }\n }\n\n private static Map prepareInput(Form form, Map rawValues, List opened) throws IOException {\n var convertedInput = new LinkedHashMap();\n for (var field : form.fields()) {\n var value = rawValues.get(field.name());\n if (value == null) {\n continue;\n }\n\n convertedInput.put(field.name(), toFormInput(value, field, opened));\n }\n return convertedInput;\n }\n\n private static Object toFormInput(Object value, FormField field, List opened) throws IOException {\n if (!FormFields.FileField.TYPE.equals(field.type())) {\n return value;\n }\n\n if (value instanceof Path path) {\n var in = Files.newInputStream(path);\n opened.add(in);\n return in;\n }\n\n if (value instanceof Collection values) {\n var result = new ArrayList<>();\n for (var item : values) {\n result.add(toFormInput(item, field, opened));\n }\n return result;\n }\n\n return value;\n }\n\n private static Map