podNames) {
for (int i = 0; i < podNames.size() + 1; i++) {
String podName = podName(resourceName, i);
if (!podNames.contains(podName)) {
return podName;
}
}
throw new RuntimeException("Can't generate pod name for '" + resourceName + "', current pods: " + podNames);
}
private static String configMapName(String resourceName) {
return resourceName + "-cfg";
}
private static String podName(String resourceName, int i) {
return String.format("%s-%05d", resourceName, i);
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/planner/TagForRemovalChange.java
================================================
package com.walmartlabs.concord.agentoperator.planner;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.agent.AgentClient;
import com.walmartlabs.concord.agentoperator.PodLabels;
import com.walmartlabs.concord.agentoperator.resources.AgentPod;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TagForRemovalChange implements Change {
private static final Logger log = LoggerFactory.getLogger(TagForRemovalChange.class);
private final String podName;
private final AgentClient agentClient;
public TagForRemovalChange(String podName, AgentClient agentClient) {
this.podName = podName;
this.agentClient = agentClient;
}
@Override
public void apply(KubernetesClient client) {
try {
agentClient.enableMaintenanceMode();
} catch (Exception e) {
log.error("Error enabling maintenance mode for pod '{}'", podName, e);
return;
}
PodLabels.applyTag(client, podName, AgentPod.TAGGED_FOR_REMOVAL_LABEL, "true");
}
@Override
public String toString() {
return "TagForRemovalChange{" +
"podName='" + podName + '\'' +
'}';
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/planner/TryToDeletePodChange.java
================================================
package com.walmartlabs.concord.agentoperator.planner;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.agent.AgentClient;
import com.walmartlabs.concord.agentoperator.PodLabels;
import com.walmartlabs.concord.agentoperator.resources.AgentPod;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
public class TryToDeletePodChange implements Change {
private static final Logger log = LoggerFactory.getLogger(TryToDeletePodChange.class);
private final String podName;
private final AgentClient agentClient;
public TryToDeletePodChange(String podName, AgentClient agentClient) {
this.podName = podName;
this.agentClient = agentClient;
}
/**
* When the agent pod is being deleted, Kubernetes calls the prestop hook configured.
* The prestop hook script configured for agent container enables maintenance mode on the agent,
* and waits for the number of workers in use to go to 0, before the pod gets terminated.
*
* Whenever the scheduler calls this `apply` method, the following conditions are checked, and
* corresponding actions are executed.
*
* - If the pod has a label `preStopHookTermination: true`, do nothing and exit, as the pod is being
* terminated, waiting for the prestop hook script to complete (that is, last running process on the agent
* container to complete).
*
* - Otherwise if the pod is in `RUNNING` phase, call the kubernetes client `delete` method on the agent pod,
* which will put the pod in `Terminating` state and start executing the prestop hook script on the
* agent container. Add the label `preStopHookTermination: true` (this will be checked on
* subsequent executions).
*
* @param client instance of Kubernetes client
*/
@Override
public void apply(KubernetesClient client) {
Pod pod = client.pods().withName(podName).get();
if (pod == null) {
log.warn("apply ['{}'] -> pod doesn't exist, nothing to do", podName);
return;
}
Map labels = pod.getMetadata().getLabels();
if ("true".equals(labels.getOrDefault(AgentPod.PRE_STOP_HOOK_TERMINATION_LABEL, "false"))) {
log.debug("['{}'] -> has already been marked for termination", podName);
return;
}
try {
if (agentClient.hasBusyWorkers()) {
return;
}
} catch (Exception e) {
log.error("Error while checking agent workers count for pod '{}'", podName, e);
return;
}
// agent pod in maintenance mode and all workers done
client.pods().withName(podName).delete();
PodLabels.applyTag(client, podName, AgentPod.PRE_STOP_HOOK_TERMINATION_LABEL, "true");
log.info("apply ['{}'] -> Marked for termination (former phase: {})", podName, pod.getStatus().getPhase());
}
@Override
public String toString() {
return "TryToDeletePodChange{" +
"podName='" + podName + '\'' +
'}';
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueClient.java
================================================
package com.walmartlabs.concord.agentoperator.processqueue;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2024 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.escape.Escaper;
import com.google.common.net.UrlEscapers;
import com.walmartlabs.concord.agentoperator.scheduler.QueueSelector;
import com.walmartlabs.concord.sdk.Constants;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.net.CookieManager;
import java.net.CookiePolicy;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.List;
public class ProcessQueueClient {
private static final TypeReference> LIST_OF_PROCESS_QUEUE_ENTRIES = new TypeReference<>() {
};
private final String baseUrl;
private final String apiToken;
private final ObjectMapper objectMapper;
private final HttpClient client;
public ProcessQueueClient(String baseUrl, String apiToken) {
this.baseUrl = baseUrl;
this.apiToken = apiToken;
this.objectMapper = new ObjectMapper();
this.client = initClient();
}
public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {
StringBuilder queryUrl = new StringBuilder(baseUrl + "/api/v2/process/requirements?status=" + processStatus + "&limit=" + limit + "&startAt.len=");
String flavor = queueSelector.getFlavor();
if (flavor != null) {
queryUrl.append("&requirements.agent.flavor.eq=").append(flavor);
}
List queryParams = queueSelector.getQueryParams();
if (queryParams != null) {
for (String queryParam : queryParams) {
queryUrl.append("&").append(escapeQueryParam(queryParam));
}
}
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(queryUrl.toString()))
.header("Authorization", apiToken)
.header("User-Agent", "k8s-agent-operator")
.header(Constants.Headers.ENABLE_HTTP_SESSION, "true")
.GET()
.build();
HttpResponse response;
try {
response = client.send(request, HttpResponse.BodyHandlers.ofString());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while fetching the queue data", e);
}
if (response.statusCode() != 200) {
throw new IOException("Error while fetching the process queue data: " + response.statusCode());
}
return objectMapper.readValue(response.body(), LIST_OF_PROCESS_QUEUE_ENTRIES);
}
private static HttpClient initClient() {
try {
TrustManager[] trustAllCerts = new TrustManager[]{
new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
}
};
SSLContext sslContext = SSLContext.getInstance("SSL");
sslContext.init(null, trustAllCerts, new SecureRandom());
CookieManager cookieManager = new CookieManager();
cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
return HttpClient.newBuilder()
.sslContext(sslContext)
.cookieHandler(cookieManager)
.build();
} catch (NoSuchAlgorithmException | KeyManagementException e) {
throw new RuntimeException("Error while initializing the HTTP client", e);
}
}
@VisibleForTesting
static String escapeQueryParam(String s) {
Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
int i = s.indexOf("=");
if (i < 0) {
return escaper.escape(s);
}
String key = s.substring(0, i);
String value = s.substring(i + 1);
return escaper.escape(key) + "=" + escaper.escape(value);
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueEntry.java
================================================
package com.walmartlabs.concord.agentoperator.processqueue;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
public class ProcessQueueEntry implements Serializable {
private static final long serialVersionUID = 1L;
private final Map requirements;
@JsonCreator
public ProcessQueueEntry(@JsonProperty("requirements") Map requirements) {
this.requirements = requirements;
}
public Map getRequirements() {
return requirements;
}
@Override
public String toString() {
return "ProcessQueueEntry{" +
"requirements=" + requirements +
'}';
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/AgentConfigMap.java
================================================
package com.walmartlabs.concord.agentoperator.resources;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.walmartlabs.concord.agentoperator.HashUtils;
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.scheduler.AgentPoolInstance;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
public final class AgentConfigMap {
private static final Logger log = LoggerFactory.getLogger(AgentConfigMap.class);
private static final ObjectMapper objectMapper = new ObjectMapper();
public static ConfigMap get(KubernetesClient client, String configMapName) {
try {
return client.configMaps().withName(configMapName).get();
} catch (KubernetesClientException e) {
log.warn("get ['{}'] -> error while getting a configmap: {}", configMapName, e.getMessage());
throw e;
}
}
public static void create(KubernetesClient client, AgentPoolInstance poolInstance, String configMapName) throws IOException {
try {
ConfigMap m = prepare(client, poolInstance, configMapName);
client.configMaps().resource(m).create();
} catch (KubernetesClientException e) {
log.warn("create ['{}', '{}'] -> error while creating a configmap: {}", poolInstance.getName(), configMapName, e.getMessage());
throw e;
}
}
public static void delete(KubernetesClient client, String configMapName) {
try {
client.configMaps().withName(configMapName).delete();
// wait till it's actually removed
while (client.configMaps().withName(configMapName).get() != null) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
} catch (KubernetesClientException e) {
log.warn("delete ['{}'] -> error while deleting a configmap: {}", configMapName, e.getMessage());
throw e;
}
}
public static boolean hasChanges(KubernetesClient client, AgentPoolInstance poolInstance, ConfigMap a) throws IOException {
ConfigMap b = prepare(client, poolInstance, a.getMetadata().getName());
String hashA = HashUtils.hashAsHexString(a.getData());
String hashB = HashUtils.hashAsHexString(b.getData());
return !hashA.equals(hashB);
}
private static ConfigMap prepare(KubernetesClient client, AgentPoolInstance poolInstance, String configMapName) throws IOException {
try {
AgentPoolConfiguration spec = poolInstance.getResource().getSpec();
String configMapYaml = objectMapper.writeValueAsString(spec.getConfigMap())
.replaceAll("%%configMapName%%", configMapName)
.replace("%%preStopHook%%", escape(Resources.get("/prestop-hook.sh")));
return client.configMaps().load(new ByteArrayInputStream(configMapYaml.getBytes())).item();
} catch (KubernetesClientException e) {
log.warn("prepare ['{}', '{}'] -> error while preparing a configmap: {}", poolInstance.getName(), configMapName, e.getMessage());
throw e;
}
}
private static String escape(String str) throws JsonProcessingException {
String result = objectMapper.writeValueAsString(str);
return result.substring(1, result.length() - 1);
}
private AgentConfigMap() {
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/AgentPod.java
================================================
package com.walmartlabs.concord.agentoperator.resources;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.fasterxml.jackson.databind.ObjectMapper;
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.scheduler.AgentPoolInstance;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
public final class AgentPod {
private static final Logger log = LoggerFactory.getLogger(AgentPod.class);
public static final String TAGGED_FOR_REMOVAL_LABEL = "concordTaggedForRemoval";
public static final String PRE_STOP_HOOK_TERMINATION_LABEL = "preStopHookTermination";
public static final String POOL_NAME_LABEL = "poolName";
public static final String CONFIG_HASH_LABEL = "concordCfgHash";
private static final ObjectMapper objectMapper = new ObjectMapper();
public static List listMarkedForRemoval(KubernetesClient client, String resourceName) {
try {
return client.pods()
.withLabel(AgentPod.TAGGED_FOR_REMOVAL_LABEL)
.withLabel(AgentPod.POOL_NAME_LABEL, resourceName)
.list()
.getItems();
} catch (KubernetesClientException e) {
log.warn("listMarkedForRemoval ['{}'] -> error while listing marked for removal pods: {}", resourceName, e.getMessage());
throw e;
}
}
public static List list(KubernetesClient client, String resourceName) {
try {
return client.pods()
.withLabel(POOL_NAME_LABEL, resourceName)
.list()
.getItems();
} catch (KubernetesClientException e) {
log.warn("list ['{}'] -> error while listing pool pods: {}", resourceName, e.getMessage());
throw e;
}
}
public static void create(KubernetesClient client,
AgentPoolInstance poolInstance,
String podName,
String configMapName,
String hash) throws IOException {
try {
AgentPoolConfiguration spec = poolInstance.getResource().getSpec();
String podYaml = objectMapper.writeValueAsString(spec.getPod())
.replaceAll("%%podName%%", podName)
.replaceAll("%%app%%", AgentPool.SERVICE_FULL_NAME)
.replaceAll("%%" + POOL_NAME_LABEL + "%%", poolInstance.getName())
.replaceAll("%%configMapName%%", configMapName)
.replaceAll("%%" + CONFIG_HASH_LABEL + "%%", hash);
client.pods().load(new ByteArrayInputStream(podYaml.getBytes())).create();
} catch (KubernetesClientException e) {
log.warn("create ['{}', '{}', '{}', '{}'] -> error while creating a pod: {}", poolInstance.getName(), podName, configMapName, hash, e.getMessage());
throw e;
}
}
private AgentPod() {
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/resources/Resources.java
================================================
package com.walmartlabs.concord.agentoperator.resources;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2020 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public final class Resources {
private static final Map resources = new ConcurrentHashMap<>();
public static String get(String name) {
return resources.computeIfAbsent(name, Resources::load);
}
private static String load(String name) {
try (InputStream in = Resources.class.getResourceAsStream(name)) {
if (in == null) {
throw new RuntimeException("Resource '" + name + "' not found");
}
return CharStreams.toString(new InputStreamReader(in, Charsets.UTF_8));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Resources() {
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AgentPoolInstance.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
public class AgentPoolInstance {
public static AgentPoolInstance updateStatus(AgentPoolInstance i, Status status) {
return new AgentPoolInstance(i.name, i.resource, status, i.targetSize, System.currentTimeMillis(), i.getLastScaleUpTimestamp(), i.getLastScaleDownTimeStamp());
}
public static AgentPoolInstance updateTargetSize(AgentPoolInstance i, int targetSize, long scaleUptimeStamp, long scaleDownTimeStamp) {
return new AgentPoolInstance(i.name, i.resource, i.status, targetSize, System.currentTimeMillis(), scaleUptimeStamp, scaleDownTimeStamp);
}
private final String name;
private final AgentPool resource;
private final Status status;
private final int targetSize;
private final long lastUpdateTimestamp;
private final long lastScaleUpTimestamp;
private final long lastScaleDownTimeStamp;
public AgentPoolInstance(String name, AgentPool resource, Status status, int targetSize, long lastUpdateTimestamp,
long lastScaleUpTimestamp, long lastScaleDownTimeStamp) {
this.name = name;
this.resource = resource;
this.status = status;
this.targetSize = targetSize;
this.lastUpdateTimestamp = lastUpdateTimestamp;
this.lastScaleUpTimestamp = lastScaleUpTimestamp;
this.lastScaleDownTimeStamp = lastScaleDownTimeStamp;
}
public String getName() {
return name;
}
public AgentPool getResource() {
return resource;
}
public Status getStatus() {
return status;
}
public int getTargetSize() {
return targetSize;
}
public long getLastUpdateTimestamp() {
return lastUpdateTimestamp;
}
public long getLastScaleUpTimestamp() {
return lastScaleUpTimestamp;
}
public long getLastScaleDownTimeStamp() {
return lastScaleDownTimeStamp;
}
public enum Status {
ACTIVE,
DELETED
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AutoScaler.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2023 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import java.io.IOException;
public interface AutoScaler {
AgentPoolInstance apply(AgentPoolInstance i) throws IOException;
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/AutoScalerFactory.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2023 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;
import com.walmartlabs.concord.agentoperator.resources.AgentPod;
import io.fabric8.kubernetes.client.KubernetesClient;
import java.util.function.Function;
public class AutoScalerFactory {
private final Function podCounter;
private final ProcessQueueClient processQueueClient;
public AutoScalerFactory(String concordBaseUrl, String concordApiToken, KubernetesClient k8sClient) {
this.podCounter = n -> AgentPod.list(k8sClient, n).size();
this.processQueueClient = new ProcessQueueClient(concordBaseUrl, concordApiToken);
}
public AutoScaler create(AgentPoolInstance poolInstance) {
AgentPoolConfiguration cfg = poolInstance.getResource().getSpec();
if (LinearAutoScaler.NAME.equals(cfg.getAutoScaleStrategy())) {
return new LinearAutoScaler(processQueueClient, podCounter);
}
return new DefaultAutoScaler(processQueueClient, podCounter);
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/DefaultAutoScaler.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;
import com.walmartlabs.concord.common.Matcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
public class DefaultAutoScaler implements AutoScaler {
public static final String NAME = "default";
private static final Logger log = LoggerFactory.getLogger(DefaultAutoScaler.class);
private final ProcessQueueClient processQueueClient;
private final Function podCounter;
private final Function canBeScaledUp;
private final Function canBeScaledDown;
private long scaleUpTimeStamp;
private long scaleDownTimeStamp;
public DefaultAutoScaler(ProcessQueueClient processQueueClient, Function podCounter) {
this(processQueueClient, podCounter, i -> {
long t = System.currentTimeMillis();
return t - i.getLastScaleUpTimestamp() > i.getResource().getSpec().getScaleUpDelayMs();
}, i -> {
long t = System.currentTimeMillis();
return t - i.getLastScaleDownTimeStamp() > i.getResource().getSpec().getScaleDownDelayMs();
});
}
public DefaultAutoScaler(ProcessQueueClient processQueueClient,
Function podCounter, Function canBeScaledUp,
Function canBeScaledDown) {
this.processQueueClient = processQueueClient;
this.podCounter = podCounter;
this.canBeScaledUp = canBeScaledUp;
this.canBeScaledDown = canBeScaledDown;
this.scaleUpTimeStamp = System.currentTimeMillis();
this.scaleDownTimeStamp = System.currentTimeMillis();
}
/**
* Scale up or Scale down the number of agent pods depending on various conditions
*
* If enqueued process count is greater than the threshold defined for incrementing to max
* pool size, increase the pool size to the maximum size.
* Otherwise, if the enqueued process count is greater than the threshold defined for incrementing
* the pool size, increase the pool size by the increment percentage defined
*
* Decrease the pool size by the decrement percentage defined only if,
* - enqueued process count is lesser than the minimum pool size threshold defined
* - running process count is lesser than the threshold defined (which depends on current size of the pool
* running the processes, and a constant factor specified - default to 1. Simplified to
* runningCount < podsCount)
*
* @param i Agent pool on which the scaling activity is to be performed
*/
public AgentPoolInstance apply(AgentPoolInstance i) throws IOException {
int queueQueryLimit = i.getResource().getSpec().getQueueQueryLimit();
QueueSelector queueSelector = QueueSelector.parse(i.getResource().getSpec().getQueueSelector());
List queueEntries = processQueueClient.query("ENQUEUED", queueQueryLimit, queueSelector);
scaleUpTimeStamp = i.getLastScaleUpTimestamp();
scaleDownTimeStamp = i.getLastScaleDownTimeStamp();
AgentPoolConfiguration cfg = i.getResource().getSpec();
boolean canBeScaled = canBeScaledUp.apply(i) || canBeScaledDown.apply(i);
if (!canBeScaled) {
// was updated recently, skipping
return i;
}
// count the currently running pods
int podsCount = podCounter.apply(i.getName());
log.info("['{}']: Current pool size: {}", i.getName(), podsCount);
// the number of processes waiting for an agent in the current pool
int enqueuedCount = getProcessCount(cfg, queueEntries);
log.info("['{}']: Enqueued process count: {}", i.getName(), enqueuedCount);
if (podsCount < cfg.getMinSize()) {
return AgentPoolInstance.updateTargetSize(i, cfg.getMinSize(), System.currentTimeMillis(), System.currentTimeMillis());
}
// The threshold above which the operator can scale up the agent pods to the defined maximum pool size
double maxPoolSizeThreshold = cfg.getMaxSize() * cfg.getIncrementThresholdFactor();
// The threshold above which the pool size can be increased by the increment percentage defined
double incrementThreshold = cfg.getIncrementThresholdFactor() * podsCount;
// The threshold, combined with threshold for running processes determine if the pool size can be
// reduced by the decrement percentage defined
double minPoolSizeThreshold = cfg.getDecrementThresholdFactor() * cfg.getMinSize();
// Initial target size of the agent pool before updation
int targetSize = i.getTargetSize();
// Try scaling up if the time elapsed after last scale up operation
// is greater than the scale up delay defined (default: 15s)
if (canBeScaledUp.apply(i)) {
targetSize = tryScaleUp(cfg, i, podsCount, enqueuedCount, targetSize, maxPoolSizeThreshold, incrementThreshold);
// Reset scaledown delay counter if enqueued count is greater than min threshold.
// Scale down should happen only if enqueued count is less than
// min threshold consistently for scaledown delay defined (default: 180s)
if (enqueuedCount >= minPoolSizeThreshold) {
log.info("['{}']: Resetting scale down delay counter - (enqueued count({}) >= minimum threshold({}))...",
i.getName(), enqueuedCount, minPoolSizeThreshold);
scaleDownTimeStamp = System.currentTimeMillis();
}
}
// Try scaling down if the time elapsed after last scale down operation
// is greater than the scale down delay defined (default: 180s)
if (canBeScaledDown.apply(i)) {
targetSize = tryScaleDown(cfg, i, podsCount, enqueuedCount, targetSize, minPoolSizeThreshold);
}
if (targetSize == i.getTargetSize()) {
log.info("['{}']: Not changing the pool size.", i.getName());
} else {
log.info("apply ['{}'] -> updated to {}", i.getName(), targetSize);
}
return AgentPoolInstance.updateTargetSize(i, targetSize, scaleUpTimeStamp, scaleDownTimeStamp);
}
private int tryScaleUp(AgentPoolConfiguration cfg, AgentPoolInstance i, int podsCount, int enqueuedCount, int poolSize,
double maxPoolSizeThreshold, double incrementThreshold) {
// To prevent scale up before previous scale down action is completed
podsCount = Math.min(podsCount, i.getTargetSize());
// Reset scaleup delay counter for every attempt to scale up
scaleUpTimeStamp = System.currentTimeMillis();
if (podsCount < cfg.getMaxSize()) {
if (enqueuedCount >= maxPoolSizeThreshold) {
poolSize = cfg.getMaxSize();
log.info("['{}']: Incrementing to max size - {}", i.getName(), poolSize);
} else if (enqueuedCount >= incrementThreshold) {
poolSize = (int) Math.round(podsCount * (1 + cfg.getPercentIncrement() / 100));
// Limit to maximum pool size if the computed target size is more than the max size
if (poolSize > cfg.getMaxSize()) {
log.warn("['{}']: Target pool size exceeds the allowed maximum: {} > {}. Updating to maximum size - {}",
i.getName(), poolSize, cfg.getMaxSize(), cfg.getMaxSize());
poolSize = cfg.getMaxSize();
}
log.info("['{}']: Scaling up to {}...", i.getName(), poolSize);
}
} else {
log.warn("['{}']: Target pool size already the allowed maximum size: {}. Not updating.",
i.getName(), cfg.getMaxSize());
}
return poolSize;
}
private int tryScaleDown(AgentPoolConfiguration cfg, AgentPoolInstance i, int podsCount, int enqueuedCount,
int poolSize, double minPoolSizeThreshold) {
// To prevent scale down before previous scale up action is completed
podsCount = Math.max(podsCount, i.getTargetSize());
// Reset scaledown delay counter for every attempt to scale down
scaleDownTimeStamp = System.currentTimeMillis();
if (podsCount > cfg.getMinSize()) {
if (enqueuedCount < minPoolSizeThreshold) {
poolSize = (int) Math.floor(podsCount * (1 - cfg.getPercentDecrement() / 100));
log.info("['{}']: Scaling down - (enqueued count({}) < minimum threshold({})) for more than {} seconds...",
i.getName(), enqueuedCount, minPoolSizeThreshold,
(i.getResource().getSpec().getScaleDownDelayMs() / 1000));
// Limit to minimum pool size if the computed target size is less than the min size
if (poolSize < cfg.getMinSize()) {
log.warn("['{}']: Target pool size lesser than the allowed minimum: {} < {}. Updating to minimum size - {}",
i.getName(), poolSize, cfg.getMinSize(), cfg.getMinSize());
poolSize = cfg.getMinSize();
} else {
log.info("['{}']: Scaling down to {}...", i.getName(), poolSize);
}
}
} else {
log.warn("['{}']: Target pool size already the allowed minimum size: {}. Not updating.",
i.getName(), cfg.getMinSize());
}
return poolSize;
}
private static int getProcessCount(AgentPoolConfiguration cfg, List processQueueEntries) {
return (int) processQueueEntries.stream()
.map(ProcessQueueEntry::getRequirements)
.filter(a -> isEmpty(cfg.getQueueSelector()) || Matcher.matches(a, cfg.getQueueSelector()))
.count();
}
private static boolean isEmpty(Map m) {
return m == null || m.isEmpty();
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/Event.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
public class Event {
private final Type type;
private final AgentPool resource;
public Event(Type type, AgentPool resource) {
this.type = type;
this.resource = resource;
}
public Type getType() {
return type;
}
public AgentPool getResource() {
return resource;
}
public enum Type {
MODIFIED,
DELETED
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/LinearAutoScaler.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2023 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.function.Function;
public class LinearAutoScaler implements AutoScaler {
private static final Logger log = LoggerFactory.getLogger(LinearAutoScaler.class);
public static final String NAME = "linear";
private final ProcessQueueClient processQueueClient;
private final Function podCounter;
private final Function canBeScaledUp;
private final Function canBeScaledDown;
public LinearAutoScaler(ProcessQueueClient processQueueClient, Function podCounter) {
this(processQueueClient, podCounter, i -> {
long t = System.currentTimeMillis();
return t - i.getLastScaleUpTimestamp() > i.getResource().getSpec().getScaleUpDelayMs();
}, i -> {
long t = System.currentTimeMillis();
return t - i.getLastScaleDownTimeStamp() > i.getResource().getSpec().getScaleDownDelayMs();
});
}
public LinearAutoScaler(ProcessQueueClient processQueueClient,
Function podCounter, Function canBeScaledUp,
Function canBeScaledDown) {
this.processQueueClient = processQueueClient;
this.podCounter = podCounter;
this.canBeScaledUp = canBeScaledUp;
this.canBeScaledDown = canBeScaledDown;
}
@Override
public AgentPoolInstance apply(AgentPoolInstance i) throws IOException {
if (!canBeScaledUp.apply(i) && !canBeScaledDown.apply(i)) {
log.info("apply [{}] -> not a time. up: {}, down: {}, delay up: {}, delay down: {}", i.getName(), (System.currentTimeMillis() - i.getLastScaleUpTimestamp()), (System.currentTimeMillis() - i.getLastScaleDownTimeStamp()), i.getResource().getSpec().getScaleUpDelayMs(), i.getResource().getSpec().getScaleDownDelayMs());
// was updated recently, skipping
return i;
}
long scaleUpTimeStamp = i.getLastScaleUpTimestamp();
long scaleDownTimeStamp = i.getLastScaleDownTimeStamp();
AgentPoolConfiguration cfg = i.getResource().getSpec();
QueueSelector queueSelector = QueueSelector.parse(cfg.getQueueSelector());
List queueEntries = processQueueClient.query("ENQUEUED", cfg.getMaxSize(), queueSelector);
// count the currently running pods
int podsCount = podCounter.apply(i.getName());
// the number of processes waiting for an agent in the current pool
int enqueuedCount = queueEntries.size();
int runningCount = processQueueClient.query("RUNNING", cfg.getMaxSize(), queueSelector).size();
int freePodsCount = Math.max(podsCount - runningCount, 0);
int increment = 0;
if (enqueuedCount > freePodsCount) {
increment = cfg.getSizeIncrement();
scaleUpTimeStamp = System.currentTimeMillis();
scaleDownTimeStamp = System.currentTimeMillis();
} else if (enqueuedCount < freePodsCount) {
increment = -cfg.getSizeIncrement();
scaleDownTimeStamp = System.currentTimeMillis();
}
int targetSize = Math.max(cfg.getMinSize(), podsCount + increment);
if (i.getTargetSize() == targetSize) {
log.info("apply ['{}'] -> targetSize = {}, enqueuedCount = {}, increment = {}, podsCount = {}", i.getName(), targetSize, enqueuedCount, increment, podsCount);
// no changes needed
return i;
}
if (increment > 0 && !canBeScaledUp.apply(i)) {
log.info("apply ['{}'] -> not a time to scale up to {}", i.getName(), targetSize);
return i;
}
if (increment < 0 && !canBeScaledDown.apply(i)) {
log.info("apply ['{}'] -> not a time to scale down to {}", i.getName(), targetSize);
return i;
}
if (targetSize > cfg.getMaxSize()) {
log.warn("apply ['{}'] -> target pool size exceeds the allowed maximum: {} > {}", i.getName(), enqueuedCount, cfg.getMaxSize());
}
targetSize = Math.min(targetSize, cfg.getMaxSize());
log.info("apply ['{}'] -> updated to {}, pods: {}, free: {}, enqueued: {}, running: {}", i.getName(), targetSize, podsCount, freePodsCount, enqueuedCount, runningCount);
return AgentPoolInstance.updateTargetSize(i, targetSize, scaleUpTimeStamp, scaleDownTimeStamp);
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/QueueSelector.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2024 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.common.ConfigurationUtils;
import java.util.List;
import java.util.Map;
public class QueueSelector {
public static QueueSelector parse(Map queueSelector) {
String flavor;
Object maybeFlavor = ConfigurationUtils.get(queueSelector, "agent", "flavor");
if (maybeFlavor != null && !(maybeFlavor instanceof String)) {
throw new IllegalArgumentException("Expected a string value as 'agent.flavor', got: " + maybeFlavor);
}
flavor = (String) maybeFlavor;
List queryParams;
Object maybeQueryParams = ConfigurationUtils.get(queueSelector, "queryParams");
if (maybeQueryParams != null) {
if (!(maybeQueryParams instanceof List)) {
throw new IllegalArgumentException("Expected a list value as 'queryParams', got: " + maybeQueryParams);
}
((List) maybeQueryParams).forEach(qp -> {
if (!(qp instanceof String)) {
throw new IllegalArgumentException("Expected a string value as 'queryParams' item, got: " + qp);
}
});
}
//noinspection unchecked
queryParams = (List) maybeQueryParams;
return new QueueSelector(flavor, queryParams);
}
private final String flavor;
private final List queryParams;
private QueueSelector(String flavor, List queryParams) {
this.flavor = flavor;
this.queryParams = queryParams;
}
/**
* "Flavor" of the current agent. Translates to the "requirements.agent.flavor.eq"
* query parameter when fetching the process queue.
*/
public String getFlavor() {
return flavor;
}
/**
* Additional query parameters to be used when fetching the process queue. Appended
* as-is to the query URL.
*/
public List getQueryParams() {
return queryParams;
}
}
================================================
FILE: agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/Scheduler.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.agent.AgentClientFactory;
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
import com.walmartlabs.concord.agentoperator.planner.Change;
import com.walmartlabs.concord.agentoperator.planner.Planner;
import com.walmartlabs.concord.agentoperator.resources.AgentPod;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
public class Scheduler {
private static final Logger log = LoggerFactory.getLogger(Scheduler.class);
private static final long POLL_DELAY = 10000;
private static final long ERROR_DELAY = 30000;
private final AutoScalerFactory autoScalerFactory;
private final KubernetesClient k8sClient;
private final Planner planner;
private final Map pools;
private final List events;
public Scheduler(AutoScalerFactory autoScalerFactory, KubernetesClient k8sClient, boolean useMaintenanceMode) {
this.autoScalerFactory = autoScalerFactory;
this.k8sClient = k8sClient;
this.planner = new Planner(k8sClient, new AgentClientFactory(useMaintenanceMode));
this.pools = new HashMap<>();
this.events = new LinkedList<>();
}
public void onEvent(Event.Type type, AgentPool resource) {
log.info("onEvent -> handling {} for {}/{}", type, resource.getMetadata().getNamespace(), resource.getMetadata().getName());
synchronized (events) {
events.add(new Event(type, resource));
}
}
public void start() {
new Thread(new Worker(), "scheduler-worker").start();
}
/**
* Process the recent events and update the cluster state.
*/
private void doRun() {
// drain the event queue
List evs;
synchronized (events) {
evs = new ArrayList<>(events);
events.clear();
}
for (Event e : evs) {
AgentPool resource = e.getResource();
String resourceName = resource.getMetadata().getName();
switch (e.getType()) {
case MODIFIED: {
onAdd(resourceName, resource);
break;
}
case DELETED: {
onDelete(resourceName);
break;
}
default:
throw new IllegalArgumentException("Unknown event type: " + e.getType());
}
}
// process the pool
List todo;
synchronized (pools) {
todo = new ArrayList<>(pools.values());
}
if (todo.isEmpty()) {
return;
}
// fetch the process queue status
todo.parallelStream().forEach(i -> {
try {
switch (i.getStatus()) {
case ACTIVE: {
AgentPoolInstance updated = updateTargetSize(i);
processActive(updated);
break;
}
case DELETED: {
processDeleted(i);
break;
}
default:
throw new IllegalArgumentException("Unknown pool status: " + i.getStatus());
}
} catch (IOException e) {
log.error("doRun -> error while processing a registered pool {} ({}): {}", i.getName(), i.getStatus(), e.getMessage());
}
});
}
private void onAdd(String resourceName, AgentPool resource) {
int targetSize = resource.getSpec().getSize();
synchronized (pools) {
long currentTimeStamp = System.currentTimeMillis();
pools.put(resourceName, new AgentPoolInstance(resourceName, resource, AgentPoolInstance.Status.ACTIVE,
targetSize, currentTimeStamp, currentTimeStamp, currentTimeStamp));
}
}
private void onDelete(String resourceName) {
synchronized (pools) {
AgentPoolInstance i = pools.get(resourceName);
if (i == null) {
return;
}
pools.put(resourceName, AgentPoolInstance.updateStatus(i, AgentPoolInstance.Status.DELETED));
}
}
private AgentPoolInstance updateTargetSize(AgentPoolInstance i) throws IOException {
if (!i.getResource().getSpec().isAutoScale()) {
return i;
}
AgentPoolInstance result = autoScalerFactory.create(i).apply(i);
synchronized (pools) {
pools.put(i.getName(), result);
}
return result;
}
private void processActive(AgentPoolInstance i) throws IOException {
log.info("processActive ['{}']", i.getName());
List changes = planner.plan(i);
apply(changes);
}
private void processDeleted(AgentPoolInstance i) throws IOException {
log.info("processDeleted ['{}']", i.getName());
String resourceName = i.getName();
// remove all pool's pods
List changes = planner.plan(i);
apply(changes);
// if no pods left - remove the pool
List pods = AgentPod.list(k8sClient, resourceName);
if (pods.isEmpty()) {
synchronized (pools) {
pools.remove(resourceName);
log.info("processDeleted ['{}'] -> no pods left, the pool was removed", resourceName);
}
} else {
log.info("processDeleted ['{}'] -> {} pod(s) left, will be deleted on the next iteration", resourceName, pods.size());
}
}
private void apply(List changes) {
changes.forEach(c -> c.apply(k8sClient));
}
private static void sleep(long ms) {
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private class Worker implements Runnable {
@Override
public void run() {
while (!Thread.currentThread().isInterrupted()) {
try {
doRun();
sleep(POLL_DELAY);
} catch (Exception e) {
log.error("run -> error while running the scheduler: {}", e.getMessage(), e);
sleep(ERROR_DELAY);
}
}
}
}
}
================================================
FILE: agent-operator/src/main/resources/logback.xml
================================================
%d{HH:mm:ss.SSS} [%thread] [%-5level] %logger{36} - %msg%n
================================================
FILE: agent-operator/src/main/resources/prestop-hook.sh
================================================
#!/usr/bin/env bash
echo "PreStop hook started at $(date)"
MAX_RETRIES=5
RETRY_DELAY=1
current_workers="1"
num_retries=0
while [ "$current_workers" != "0" ] && [ "$num_retries" -lt "$MAX_RETRIES" ]
do
echo "[$HOSTNAME]: Agent is still executing a process.. enabling maintenance mode and checking the number of current_workers"
response=$(exec 3<>/dev/tcp/127.0.0.1/8010;
echo -e "POST /maintenance-mode HTTP/1.1\r\nHost: 127.0.0.1:8010\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" >&3;
cat <&3;
exec 3>&-)
mmode_response=$(echo "$response" | sed -n '/^\r*$/,$p' | tail -n +2)
mmode_enabled=$(echo "$mmode_response" | sed -n 's/^.*\"maintenanceMode\":\([a-z]*\).*$/\1/p')
if [ "$mmode_enabled" == "true" ]; then
echo "[$HOSTNAME]: Maintenance mode enabled: $mmode_enabled"
current_workers=$(echo "$mmode_response" | sed -n 's/^.*\"workersAlive\":\([0-9]*\).*$/\1/p')
echo "[$HOSTNAME]: Number of current_workers: $current_workers"
else
echo "[$HOSTNAME]: trouble enabling maintenance mode"
num_retries=$(("$num_retries" + 1))
fi
sleep ${RETRY_DELAY}
done
if [ "$num_retries" -ge "$MAX_RETRIES" ]; then
echo "[$HOSTNAME]: Number of retries to enable exceeded $MAX_RETRIES times. Exiting ..."
exit 1
fi
echo "[$HOSTNAME]: There are no processes running on this agent. Terminating..."
================================================
FILE: agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/processqueue/ProcessQueueClientTest.java
================================================
package com.walmartlabs.concord.agentoperator.processqueue;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2024 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import org.junit.jupiter.api.Test;
import static com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient.escapeQueryParam;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class ProcessQueueClientTest {
@Test
public void testEscapeQueryParam() {
assertEquals("foo%20bar", escapeQueryParam("foo bar"));
assertEquals("foo=bar", escapeQueryParam("foo=bar"));
assertEquals("foo=bar%20baz", escapeQueryParam("foo=bar baz"));
assertEquals("foo.bar.baz=.*()%23%2F%2F&++$", escapeQueryParam("foo.bar.baz=.*()#//&++$"));
}
}
================================================
FILE: agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/scheduler/DefaultAutoScalerTest.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class DefaultAutoScalerTest {
@Test
public void testStill() throws Exception {
AtomicInteger podCount = new AtomicInteger(1);
List queue = new ArrayList<>();
DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setPercentIncrement(50);
spec.setDecrementThresholdFactor(1.0);
spec.setIncrementThresholdFactor(1.5);
spec.setPercentDecrement(10);
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
}
@Test
public void testZeroStart() throws IOException {
AtomicInteger podCount = new AtomicInteger(0);
List queue = new ArrayList<>();
DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setPercentIncrement(50);
spec.setDecrementThresholdFactor(1.0);
spec.setIncrementThresholdFactor(1.5);
spec.setPercentDecrement(10);
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
podCount.set(2);
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
}
private ProcessQueueClient mockProcessQueueClient(List queue) {
return new ProcessQueueClient("test", "test") {
@Override
public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {
return queue;
}
};
}
@Test
public void testQueue() throws IOException {
AtomicInteger podCount = new AtomicInteger(1);
List queue = new ArrayList<>();
for (int i = 0; i < 10; i++) {
queue.add(new ProcessQueueEntry(Collections.singletonMap("test", 123)));
}
DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setPercentIncrement(50);
spec.setDecrementThresholdFactor(1.0);
spec.setIncrementThresholdFactor(1.5);
spec.setPercentDecrement(10);
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(2, pool.getTargetSize());
podCount.set(2);
pool = as.apply(pool);
assertEquals(3, pool.getTargetSize());
podCount.set(3);
pool = as.apply(pool);
assertEquals(5, pool.getTargetSize());
podCount.set(5);
pool = as.apply(pool);
assertEquals(8, pool.getTargetSize());
podCount.set(8);
// ---
queue.clear();
pool = as.apply(pool);
assertEquals(7, pool.getTargetSize());
podCount.set(7);
pool = as.apply(pool);
assertEquals(6, pool.getTargetSize());
}
}
================================================
FILE: agent-operator/src/test/java/com/walmartlabs/concord/agentoperator/scheduler/LinearAutoScalerTest.java
================================================
package com.walmartlabs.concord.agentoperator.scheduler;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.agentoperator.crd.AgentPool;
import com.walmartlabs.concord.agentoperator.crd.AgentPoolConfiguration;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueClient;
import com.walmartlabs.concord.agentoperator.processqueue.ProcessQueueEntry;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class LinearAutoScalerTest {
@Test
public void testStill() throws Exception {
AtomicInteger podCount = new AtomicInteger(1);
List queue = new ArrayList<>();
LinearAutoScaler as = new LinearAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
}
@Test
public void testZeroStart() throws IOException {
AtomicInteger podCount = new AtomicInteger(0);
List queue = new ArrayList<>();
AutoScaler as = new LinearAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setMinSize(0);
spec.setSize(0);
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 0, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(0, pool.getTargetSize());
// 1 enqueued process -> inc pods count
queue.add(new ProcessQueueEntry(Collections.singletonMap("test", 123)));
pool = as.apply(pool);
assertEquals(1, pool.getTargetSize());
podCount.set(1);
// no processes -> 0 pods
queue.clear();
pool = as.apply(pool);
assertEquals(0, pool.getTargetSize());
}
private ProcessQueueClient mockProcessQueueClient(List queue) {
return new ProcessQueueClient("test", "test") {
@Override
public List query(String processStatus, int limit, QueueSelector queueSelector) throws IOException {
return queue;
}
};
}
@Test
public void testQueue() throws IOException {
AtomicInteger podCount = new AtomicInteger(1);
List queue = new ArrayList<>();
for (int i = 0; i < 10; i++) {
queue.add(new ProcessQueueEntry(Collections.singletonMap("test", 123)));
}
DefaultAutoScaler as = new DefaultAutoScaler(mockProcessQueueClient(queue), n -> podCount.get(), i -> true, i -> true);
AgentPoolConfiguration spec = new AgentPoolConfiguration();
spec.setPercentIncrement(50);
spec.setDecrementThresholdFactor(1.0);
spec.setIncrementThresholdFactor(1.5);
spec.setPercentDecrement(10);
spec.setQueueSelector(Collections.singletonMap("test", 123));
AgentPool resource = new AgentPool();
resource.setSpec(spec);
AgentPoolInstance pool = new AgentPoolInstance("test", resource, AgentPoolInstance.Status.ACTIVE, 1, 0, 0, 0);
// ---
pool = as.apply(pool);
assertEquals(2, pool.getTargetSize());
podCount.set(2);
pool = as.apply(pool);
assertEquals(3, pool.getTargetSize());
podCount.set(3);
pool = as.apply(pool);
assertEquals(5, pool.getTargetSize());
podCount.set(5);
pool = as.apply(pool);
assertEquals(8, pool.getTargetSize());
podCount.set(8);
// ---
queue.clear();
pool = as.apply(pool);
assertEquals(7, pool.getTargetSize());
podCount.set(7);
pool = as.apply(pool);
assertEquals(6, pool.getTargetSize());
}
}
================================================
FILE: checkstyle.xml
================================================
================================================
FILE: cli/pom.xml
================================================
4.0.0
com.walmartlabs.concord
parent
2.40.1-SNAPSHOT
../pom.xml
concord-cli
jar
${project.groupId}:${project.artifactId}
com.walmartlabs.concord.cli.Main
com.walmartlabs.concord.runtime
concord-runtime-common
com.walmartlabs.concord.runtime.v1
concord-runtime-model-v1
com.walmartlabs.concord.runtime.v2
concord-runtime-model-v2
com.walmartlabs.concord.runtime.v2
concord-runner-v2
com.walmartlabs.concord.runtime.v2
concord-runtime-sdk-v2
com.walmartlabs.concord.runtime.v2
concord-runtime-vm-v2
com.walmartlabs.concord
concord-common
com.walmartlabs.concord.runtime
concord-runtime-loader
com.walmartlabs.concord
concord-imports
com.walmartlabs.concord
concord-repository
com.walmartlabs.concord
concord-dependency-manager
com.walmartlabs.concord
concord-sdk
com.walmartlabs.concord
concord-client2
javax.inject
javax.inject
com.google.inject
guice
org.glassfish
javax.el
info.picocli
picocli
com.fasterxml.jackson.core
jackson-core
com.fasterxml.jackson.core
jackson-databind
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
org.fusesource.jansi
jansi
org.eclipse.jgit
org.eclipse.jgit
org.junit.jupiter
junit-jupiter-api
test
false
${project.basedir}/src/main/resources
**/*
true
${project.basedir}/src/main/filtered-resources
**/*
false
${project.basedir}/src/test/resources
**/*
true
${project.basedir}/src/test/filtered-resources
**/*
org.apache.maven.plugins
maven-shade-plugin
package
shade
true
executable
META-INF/sisu/javax.inject.Named
META-INF/services/com.oracle.truffle.api.TruffleLanguage$Provider
${main.class}
java.base/java.lang java.base/java.util
*:*
META-INF/*.SF
META-INF/*.RSA
org.skife.maven
really-executable-jar-maven-plugin
package
really-executable-jar
-Xmx128m -client
executable
true
concord-cli-${project.version}
sh
true
================================================
FILE: cli/src/main/filtered-resources/defaultCfg.yml
================================================
configuration:
dependencies:
- "mvn://com.walmartlabs.concord.plugins.basic:http-tasks:${project.version}"
- "mvn://com.walmartlabs.concord.plugins.basic:slack-tasks:${project.version}"
- "mvn://com.walmartlabs.concord.plugins.basic:concord-tasks:${project.version}"
================================================
FILE: cli/src/main/filtered-resources/project.properties
================================================
project.version=${project.version}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/AbortException.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2025 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
public class AbortException extends RuntimeException {
public AbortException() {
super("Aborted");
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/App.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import picocli.CommandLine.Command;
import picocli.CommandLine.Model.CommandSpec;
import picocli.CommandLine.Option;
import picocli.CommandLine.Spec;
@Command(name = "concord", subcommands = {Lint.class, Run.class, Resume.class, RemoteRun.class, SelfUpdate.class})
public class App implements Runnable {
@Spec
private CommandSpec spec;
@Option(names = {"-h", "--help"}, usageHelp = true, description = "display a help message")
boolean helpRequested = false;
@Option(names = {"--version"}, description = "display version")
boolean versionRequested = false;
@Override
public void run() {
if (versionRequested) {
System.out.println(Version.getVersion());
return;
}
spec.commandLine().usage(System.out);
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/CliConfig.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2025 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Optional;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static org.fusesource.jansi.Ansi.ansi;
public record CliConfig(Map contexts) {
private static final Logger log = LoggerFactory.getLogger(CliConfig.class);
public static CliConfig.CliConfigContext load(Verbosity verbosity, String context, Overrides overrides) {
try {
return loadOrThrow(verbosity, context, overrides);
} catch (Exception e) {
handleCliConfigErrorAndBail(resolveCliConfigPath().toAbsolutePath().toString(), e);
throw new IllegalStateException("should be unreachable");
}
}
public static CliConfig.CliConfigContext loadOrThrow(Verbosity verbosity,
String context,
Overrides overrides) throws IOException {
var cfgFile = resolveCliConfigPath();
if (Files.notExists(cfgFile)) {
var cfg = CliConfig.create();
return requireCliConfigContext(cfg, context, false).withOverrides(overrides);
}
if (verbosity.verbose()) {
log.info("Using CLI configuration file: {} (\"{}\" context)", cfgFile, context);
}
var cfg = loadConfigFile(cfgFile);
return requireCliConfigContext(cfg, context, true).withOverrides(overrides);
}
private static void handleCliConfigErrorAndBail(String cfgPath, Throwable e) {
// unwrap runtime exceptions
if (e instanceof RuntimeException ex) {
if (ex.getCause() instanceof IllegalArgumentException) {
e = ex.getCause();
}
}
if (e instanceof MissingContextException) {
System.out.println(ansi().fgRed().a(e.getMessage()));
System.exit(1);
}
// handle YAML errors
if (e instanceof IllegalArgumentException) {
if (e.getCause() instanceof UnrecognizedPropertyException ex) {
System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(ex.getMessage()));
System.exit(1);
}
System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
System.exit(1);
}
// all other errors
System.out.println(ansi().fgRed().a("Failed to read the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
System.exit(1);
}
private static CliConfig.CliConfigContext requireCliConfigContext(CliConfig config, String context, boolean userConfigLoaded) {
var result = config.contexts().get(context);
if (result == null) {
throw new MissingContextException(context, userConfigLoaded);
}
return result;
}
static final class MissingContextException extends IllegalArgumentException {
private MissingContextException(String context, boolean userConfigLoaded) {
super(message(context, userConfigLoaded));
}
private static String message(String context, boolean userConfigLoaded) {
if (userConfigLoaded) {
return "Configuration context not found: " + context + ". Check the CLI configuration file.";
}
return "Configuration context not found: " + context + ". No CLI configuration file was found in ~/.concord; only the built-in 'default' context is available.";
}
}
@VisibleForTesting
static CliConfig loadConfigFile(Path path) throws IOException {
var mapper = new YAMLMapper()
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
JsonNode defaults = mapper.readTree(readDefaultConfig());
JsonNode cfg;
try (var reader = Files.newBufferedReader(path)) {
cfg = mapper.readTree(reader);
}
// merge the loaded config file with the default built-in config
var cfgWithDefaults = mapper.updateValue(defaults, cfg);
// merge each non-default context with the default context
var contexts = assertContexts(cfgWithDefaults);
var defaultCtx = contexts.get("default");
if (defaultCtx == null) {
throw new IllegalArgumentException("Missing 'default' context.");
}
contexts.fieldNames().forEachRemaining(ctxName -> {
if ("default".equals(ctxName)) {
return;
}
var ctx = contexts.get(ctxName);
try {
var mergedCtx = mapper.updateValue(defaultCtx, ctx);
contexts.set(ctxName, mergedCtx);
} catch (JsonMappingException e) {
throw new RuntimeException(e);
}
});
return mapper.convertValue(cfgWithDefaults, CliConfig.class);
}
private static ObjectNode assertContexts(JsonNode cfg) {
var maybeContexts = cfg.get("contexts");
if (maybeContexts == null) {
throw new IllegalArgumentException("Missing 'contexts' object.");
}
if (!maybeContexts.isObject()) {
throw new IllegalArgumentException("The 'contexts' field must be an object.");
}
return (ObjectNode) maybeContexts;
}
public static CliConfig create() {
var mapper = new YAMLMapper();
try {
return mapper.readValue(readDefaultConfig(), CliConfig.class);
} catch (IOException e) {
throw new IllegalStateException("Can't parse the default CLI config file. " + e.getMessage());
}
}
public record Overrides(@Nullable Path secretStoreDir, @Nullable Path vaultDir, @Nullable String vaultId) {
}
static boolean hasUserConfig() {
return Files.exists(Paths.get(System.getProperty("user.home"), ".concord", "cli.yaml"))
|| Files.exists(Paths.get(System.getProperty("user.home"), ".concord", "cli.yml"));
}
private static Path resolveCliConfigPath() {
var baseDir = Paths.get(System.getProperty("user.home"), ".concord");
var cfgFile = baseDir.resolve("cli.yaml");
if (Files.exists(cfgFile)) {
return cfgFile;
}
return baseDir.resolve("cli.yml");
}
public record CliConfigContext(@Nullable RemoteRunConfiguration remoteRun, SecretsConfiguration secrets) {
public CliConfigContext withOverrides(@Nullable Overrides overrides) {
if (overrides == null) {
return this;
}
var remoteRun = this.remoteRun();
var secrets = this.secrets().withOverrides(overrides);
return new CliConfigContext(remoteRun, secrets);
}
}
public record SecretRef(String orgName, String secretName) {
public SecretRef(String orgName, String secretName) {
this.orgName = orgName == null ? "Default" : orgName;
if (this.orgName.isBlank()) {
throw new IllegalArgumentException("'orgName' is required");
}
this.secretName = requireNonNull(secretName);
if (this.secretName.isBlank()) {
throw new IllegalArgumentException("'secretName' is required");
}
}
}
public record RemoteRunConfiguration(@Nullable String baseUrl, @Nullable SecretRef apiKeyRef) {
}
public record SecretsConfiguration(VaultConfiguration vault,
FileSecretsProviderConfiguration local,
RemoteSecretsProviderConfiguration remote) {
public SecretsConfiguration withOverrides(@Nullable Overrides overrides) {
if (overrides == null) {
return this;
}
var vault = this.vault().withOverrides(overrides);
var localFiles = this.local().withOverrides(overrides);
return new SecretsConfiguration(vault, localFiles, this.remote);
}
public record VaultConfiguration(Path dir, String id) {
public VaultConfiguration withOverrides(@Nullable Overrides overrides) {
if (overrides == null) {
return this;
}
return new VaultConfiguration(
Optional.ofNullable(overrides.vaultDir()).orElse(this.dir()),
Optional.ofNullable(overrides.vaultId()).orElse(this.id()));
}
}
public record FileSecretsProviderConfiguration(boolean enabled, boolean writable, Path dir) {
public FileSecretsProviderConfiguration withOverrides(@Nullable Overrides overrides) {
if (overrides == null) {
return this;
}
return new FileSecretsProviderConfiguration(
this.enabled,
this.writable,
Optional.ofNullable(overrides.secretStoreDir()).orElse(this.dir()));
}
}
public record RemoteSecretsProviderConfiguration(boolean enabled,
boolean writable,
@Nullable String baseUrl,
@Nullable String apiKey,
boolean confirmAccess) {
}
}
private static String readDefaultConfig() {
try (var in = CliConfig.class.getResourceAsStream("defaultCliConfig.yaml")) {
if (in == null) {
throw new IllegalStateException("defaultCliConfig.yaml resource not found");
}
var ab = in.readAllBytes();
var s = new String(ab, UTF_8);
var dotConcordPath = Paths.get(System.getProperty("user.home")).resolve(".concord");
return s.replace("${configDir}", dotConcordPath.normalize().toAbsolutePath().toString());
} catch (IOException e) {
throw new IllegalStateException("Can't load the default CLI config file. " + e.getMessage());
}
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/CliExitCodes.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2026 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
final class CliExitCodes {
static final int SUCCESS = 0;
static final int ERROR = 1;
static final int USAGE = 2;
static final int SUSPENDED = 20;
static final int INPUT_REQUIRED = 21;
static final int NON_INTERACTIVE_UNSUPPORTED = 22;
static final int PROCESS_FAILED = -1;
private CliExitCodes() {
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/CliPaths.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2026 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import java.nio.file.Path;
public final class CliPaths {
public static final String DEFAULT_TARGET_DIR_NAME = "target";
public static Path defaultTargetDir(Path sourceDir) {
return sourceDir.resolve(DEFAULT_TARGET_DIR_NAME);
}
public static Path preferredResumeDir(Path sourceDir, Path workDir) {
var normalizedSourceDir = sourceDir.normalize().toAbsolutePath();
var normalizedWorkDir = workDir.normalize().toAbsolutePath();
if (normalizedWorkDir.equals(defaultTargetDir(normalizedSourceDir))) {
return normalizedSourceDir;
}
return normalizedWorkDir;
}
private CliPaths() {
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/Confirmation.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2025 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import java.io.IOException;
import static org.fusesource.jansi.Ansi.ansi;
public final class Confirmation {
public static boolean confirm(String message) throws IOException {
return confirm(message, false);
}
public static boolean confirm(String message, boolean defaultValue) throws IOException {
System.out.println(ansi().fgBrightYellow().bold().a(message).reset());
var response = new StringBuilder();
while (true) {
int ch = System.in.read();
if (ch == -1 || ch == '\n') {
break;
}
if (ch != '\r') {
response.append((char) ch);
}
}
var value = response.toString().trim().toLowerCase();
if (value.isEmpty()) {
return defaultValue;
}
return "y".equals(value) || "yes".equals(value);
}
private Confirmation() {
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/GitIgnoreFilter.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2025 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import org.eclipse.jgit.ignore.IgnoreNode;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
/**
* Utility class for filtering files based on .gitignore patterns.
* Supports hierarchical .gitignore files in subdirectories per git spec.
*/
public class GitIgnoreFilter {
private final Map ignoreNodes;
private GitIgnoreFilter(Map ignoreNodes) {
this.ignoreNodes = ignoreNodes;
}
/**
* Loads all .gitignore files from the given directory and its subdirectories.
*
* @param baseDir the base directory to scan for .gitignore files
* @return a GitIgnoreFilter instance, or null if no .gitignore files exist
*/
public static GitIgnoreFilter load(Path baseDir) throws IOException {
var nodes = new HashMap();
Files.walkFileTree(baseDir, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
var gitignore = dir.resolve(".gitignore");
if (Files.isRegularFile(gitignore)) {
var node = new IgnoreNode();
try (var in = Files.newInputStream(gitignore)) {
node.parse(in);
}
if (!node.getRules().isEmpty()) {
var relDir = baseDir.relativize(dir);
nodes.put(relDir, node);
}
}
return FileVisitResult.CONTINUE;
}
});
if (nodes.isEmpty()) {
return null;
}
return new GitIgnoreFilter(nodes);
}
/**
* Checks if the given path should be ignored based on .gitignore rules.
*
* @param relativePath the path relative to the base directory
* @param isDirectory true if the path is a directory
* @return true if the path should be ignored
*/
public boolean isIgnored(Path relativePath, boolean isDirectory) {
var pathStr = relativePath.toString().replace('\\', '/');
Boolean ignored = null;
// Check from root down to parent directory
for (var ancestor : getAncestorPaths(relativePath)) {
var node = ignoreNodes.get(ancestor);
if (node != null) {
// Make path relative to this ignore node's directory
String relativeToNode;
if (ancestor.toString().isEmpty()) {
relativeToNode = pathStr;
} else {
var ancestorStr = ancestor.toString().replace('\\', '/');
relativeToNode = pathStr.substring(ancestorStr.length() + 1);
}
var result = node.isIgnored(relativeToNode, isDirectory);
if (result == IgnoreNode.MatchResult.IGNORED) {
ignored = true;
} else if (result == IgnoreNode.MatchResult.NOT_IGNORED) {
ignored = false;
}
// CHECK_PARENT means no match, keep current value
}
}
return Boolean.TRUE.equals(ignored);
}
/**
* Returns all ancestor paths from root (empty path) to the parent of the given path.
*/
private List getAncestorPaths(Path relativePath) {
var ancestors = new ArrayList();
ancestors.add(Paths.get("")); // root
var current = Paths.get("");
for (var i = 0; i < relativePath.getNameCount() - 1; i++) {
current = current.resolve(relativePath.getName(i));
ancestors.add(current);
}
return ancestors;
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/Lint.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2019 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.cli.lint.*;
import com.walmartlabs.concord.cli.lint.LintResult.Type;
import com.walmartlabs.concord.cli.runner.CliImportsListener;
import com.walmartlabs.concord.imports.ImportManager;
import com.walmartlabs.concord.imports.NoopImportManager;
import com.walmartlabs.concord.process.loader.DelegatingProjectLoader;
import com.walmartlabs.concord.runtime.model.ProcessDefinition;
import com.walmartlabs.concord.runtime.model.SourceMap;
import com.walmartlabs.concord.runtime.v1.ProjectLoaderV1;
import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;
import org.fusesource.jansi.Ansi;
import picocli.CommandLine.Command;
import picocli.CommandLine.Model.CommandSpec;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
import picocli.CommandLine.Spec;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import static org.fusesource.jansi.Ansi.ansi;
@Command(name = "lint", description = "Parse and validate Concord YAML files")
public class Lint implements Callable {
@Spec
private CommandSpec spec;
@Option(names = {"-h", "--help"}, usageHelp = true, description = "display the command's help message")
boolean helpRequested = false;
@Option(names = {"-v", "--verbose"}, description = "Verbose output")
boolean verbose = false;
@Parameters(arity = "0..1")
Path targetDir = Paths.get(System.getProperty("user.dir"));
@Override
public Integer call() throws Exception {
targetDir = targetDir.normalize().toAbsolutePath();
if (!Files.isDirectory(targetDir)) {
throw new IllegalArgumentException("Not a directory: " + targetDir);
}
ImportManager importManager = new NoopImportManager();
ProjectLoaderV1 v1 = new ProjectLoaderV1(importManager);
ProjectLoaderV2 v2 = new ProjectLoaderV2(importManager);
DelegatingProjectLoader loader = new DelegatingProjectLoader(Set.of(v1, v2));
ProcessDefinition pd = loader.loadProject(targetDir, new DummyImportsNormalizer(), verbose ? new CliImportsListener() : null).projectDefinition();
List lintResults = new ArrayList<>();
linters().forEach(l -> lintResults.addAll(l.apply(pd)));
if (!lintResults.isEmpty()) {
print(lintResults);
println();
}
println("Found:");
println(" imports: " + pd.imports().items().size());
println(" profiles: " + pd.profiles().size());
println(" flows: " + pd.flows().size());
println(" forms: " + pd.forms().size());
println(" triggers: " + pd.triggers().size());
println(" (not counting dynamically imported resources)");
println();
printStats(lintResults);
println();
boolean hasErrors = hasErrors(lintResults);
if (hasErrors) {
System.out.println(ansi().fgBrightRed().bold().a("INVALID").reset());
} else {
System.out.println(ansi().fgBrightGreen().bold().a("VALID").reset());
}
return hasErrors ? 10 : 0;
}
private List linters() {
return Arrays.asList(
new ExpressionLinter(verbose),
new TaskCallLinter(verbose)
);
}
private void print(List results) {
for (LintResult r : results) {
Ansi msg = ansi();
switch (r.getType()) {
case ERROR: {
ansi().fgBrightRed().a("ERROR:").reset();
break;
}
case WARNING: {
ansi().fgBrightYellow().a("WARN:").reset();
break;
}
default:
throw new IllegalArgumentException("Unsupported result type: " + r.getType());
}
SourceMap sm = r.getSourceMap();
if (sm != null) {
msg.a("@ [").a(sm.source()).a("] line: ").a(sm.line()).a(", col: ").a(sm.column());
}
msg.append("\n\t").append(r.getMessage());
println(msg);
println("------------------------------------------------------------");
}
}
private void printStats(List results) {
long errors = results.stream().filter(r -> r.getType() == Type.ERROR).count();
long warns = results.stream().filter(r -> r.getType() == Type.WARNING).count();
println("Result: " + errors + " error(s), " + warns + " warning(s)");
}
private void println(Object o) {
System.out.println(o);
}
private void println() {
System.out.println();
}
private static boolean hasErrors(List results) {
return results.stream().anyMatch(l -> l.getType() == Type.ERROR);
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/LocalCliRuntime.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2026 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.google.inject.Injector;
import com.walmartlabs.concord.cli.CliConfig.CliConfigContext;
import com.walmartlabs.concord.cli.runner.CliServicesModule;
import com.walmartlabs.concord.dependencymanager.DependencyManager;
import com.walmartlabs.concord.dependencymanager.DependencyManagerConfiguration;
import com.walmartlabs.concord.dependencymanager.DependencyManagerRepositories;
import com.walmartlabs.concord.imports.ImportsListener;
import com.walmartlabs.concord.imports.NoopImportManager;
import com.walmartlabs.concord.runtime.common.cfg.RunnerConfiguration;
import com.walmartlabs.concord.runtime.v2.NoopImportsNormalizer;
import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;
import com.walmartlabs.concord.runtime.v2.runner.InjectorFactory;
import com.walmartlabs.concord.runtime.v2.runner.guice.ProcessDependenciesModule;
import com.walmartlabs.concord.runtime.v2.sdk.ProcessConfiguration;
import com.walmartlabs.concord.runtime.v2.sdk.WorkingDirectory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
final class LocalCliRuntime {
static DependencyManager createDependencyManager(Path depsCacheDir) throws IOException {
var cfgFile = Paths.get(System.getProperty("user.home"), ".concord", "mvn.json");
if (Files.exists(cfgFile)) {
return new DependencyManager(DependencyManagerConfiguration.of(depsCacheDir, DependencyManagerRepositories.get(cfgFile)));
}
return new DependencyManager(DependencyManagerConfiguration.of(depsCacheDir));
}
static Injector createInjector(Path workDir,
RunnerConfiguration runnerCfg,
ProcessConfiguration processCfg,
CliConfigContext cliConfigContext,
Path defaultTaskVars,
DependencyManager dependencyManager,
Verbosity verbosity) {
return new InjectorFactory(new WorkingDirectory(workDir),
runnerCfg,
() -> processCfg,
new ProcessDependenciesModule(workDir, runnerCfg.dependencies(), processCfg.debug()),
new CliServicesModule(cliConfigContext, workDir, defaultTaskVars, dependencyManager, verbosity))
.create();
}
static void notifyProjectLoaded(Path workDir) throws Exception {
var loader = new ProjectLoaderV2(new NoopImportManager());
loader.load(workDir, new NoopImportsNormalizer(), ImportsListener.NOP_LISTENER);
}
private LocalCliRuntime() {
}
}
================================================
FILE: cli/src/main/java/com/walmartlabs/concord/cli/LocalFormInputs.java
================================================
package com.walmartlabs.concord.cli;
/*-
* *****
* Concord
* -----
* Copyright (C) 2017 - 2026 Walmart Inc.
* -----
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =====
*/
import com.walmartlabs.concord.forms.DefaultFormValidator;
import com.walmartlabs.concord.forms.DefaultFormValidatorLocale;
import com.walmartlabs.concord.forms.Form;
import com.walmartlabs.concord.forms.FormField;
import com.walmartlabs.concord.forms.FormFields;
import com.walmartlabs.concord.forms.FormUtils;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static com.walmartlabs.concord.forms.Constants.FORM_FILES;
final class LocalFormInputs {
private static final DefaultFormValidatorLocale LOCALE = new DefaultFormValidatorLocale();
private static final DefaultFormValidator VALIDATOR = new DefaultFormValidator(LOCALE);
static Converted convertAndValidate(Form form, Map rawValues) throws InputException {
return convertAndValidate(form, rawValues, false);
}
static Converted convertAndValidate(Form form, Map input, boolean unwrapFormName) throws InputException {
var rawValues = unwrapFormName ? unwrapFormValues(form, input) : input;
var tmpFiles = new LinkedHashMap();
var opened = new ArrayList();
try {
var convertedInput = prepareInput(form, rawValues, opened);
var converted = new LinkedHashMap<>(FormUtils.convert(LOCALE, form, convertedInput, tmpFiles));
var errors = VALIDATOR.validate(form, converted);
if (!errors.isEmpty()) {
cleanupTempFiles(tmpFiles);
throw new InputException(errors.stream().map(e -> e.error()).toList());
}
return new Converted(converted, tmpFiles);
} catch (FormUtils.ValidationException e) {
cleanupTempFiles(tmpFiles);
throw new InputException(List.of(e.getMessage()), e);
} catch (IOException e) {
cleanupTempFiles(tmpFiles);
throw new InputException(List.of(e.getMessage()), e);
} finally {
close(opened);
}
}
private static Map prepareInput(Form form, Map rawValues, List opened) throws IOException {
var convertedInput = new LinkedHashMap();
for (var field : form.fields()) {
var value = rawValues.get(field.name());
if (value == null) {
continue;
}
convertedInput.put(field.name(), toFormInput(value, field, opened));
}
return convertedInput;
}
private static Object toFormInput(Object value, FormField field, List opened) throws IOException {
if (!FormFields.FileField.TYPE.equals(field.type())) {
return value;
}
if (value instanceof Path path) {
var in = Files.newInputStream(path);
opened.add(in);
return in;
}
if (value instanceof Collection values) {
var result = new ArrayList<>();
for (var item : values) {
result.add(toFormInput(item, field, opened));
}
return result;
}
return value;
}
private static Map