force refresh of cluster agents every hour

Author: claykirk

build.gradle

@@ -19,7 +19,7 @@ apply from: "https://raw.githubusercontent.com/gocd/gocd-plugin-gradle-task-help
 
 gocdPlugin {
   id = 'cd.go.contrib.elastic-agent.docker'
-  pluginVersion = '3.1.0'
+  pluginVersion = '3.1.1'
   goCdVersion = '19.3.0'
   name = 'Docker Elastic Agent Plugin'
   description = 'Docker Based Elastic Agent Plugins for GoCD'

src/main/java/cd/go/contrib/elasticagents/docker/DockerContainers.java

@@ -30,8 +30,8 @@
 import org.joda.time.Period;
 
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Semaphore;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static cd.go.contrib.elasticagents.docker.DockerPlugin.LOG;
@@ -40,11 +40,19 @@
 public class DockerContainers implements AgentInstances<DockerContainer> {
     private final Map<String, DockerContainer> instances = new ConcurrentHashMap<>();
     private List<JobIdentifier> jobsWaitingForAgentCreation = new ArrayList<>();
-    private boolean refreshed;
+    private AtomicBoolean refreshed = new AtomicBoolean(false);
+    private final int FORCE_REFRESH_TIMEOUT_MINUTES = 60;
+
     public Clock clock = Clock.DEFAULT;
 
     final Semaphore semaphore = new Semaphore(0, true);
 
+    private final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
+
+    public DockerContainers() {
+        scheduleForceRefresh();
+    }
+
     @Override
     public DockerContainer create(CreateAgentRequest request, PluginRequest pluginRequest, ConsoleLogAppender consoleLogAppender) throws Exception {
         LOG.info(String.format("[Create Agent] Processing create agent request for %s", request.jobIdentifier()));
@@ -77,16 +85,26 @@ public DockerContainer create(CreateAgentRequest request, PluginRequest pluginRe
         }
     }
 
+    private void scheduleForceRefresh() {
+        timerService.scheduleAtFixedRate(this::forceNextRefresh, 0, FORCE_REFRESH_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    }
+
     private void doWithLockOnSemaphore(Runnable runnable) {
         synchronized (semaphore) {
             runnable.run();
         }
     }
 
+    protected void forceNextRefresh() {
+        refreshed.set(false);
+    }
+
     @Override
     public void terminate(String agentId, PluginSettings settings) throws Exception {
+        LOG.info("(claykirk) -> terminating agent " + agentId);
         DockerContainer instance = instances.get(agentId);
         if (instance != null) {
+            LOG.info("(claykirk) -> found instance " + instance.name() + " to terminate.");
             instance.terminate(docker(settings));
         } else {
             LOG.warn("Requested to terminate an instance that does not exist " + agentId);
@@ -120,29 +138,36 @@ public void terminateUnregisteredInstances(PluginSettings settings, Agents agent
     @Override
     public Agents instancesCreatedAfterTimeout(PluginSettings settings, Agents agents) {
         ArrayList<Agent> oldAgents = new ArrayList<>();
+        LOG.info("(claykirk) Looking to retire agents from list: {}", String.join(", ", agents.agentIds()));
         for (Agent agent : agents.agents()) {
             DockerContainer instance = instances.get(agent.elasticAgentId());
             if (instance == null) {
+                LOG.info("(claykirk) couldn't find agent with elastic id: {}", agent.elasticAgentId());
                 continue;
             }
 
+            LOG.info("(claykirk) instance: {} has creation time: {}", agent.elasticAgentId(), instance.createdAt());
             if (clock.now().isAfter(instance.createdAt().plus(settings.getAutoRegisterPeriod()))) {
+                LOG.info("(claykirk) agent with id: {} is out of date", agent.elasticAgentId());
                 oldAgents.add(agent);
+            } else {
+                LOG.info("(claykirk) agent with id: {} is not out of date", agent.elasticAgentId());
             }
         }
 
+        LOG.info("(claykirk) -> instances created after timeout: {}", oldAgents.stream().map(Agent::elasticAgentId).collect(Collectors.joining(", ")));
         return new Agents(oldAgents);
     }
 
     @Override
     public void refreshAll(ClusterProfileProperties clusterProfileProperties) throws Exception {
-        if (!refreshed) {
+        if (!refreshed.get()){
             DockerClient docker = docker(clusterProfileProperties);
             List<Container> containers = docker.listContainers(DockerClient.ListContainersParam.withLabel(Constants.CREATED_BY_LABEL_KEY, Constants.PLUGIN_ID));
             for (Container container : containers) {
                 register(DockerContainer.fromContainerInfo(docker.inspectContainer(container.id())));
             }
-            refreshed = true;
+            refreshed.set(true);
         }
     }
 
@@ -219,4 +244,9 @@ public Optional<DockerContainer> find(JobIdentifier jobIdentifier) {
     protected boolean isEmpty() {
         return instances.isEmpty();
     }
+
+
+    public String   getInstanceIds() {
+        return String.join(", ", instances.keySet());
+    }
 }

src/test/java/cd/go/contrib/elasticagents/docker/DockerContainersTest.java

@@ -231,4 +231,20 @@ public void shouldGetAgentStatusReportUsingDockerContainer() throws Exception {
         assertThat(agentStatusReport.getElasticAgentId(), is(container.name()));
         assertThat(agentStatusReport.getJobIdentifier(), is(request.jobIdentifier()));
     }
+
+    @Test
+    public void shouldForceRefreshAgentInstancesAfterTimeout() throws Exception {
+
+        DockerContainers dockerContainers = new DockerContainers();
+        ClusterProfileProperties profileProperties = createClusterProfiles();
+        dockerContainers.refreshAll(profileProperties);
+
+        DockerContainer container = DockerContainer.create(request, clusterProfile, docker, consoleLogAppender);
+        containers.add(container.name());
+
+        dockerContainers.forceNextRefresh();
+
+        dockerContainers.refreshAll(profileProperties);
+        assertEquals(dockerContainers.find(container.name()), container);
+    }
 }