Merge pull request #360 from code-dot-org/molly/validation-system-out-tracker

Add validation api for checking system out messages

Author: molly-moen

org-code-javabuilder/lib/src/main/java/org/code/javabuilder/ValidationRunner.java

@@ -6,6 +6,7 @@
 import org.code.javabuilder.util.ProjectLoadUtils;
 import org.code.protocol.*;
 import org.code.validation.support.NeighborhoodTracker;
+import org.code.validation.support.SystemOutTracker;
 import org.code.validation.support.ValidationProtocol;
 
 public class ValidationRunner extends BaseTestRunner {
@@ -49,6 +50,6 @@ private void setUpForValidation(URLClassLoader urlClassLoader) throws UserInitia
     JavabuilderContext.getInstance()
         .register(
             ValidationProtocol.class,
-            new ValidationProtocol(mainMethod, new NeighborhoodTracker()));
+            new ValidationProtocol(mainMethod, new NeighborhoodTracker(), new SystemOutTracker()));
   }
 }

org-code-javabuilder/validation/src/main/java/org/code/validation/SystemOutTestRunner.java

@@ -0,0 +1,16 @@
+package org.code.validation;
+
+import java.util.List;
+import org.code.protocol.JavabuilderContext;
+import org.code.validation.support.ValidationProtocol;
+
+public class SystemOutTestRunner {
+  // Run the main method of the user's program and return a list
+  // of all system.out messages in order.
+  public static List<String> run() {
+    ValidationProtocol protocolInstance =
+        (ValidationProtocol) JavabuilderContext.getInstance().get(ValidationProtocol.class);
+    protocolInstance.invokeMainMethod();
+    return protocolInstance.getSystemOutMessages();
+  }
+}

org-code-javabuilder/validation/src/main/java/org/code/validation/support/NeighborhoodTracker.java

@@ -18,7 +18,7 @@
 import org.code.validation.Position;
 
 public class NeighborhoodTracker {
-  private final Map<String, PainterTracker> painterTrackers;
+  private Map<String, PainterTracker> painterTrackers;
   private String[][] neighborhoodState;
   private boolean isInitialized;
 
@@ -83,6 +83,11 @@ public void trackEvent(ClientMessage message) {
     }
   }
 
+  public void reset() {
+    this.painterTrackers = new HashMap<>();
+    this.isInitialized = false;
+  }
+
   private void initializeGrid() {
     World world = (World) JavabuilderContext.getInstance().get(World.class);
     final int gridSize = world.getGrid().getSize();

org-code-javabuilder/validation/src/main/java/org/code/validation/support/SystemOutTracker.java

@@ -0,0 +1,36 @@
+package org.code.validation.support;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.code.protocol.ClientMessage;
+import org.code.protocol.ClientMessageType;
+
+public class SystemOutTracker {
+  private List<String> messages;
+
+  public SystemOutTracker() {
+    this.messages = new ArrayList<>();
+  }
+
+  public List<String> getSystemOutMessages() {
+    return this.messages;
+  }
+
+  // Track a single client message event. It will only be tracked if it is a system out message
+  // that is not just a newline, otherwise it will be ignored.
+  public void trackEvent(ClientMessage message) {
+    if (message.getType() != ClientMessageType.SYSTEM_OUT) {
+      return;
+    }
+    String value = message.getValue();
+    // Ignore new lines as those are indications of a println vs a print.
+    // For validation we don't care which is used.
+    if (!value.equals("\n")) {
+      this.messages.add(message.getValue());
+    }
+  }
+
+  public void reset() {
+    this.messages = new ArrayList<>();
+  }
+}

org-code-javabuilder/validation/src/main/java/org/code/validation/support/UserTestOutputAdapter.java

@@ -29,16 +29,16 @@ public void sendMessage(ClientMessage message) {
           delegateOutputAdapter.sendMessage(message);
         }
         break;
-      case NEIGHBORHOOD:
-        if (this.isValidation) {
-          ValidationProtocol protocolInstance =
-              (ValidationProtocol) JavabuilderContext.getInstance().get(ValidationProtocol.class);
-          protocolInstance.trackEvent(message);
-        }
-        break;
       default:
         break;
     }
+    // If this is validation, send all messages to the validation protocol, which will decide
+    // whether or not to track them.
+    if (this.isValidation) {
+      ValidationProtocol protocolInstance =
+          (ValidationProtocol) JavabuilderContext.getInstance().get(ValidationProtocol.class);
+      protocolInstance.trackEvent(message);
+    }
   }
 
   public void setIsValidation(boolean isValidation) {

org-code-javabuilder/validation/src/main/java/org/code/validation/support/ValidationProtocol.java

@@ -2,31 +2,44 @@
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.List;
 import org.code.protocol.ClientMessage;
 import org.code.protocol.JavabuilderSharedObject;
 import org.code.validation.NeighborhoodLog;
 
 public class ValidationProtocol extends JavabuilderSharedObject {
   private final Method mainMethod;
   private final NeighborhoodTracker neighborhoodTracker;
+  private final SystemOutTracker systemOutTracker;
 
-  public ValidationProtocol(Method mainMethod, NeighborhoodTracker neighborhoodTracker) {
+  public ValidationProtocol(
+      Method mainMethod,
+      NeighborhoodTracker neighborhoodTracker,
+      SystemOutTracker systemOutTracker) {
     this.mainMethod = mainMethod;
     this.neighborhoodTracker = neighborhoodTracker;
+    this.systemOutTracker = systemOutTracker;
   }
 
   public NeighborhoodLog getNeighborhoodLog() {
     return this.neighborhoodTracker.getNeighborhoodLog();
   }
 
+  public List<String> getSystemOutMessages() {
+    return this.systemOutTracker.getSystemOutMessages();
+  }
+
   public void trackEvent(ClientMessage message) {
     this.neighborhoodTracker.trackEvent(message);
+    this.systemOutTracker.trackEvent(message);
   }
 
   public void invokeMainMethod() {
     if (this.mainMethod == null) {
       throw new ValidationRuntimeException(ExceptionKey.NO_MAIN_METHOD_VALIDATION);
     }
+    this.neighborhoodTracker.reset();
+    this.systemOutTracker.reset();
     try {
       this.mainMethod.invoke(null, new Object[] {null});
     } catch (IllegalAccessException e) {

org-code-javabuilder/validation/src/test/java/org/code/validation/support/NeighborhoodTrackerTest.java

@@ -47,7 +47,7 @@ public void testInitializeMessageCreatesNewPainterTracker() {
 
   @Test
   public void testUpdatesPainterTrackerWithPainterMessages() {
-    final String id = "id";
+    final String id = "id1";
     // Initialize
     unitUnderTest.trackEvent(createInitEvent(id, Direction.EAST, 5, 10, 20));
 
@@ -66,7 +66,7 @@ public void testUpdatesPainterTrackerWithPainterMessages() {
 
     final PainterLog painterLog = unitUnderTest.getNeighborhoodLog().getPainterLogs()[0];
 
-    assertEquals(ID, painterLog.getPainterId());
+    assertEquals(id, painterLog.getPainterId());
     assertEquals(4, painterLog.getEndingPosition().getX());
     assertEquals(10, painterLog.getEndingPosition().getY());
     assertEquals(21, painterLog.getEndingPaintCount());
@@ -132,7 +132,7 @@ public void testSelectsCorrectPainterTrackerById() {
 
   @Test
   public void testUpdatesPainterLogWithNonAnimatedMoves() {
-    final String id = "id";
+    final String id = "id1";
     // Initialize
     unitUnderTest.trackEvent(createInitEvent(id, Direction.EAST, 5, 10, 20));
 
@@ -144,11 +144,38 @@ public void testUpdatesPainterLogWithNonAnimatedMoves() {
 
     final PainterLog painterLog = unitUnderTest.getNeighborhoodLog().getPainterLogs()[0];
 
-    assertEquals(ID, painterLog.getPainterId());
+    assertEquals(id, painterLog.getPainterId());
     assertTrue(painterLog.didActionOnce(NeighborhoodActionType.IS_ON_BUCKET));
     assertFalse(painterLog.didActionAtLeast(NeighborhoodActionType.CAN_MOVE, 1));
   }
 
+  @Test
+  public void resetsSuccessfully() {
+    final String id = "id1";
+    // Initialize
+    unitUnderTest.trackEvent(createInitEvent(id, Direction.EAST, 5, 10, 20));
+    // create paint event
+    final HashMap<String, String> paintDetails = new HashMap<>();
+    paintDetails.put(COLOR, "orange");
+    paintDetails.put(ID, id);
+    unitUnderTest.trackEvent(
+        new NeighborhoodSignalMessage(NeighborhoodSignalKey.PAINT, paintDetails));
+    unitUnderTest.reset();
+    // re-initialize
+    unitUnderTest.trackEvent(createInitEvent(id, Direction.EAST, 5, 10, 20));
+    // create move event
+    final HashMap<String, String> moveDetails = new HashMap<>();
+    moveDetails.put(DIRECTION, Direction.WEST.getDirectionString());
+    moveDetails.put(ID, id);
+    unitUnderTest.trackEvent(
+        new NeighborhoodSignalMessage(NeighborhoodSignalKey.MOVE, moveDetails));
+
+    final PainterLog painterLog = unitUnderTest.getNeighborhoodLog().getPainterLogs()[0];
+
+    assertEquals(false, painterLog.didActionOnce(NeighborhoodActionType.PAINT));
+    assertEquals(true, painterLog.didActionOnce(NeighborhoodActionType.MOVE));
+  }
+
   private NeighborhoodSignalMessage createInitEvent(
       String id, Direction direction, int x, int y, int paint) {
     final HashMap<String, String> details = new HashMap<>();

org-code-javabuilder/validation/src/test/java/org/code/validation/support/SystemOutTrackerTest.java

@@ -0,0 +1,44 @@
+package org.code.validation.support;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import org.code.protocol.ClientMessageType;
+import org.code.validation.ClientMessageHelper;
+import org.junit.jupiter.api.Test;
+
+public class SystemOutTrackerTest {
+  private SystemOutTracker unitUnderTest;
+
+  @Test
+  public void tracksAllNonNewlineMessages() {
+    unitUnderTest = new SystemOutTracker();
+    // should be tracked
+    unitUnderTest.trackEvent(new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "hello"));
+    // should be ignored
+    unitUnderTest.trackEvent(new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "\n"));
+    // should be tracked
+    unitUnderTest.trackEvent(new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "hello world"));
+
+    List<String> messages = unitUnderTest.getSystemOutMessages();
+    assertEquals(2, messages.size());
+    assertEquals("hello world", messages.get(1));
+  }
+
+  @Test
+  public void resetsSuccessfully() {
+    unitUnderTest = new SystemOutTracker();
+    unitUnderTest.trackEvent(new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "hello"));
+    // should reset message list
+    unitUnderTest.reset();
+    unitUnderTest.trackEvent(
+        new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "a new message"));
+    unitUnderTest.trackEvent(new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "hello world"));
+    unitUnderTest.trackEvent(
+        new ClientMessageHelper(ClientMessageType.SYSTEM_OUT, "a third message"));
+
+    List<String> messages = unitUnderTest.getSystemOutMessages();
+    assertEquals(3, messages.size());
+    assertEquals("a new message", messages.get(0));
+  }
+}

org-code-javabuilder/validation/src/test/java/org/code/validation/support/UserTestOutputAdapterTest.java

@@ -15,6 +15,8 @@ public class UserTestOutputAdapterTest {
   @BeforeEach
   public void setUp() {
     testOutputAdapter = new UserTestOutputAdapter(delegateOutputAdapter);
+    JavabuilderContext.getInstance()
+        .register(ValidationProtocol.class, mock(ValidationProtocol.class));
   }
 
   @Test