Only run sentence splitting once, even if the user has manually added a redundant sentence splitter

Fix a couple broken tests - the changed ssplit behavior makes the tests work differently

Removing the SentencesAnnotation if a tokenizer is run twice will allow the sentence splitter to run again, although it should be pointed out that re-annotating a document is not a great idea in the first place

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/RequirementsCorrectSlowITest.java

@@ -119,27 +119,27 @@ private void testAnnotatorSequence(List<String> annotators) {
 
   @Test
   public void testDefaultPipeline() {
-    testAnnotatorSequence(Arrays.asList("tokenize", "ssplit", "pos", "lemma", "ner", "gender", "parse", "coref"));
+    testAnnotatorSequence(Arrays.asList("tokenize", "pos", "lemma", "ner", "gender", "parse", "coref"));
   }
 
   @Test
   public void testDepparsePipeline() {
-    testAnnotatorSequence(Arrays.asList("tokenize", "ssplit", "pos", "depparse"));
+    testAnnotatorSequence(Arrays.asList("tokenize", "pos", "depparse"));
   }
 
   @Test
   public void testQuotePipeline() {
-    testAnnotatorSequence(Arrays.asList("tokenize","ssplit","pos","lemma","ner","depparse","coref","quote"));
+    testAnnotatorSequence(Arrays.asList("tokenize","pos","lemma","ner","depparse","coref","quote"));
   }
 
-   @Test
-   public void testTrueCasePipeline() {
-     testAnnotatorSequence(Arrays.asList("tokenize","ssplit","pos","lemma","truecase"));
+  @Test
+  public void testTrueCasePipeline() {
+    testAnnotatorSequence(Arrays.asList("tokenize","pos","lemma","truecase"));
    }
 
   @Test
   public void testOpenIEPipeline() {
-    testAnnotatorSequence(Arrays.asList("tokenize","ssplit","pos","lemma","depparse","natlog","openie"));
+    testAnnotatorSequence(Arrays.asList("tokenize","pos","lemma","depparse","natlog","openie"));
   }
 
   @Test

src/edu/stanford/nlp/pipeline/TokenizerAnnotator.java

@@ -446,6 +446,10 @@ public void annotate(Annotation annotation) {
       throw new RuntimeException("Tokenizer unable to find text in annotation: " + annotation);
     }
 
+    // If the annotation was already processed before and already has
+    // a SentenceAnnotation.class, recreating the tokenization
+    // invalidates any existing sentence annotation
+    annotation.remove(CoreAnnotations.SentencesAnnotation.class);
     if (this.cleanxmlAnnotator != null) {
       this.cleanxmlAnnotator.annotate(annotation);
     }

src/edu/stanford/nlp/pipeline/WordsToSentencesAnnotator.java

@@ -35,6 +35,8 @@ public class WordsToSentencesAnnotator implements Annotator  {
 
   private final boolean countLineNumbers;
 
+  private boolean loggedExtraSplit = false;
+
   public WordsToSentencesAnnotator() {
     this(false);
   }
@@ -177,10 +179,18 @@ public void annotate(Annotation annotation) {
     if (VERBOSE) {
       log.info("Sentence splitting ... " + annotation);
     }
-    if ( ! annotation.containsKey(CoreAnnotations.TokensAnnotation.class)) {
+    if (!annotation.containsKey(CoreAnnotations.TokensAnnotation.class)) {
       throw new IllegalArgumentException("WordsToSentencesAnnotator: unable to find words/tokens in: " + annotation);
     }
 
+    if (annotation.containsKey(CoreAnnotations.SentencesAnnotation.class)) {
+      if (!loggedExtraSplit) {
+        log.error("Multiple WordsToSentencesAnnotator or other sentence splitters are operating on this document!");
+        loggedExtraSplit = true;
+      }
+      return;
+    }
+
     // get text and tokens from the document
     String text = annotation.get(CoreAnnotations.TextAnnotation.class);
     List<CoreLabel> tokens = annotation.get(CoreAnnotations.TokensAnnotation.class);

test/src/edu/stanford/nlp/pipeline/CleanXmlAnnotatorTest.java

@@ -37,14 +37,21 @@ public class CleanXmlAnnotatorTest {
   @Before
   public void setUp() throws Exception {
     synchronized(CleanXmlAnnotatorTest.class) {
+      // we create the TokenizerAnnotator without the ssplit so we can
+      // manually control the pieces
+      // another alternative would be to create TokenizerAnnotators
+      // with the CleanXML as part of it
       if (ptbInvertible == null) {
-        ptbInvertible =
-          new TokenizerAnnotator(false, "en", "invertible,ptb3Escaping=true");
+        Properties props = new Properties();
+        props.setProperty("tokenize.language", "en");
+        props.setProperty("tokenize.ssplit", "false");
+        ptbInvertible = new TokenizerAnnotator(false, props, "invertible,ptb3Escaping=true");
       }
       if (ptbNotInvertible == null) {
-        ptbNotInvertible =
-          new TokenizerAnnotator(false, "en",
-                                    "invertible=false,ptb3Escaping=true");
+        Properties props = new Properties();
+        props.setProperty("tokenize.language", "en");
+        props.setProperty("tokenize.ssplit", "false");
+        ptbNotInvertible = new TokenizerAnnotator(false, props, "invertible=false,ptb3Escaping=true");
       }
       if (cleanXmlAllTags == null) {
         cleanXmlAllTags = new CleanXmlAnnotator(".*", "", "", false);