added flag to support ./2 as list, small refactoring of Tokenizer

Author: raydac

src/main/java/com/igormaznitsa/prologparser/PrologParser.java

@@ -96,7 +96,6 @@ public abstract class PrologParser implements Iterable<PrologTerm>, Closeable {
   protected final ParserContext context;
   protected final int parserFlags;
   private final Tokenizer tokenizer;
-  private PrologTerm lastFoundTerm;
 
   public PrologParser(final Reader source, final ParserContext context) {
     this.context = context == null ? of(ParserContext.FLAG_NONE) : context;
@@ -157,30 +156,20 @@ public ParserContext getContext() {
   }
 
   public boolean hasNext() {
-    if (this.lastFoundTerm == null) {
-      final PrologTerm found = readBlock(OPERATORS_PHRASE);
-      if (found != null) {
-        final TokenizerResult endAtom = this.tokenizer.readNextToken();
-        if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
-          throw new PrologParserException("End operator is not found", this.tokenizer.getLine(), this.tokenizer.getPos());
-        }
-      }
-      this.lastFoundTerm = found;
-    }
-
-    return this.lastFoundTerm != null;
+    return this.tokenizer.peek() != null;
   }
 
   public PrologTerm next() {
-    try {
-      if (hasNext()) {
-        return this.lastFoundTerm;
-      } else {
-        throw new NoSuchElementException("No terms in source");
+    final PrologTerm found = readBlock(OPERATORS_PHRASE);
+    if (found == null) {
+      throw new NoSuchElementException("No terms in source");
+    } else {
+      final TokenizerResult endAtom = this.tokenizer.readNextToken();
+      if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
+        throw new PrologParserException("End operator is not found", this.tokenizer.getLine(), this.tokenizer.getPos());
       }
-    } finally {
-      this.lastFoundTerm = null;
     }
+    return found;
   }
 
   private PrologStruct readStruct(final PrologTerm functor) {
@@ -582,12 +571,12 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
       return null;
     } else {
       PrologTerm result = currentTreeItem.findRoot().convertToTermAndRelease(this);
-      if ((this.parserFlags & FLAG_DOT2_AS_LIST) != 0 
+      if ((this.parserFlags & FLAG_DOT2_AS_LIST) != 0
               && result.getType() == TermType.STRUCT
-              && result.getText().equals(".") 
+              && result.getText().equals(".")
               && result.getArity() == 2) {
-              final PrologStruct asStruct = (PrologStruct) result;
-              result = new PrologList(asStruct.getTermAt(0), asStruct.getTermAt(1));
+        final PrologStruct asStruct = (PrologStruct) result;
+        result = new PrologList(asStruct.getTermAt(0), asStruct.getTermAt(1));
       }
       return result;
     }

src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java

@@ -102,11 +102,11 @@ public static boolean isCharAllowedForRadix(final char chr, final int radix) {
     }
   }
 
-  public TokenizerResult getLastPushed() {
+  TokenizerResult getLastPushed() {
     return this.lastPushedTerm;
   }
 
-  public synchronized int readChar() throws IOException {
+  private synchronized int doReadChar() throws IOException {
     int ch;
     if (this.insideCharBuffer.isEmpty()) {
       ch = this.reader.read();
@@ -246,7 +246,7 @@ public void fixPosition() {
   private void skipUntilBlockCommentEnd() throws IOException {
     boolean starCharDetected = false;
     while (!Thread.currentThread().isInterrupted()) {
-      final int readChar = this.readChar();
+      final int readChar = this.doReadChar();
       if (readChar < 0 || (readChar == '/' && starCharDetected)) {
         break;
       } else {
@@ -257,7 +257,7 @@ private void skipUntilBlockCommentEnd() throws IOException {
 
   private void skipUntilNextString() throws IOException {
     while (!Thread.currentThread().isInterrupted()) {
-      final int readChar = this.readChar();
+      final int readChar = this.doReadChar();
       if (readChar < 0 || readChar == '\n') {
         break;
       }
@@ -275,7 +275,7 @@ public TokenizerResult pop() {
   /**
    * Read next token
    *
-   * @return next token or null if not found or thread interruption detected
+   * @return next token or null if not found or stream ended
    */
   public TokenizerResult readNextToken() {
 
@@ -306,7 +306,7 @@ public TokenizerResult readNextToken() {
 
     try {
       while (!Thread.currentThread().isInterrupted()) {
-        final int readChar = this.readChar();
+        final int readChar = this.doReadChar();
 
         if (readChar < 0) {
           switch (state) {

src/test/java/com/igormaznitsa/prologparser/IntegrationTest.java

@@ -46,10 +46,10 @@ public class IntegrationTest extends AbstractIntegrationTest {
 
   @Test
   public void testReadingClausesTokensAndCharsFromSameParser() throws IOException {
-    final PrologParser parser = parseEd("some(a). alone. next(b).c");
+    final PrologParser parser = parseEd("lsome(a). alone. next(b).c");
     assertTrue(parser.hasNext());
     final PrologTerm term1 = parser.next();
-    assertEquals("some(a)", term1.toString());
+    assertEquals("lsome(a)", term1.toString());
     assertTrue(parser.hasNext());
     final TokenizerResult token1 = parser.getInternalTokenizer().readNextToken();
     assertEquals("alone", token1.getResult().toString());
@@ -59,9 +59,8 @@ public void testReadingClausesTokensAndCharsFromSameParser() throws IOException
     assertTrue(parser.hasNext());
     final PrologTerm term2 = parser.next();
     assertEquals("next(b)", term2.toString());
-    assertEquals('c', parser.getInternalTokenizer().readChar());
-    assertFalse(parser.hasNext());
-    assertEquals(-1, parser.getInternalTokenizer().readChar());
+    assertTrue(parser.hasNext());
+    assertThrows(PrologParserException.class, () -> parser.next());
   }
 
   @Test
@@ -74,18 +73,18 @@ public void testEmptyAndNonClause() {
 
   @Test
   public void testDot2StructAsList() {
-     assertEquals("'.'(a, [])", parseGen("'.'(a,[]).").next().toString());
-     assertEquals("'.'(a, X)", parseGen("'.'(a,X).").next().toString());
-     assertEquals("'.'(a, b)", parseGen("'.'(a,b).").next().toString());
-
-     assertEquals("[a]", parseIso("'.'(a,[]).").next().toString());
-     assertEquals("[a|X]", parseIso("'.'(a,X).").next().toString());
-     assertEquals("[a|b]", parseIso("'.'(a,b).").next().toString());
-  
-     assertEquals("[a, b, c]", parseIso("'.'(a,'.'(b,'.'(c,[]))).").next().toString());
-     assertEquals("[a, b, c|X]", parseIso("'.'(a,'.'(b,'.'(c,X))).").next().toString());
-  }
-  
+    assertEquals("'.'(a, [])", parseGen("'.'(a,[]).").next().toString());
+    assertEquals("'.'(a, X)", parseGen("'.'(a,X).").next().toString());
+    assertEquals("'.'(a, b)", parseGen("'.'(a,b).").next().toString());
+
+    assertEquals("[a]", parseIso("'.'(a,[]).").next().toString());
+    assertEquals("[a|X]", parseIso("'.'(a,X).").next().toString());
+    assertEquals("[a|b]", parseIso("'.'(a,b).").next().toString());
+
+    assertEquals("[a, b, c]", parseIso("'.'(a,'.'(b,'.'(c,[]))).").next().toString());
+    assertEquals("[a, b, c|X]", parseIso("'.'(a,'.'(b,'.'(c,X))).").next().toString());
+  }
+
   @Test
   public void testSwiCpl() {
     assertEquals("X in 5 .. 10 , 5 , Y #=< X + -1 , 6 , Y in 4 .. 8", parseCpl("X in 5..10,5,Y#=<X+ -1,6,Y in 4..8.").next().toString());