Skip to content

Commit b3d7892

Browse files
raydacraydac
committed
improved tokenizer
1 parent a91a6ec commit b3d7892

File tree

2 files changed

+31
-30
lines changed

2 files changed

+31
-30
lines changed

‎src/main/java/com/igormaznitsa/prologparser/PrologParser.java‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
* specific language governing permissions and limitations
1919
* under the License.
2020
*/
21-
2221
package com.igormaznitsa.prologparser;
2322

2423
import com.igormaznitsa.prologparser.exceptions.CriticalUnexpectedError;
@@ -136,6 +135,10 @@ public final TreeItem get() {
136135
};
137136
}
138137

138+
public Tokenizer getInternalTokenizer() {
139+
return this.tokenizer;
140+
}
141+
139142
public static Op findBaseMetaOperator(final String text, final OpAssoc type) {
140143
if (text.length() != 1) {
141144
return null;
@@ -306,12 +309,12 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
306309
rightPart = readBlock(OPERATORS_END_LIST);
307310

308311
if (rightPart != null
309-
&& rightPart.getType() == TermType.STRUCT
310-
&& rightPart.getFunctor().getText().equals(OPERATOR_VERTICALBAR.getText())) {
312+
&& rightPart.getType() == TermType.STRUCT
313+
&& rightPart.getFunctor().getText().equals(OPERATOR_VERTICALBAR.getText())) {
311314
throw new PrologParserException(
312-
"Duplicated list tail definition",
313-
tokenizer.getLastTokenLine(),
314-
tokenizer.getLastTokenPos(), null);
315+
"Duplicated list tail definition",
316+
tokenizer.getLastTokenLine(),
317+
tokenizer.getLastTokenPos(), null);
315318
}
316319

317320
final TokenizerResult nextAtomTwo = tokenizer.readNextToken();
@@ -439,7 +442,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
439442
// we didn't get any operator for our criteria, so throw
440443
// an exception
441444
throw new PrologParserException("Operator clash detected [" + readAtomContainer.getResult().getText() + ']',
442-
readAtomContainer.getLine(), readAtomContainer.getPos());
445+
readAtomContainer.getLine(), readAtomContainer.getPos());
443446
}
444447
// we have found needed operator so get its precedence
445448
readAtomPrecedence = readAtom.getPrecedence();
@@ -478,7 +481,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
478481
emptyCurly = true;
479482
} else {
480483
throw new PrologParserException("Illegal start of term",
481-
readAtomContainer.getLine(), readAtomContainer.getPos());
484+
readAtomContainer.getLine(), readAtomContainer.getPos());
482485
}
483486
}
484487

@@ -487,7 +490,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
487490
} else {
488491
readAtom.setLine(readAtomContainer.getLine());
489492
readAtom.setPos(readAtomContainer.getPos());
490-
readAtom = new PrologStruct(onlyCharCode == '{' ? Op.VIRTUAL_OPERATOR_CURLY_BLOCK : Op.VIRTUAL_OPERATOR_BLOCK, new PrologTerm[] {readAtom}, readAtomContainer.getLine(), readAtomContainer.getPos());
493+
readAtom = new PrologStruct(onlyCharCode == '{' ? Op.VIRTUAL_OPERATOR_CURLY_BLOCK : Op.VIRTUAL_OPERATOR_BLOCK, new PrologTerm[]{readAtom}, readAtomContainer.getLine(), readAtomContainer.getPos());
491494
}
492495

493496
final TokenizerResult token = this.tokenizer.readNextToken();
@@ -529,19 +532,17 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
529532
final int nextTokenStrPosition = nextToken.getPos();
530533

531534
// it is a structure
532-
if (
533-
readAtom.getType() == TermType.ATOM
535+
if (readAtom.getType() == TermType.ATOM
534536
|| (readAtom.getType() == TermType.VAR
535-
&& (this.parserFlags & FLAG_VAR_AS_FUNCTOR) != 0)
536-
) {
537+
&& (this.parserFlags & FLAG_VAR_AS_FUNCTOR) != 0)) {
537538

538539
final PrologTerm prev = readAtom;
539540
readAtom = readStruct(readAtom);
540541
if (readAtom == null) {
541542
// we have met the empty brackets
542543
if ((this.parserFlags & FLAG_ZERO_STRUCT) == 0) {
543544
throw new PrologParserException("Empty structure is not allowed",
544-
nextTokenLineNumber, nextTokenStrPosition);
545+
nextTokenLineNumber, nextTokenStrPosition);
545546
} else {
546547
final TokenizerResult pushed = this.tokenizer.pop();
547548
if (pushed.getResult() == OPERATOR_RIGHTBRACKET) {
@@ -555,7 +556,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
555556
tokenizer.push(nextToken);
556557
nextToken = null;
557558
throw new PrologParserException("You must have an atom as the structure functor",
558-
nextTokenLineNumber, nextTokenStrPosition);
559+
nextTokenLineNumber, nextTokenStrPosition);
559560
}
560561
} else {
561562
// push back the next atom
@@ -571,8 +572,8 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
571572
}
572573

573574
final TreeItem readAtomTreeItem = this.treeItemPool.find().setData(readAtom,
574-
readAtomContainer.getLine(),
575-
readAtomContainer.getPos());
575+
readAtomContainer.getLine(),
576+
readAtomContainer.getPos());
576577

577578
if (currentTreeItem == null) {
578579
// it's first
@@ -619,21 +620,21 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
619620
if (readAtomTreeItem.getType() != TermType.OPERATOR && currentTreeItem.getRightBranch() != null) {
620621
// it's a ground atom and its right branch is not empty
621622
throw new PrologParserException(
622-
"There is no any operator before the atom",
623-
readAtomContainer.getLine(),
624-
readAtomContainer.getPos());
623+
"There is no any operator before the atom",
624+
readAtomContainer.getLine(),
625+
readAtomContainer.getPos());
625626
}
626627
// make it as right
627628
currentTreeItem = currentTreeItem.makeAsRightBranch(readAtomTreeItem);
628629
}
629630
} else {
630631
// check that it is an operator
631632
if (currentTreeItem.getType() != TermType.OPERATOR
632-
&& readAtomTreeItem.getType() != TermType.OPERATOR) {
633+
&& readAtomTreeItem.getType() != TermType.OPERATOR) {
633634
throw new PrologParserException(
634-
"There must be an operator between atoms or structures",
635-
readAtomContainer.getLine(),
636-
readAtomContainer.getPos());
635+
"There must be an operator between atoms or structures",
636+
readAtomContainer.getLine(),
637+
readAtomContainer.getPos());
637638
}
638639

639640
// make it as left branch
@@ -670,10 +671,10 @@ public PrologTerm next() {
670671

671672
public Stream<PrologTerm> stream() {
672673
return StreamSupport.stream(
673-
Spliterators.spliteratorUnknownSize(
674-
this.iterator(),
675-
Spliterator.ORDERED | Spliterator.IMMUTABLE | Spliterator.NONNULL),
676-
false
674+
Spliterators.spliteratorUnknownSize(
675+
this.iterator(),
676+
Spliterator.ORDERED | Spliterator.IMMUTABLE | Spliterator.NONNULL),
677+
false
677678
);
678679
}
679680

‎src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
import static com.igormaznitsa.prologparser.utils.StringUtils.isCharAllowedForUnquotedAtom;
4646

4747
/**
48-
* Internal tokenizer to gen next token from reader.
48+
* Internal tokenizer to extract next token from reader.
4949
*/
5050
public final class Tokenizer {
5151

@@ -117,7 +117,7 @@ public TokenizerResult getLastPushed() {
117117
return this.lastPushedTerm;
118118
}
119119

120-
public int readChar() throws IOException {
120+
public synchronized int readChar() throws IOException {
121121
int ch;
122122
if (this.insideCharBuffer.isEmpty()) {
123123
ch = this.reader.read();

0 commit comments

Comments
 (0)