Initial API for Scanner. (#12)

Provides an interface for _lexing_ PartiQL.  Traditionally we would
factor this as the low-level interface for the parser, but with the PEG
based implementation, we just surface the relevant parts of the parser
as a rule to parse with.

This commit helps explore how to effectively integrate with the somewhat
low-level APIs for parsing that Pest provides.

* Adds `scanner` module.
* Adds `Scanner` trait to the prelude.
* Makes `PartiQLParser` public to crate.
* Refactors `LineAndColumn` as a tuple for `Position::At`.
  - Adds this to the `prelude`.
* Changes entry point for PEG to `Query` and added `Scanner` as the
  entry point rules for implementing the `Scanner` API.
* Adds `PairsExt`/`PairExt` trait/impl to add utility methods for working
  with Pest `Pairs`/`Pair`.
* Adds `LineAndColumn::position_from` and cleans up some doc/doc tests.
* Adds `TryFrom` and `TryInto` into the prelude.

Resolves #13.

Author: almann

partiql-parser/src/lib.rs

@@ -2,10 +2,54 @@
 
 //! Provides a parser for the [PartiQL][partiql] query language.
 //!
+//! # Usage
+//!
+//! An API to interact with PartiQL tokens is the [`mod@scanner`] module.
+//! The [`scanner()`] function creates a [`Scanner`](scanner::Scanner) instance
+//! that one can use to parse tokens incrementally from some input slice.
+//!
+//! ```
+//! use partiql_parser::prelude::*;
+//! use partiql_parser::scanner;
+//!
+//! fn main() -> ParserResult<()> {
+//!     use partiql_parser::scanner::Content::*;
+//!
+//!     let mut scanner = scanner("SELECT FROM");
+//!     let first = scanner.next_token()?;
+//!
+//!     // get the parsed variant of the token
+//!     match first.content() {
+//!         Keyword(kw) => assert_eq!("SELECT", kw),
+//!     }
+//!     // the entire text of a token can be fetched--which looks the roughly the
+//!     // same for a keyword.
+//!     assert_eq!("SELECT", first.text());
+//!     
+//!     let second = scanner.next_token()?;
+//!     // get the parsed variant of the token
+//!     match second.content() {
+//!         Keyword(kw) => assert_eq!("FROM", kw),
+//!     }
+//!     // the other thing we can do is get line/column information from a token
+//!     assert_eq!(LineAndColumn::try_at(1, 8)?, second.start());
+//!     assert_eq!(LineAndColumn::try_at(1, 12)?, second.end());
+//!
+//!     // this API is built on immutable slices, so we can restart scanning from any token
+//!     scanner = first.into();
+//!     let second_again = scanner.next_token()?;
+//!     assert_eq!(second, second_again);
+//!     
+//!     Ok(())
+//! }
+//! ```
+//!
 //! [partiql]: https://partiql.org
 
 mod peg;
 pub mod prelude;
 pub mod result;
+pub mod scanner;
 
 pub use peg::recognize_partiql;
+pub use scanner::scanner;

partiql-parser/src/partiql.pest

@@ -1,9 +1,15 @@
 WHITESPACE = _{ " " | "\t" | "\x0B" | "\x0C" | "\r" | "\n" }
 
-// really basic rules to just detect a sequence of keywords
-// two dip our toes in Pest
+// TODO implement a full grammar, this is a very primitive version to start
+//      working with Pest and its APIs.
 
-Keywords = _{ SOI ~ Keyword+ ~ EOI}
+// Entry point for full query parsing
+Query = _{ SOI ~ Keyword+ ~ EOI}
+
+// Entry point for query "scanning"
+// Note that this is factored this way to support an iteration style API
+// where we can call back into this rule on subsequent input.
+Scanner = _{ SOI ~ Keyword }
 
 Keyword = { AllKeywords }
 

partiql-parser/src/peg.rs

@@ -4,12 +4,63 @@
 //! can be exported for users to consume.
 
 use crate::prelude::*;
-use pest::Parser;
+use crate::result::syntax_error;
+use pest::iterators::{Pair, Pairs};
+use pest::{Parser, RuleType};
 use pest_derive::Parser;
 
 #[derive(Parser)]
 #[grammar = "partiql.pest"]
-struct PartiQLParser;
+pub(crate) struct PartiQLParser;
+
+/// Extension methods for working with [`Pairs`].
+pub(crate) trait PairsExt<'val, R: RuleType> {
+    /// Consumes a [`Pairs`] as a singleton, returning an error if there are less or more than
+    /// one [`Pair`].
+    fn exactly_one(self) -> ParserResult<Pair<'val, R>>;
+}
+
+impl<'val, R: RuleType> PairsExt<'val, R> for Pairs<'val, R> {
+    fn exactly_one(mut self) -> ParserResult<Pair<'val, R>> {
+        match self.next() {
+            Some(pair) => {
+                // make sure there isn't something more...
+                if let Some(other_pair) = self.next() {
+                    syntax_error(
+                        format!("Expected one token pair, got: {:?}, {:?}", pair, other_pair),
+                        pair.start()?.into(),
+                    )?;
+                }
+                Ok(pair)
+            }
+            None => syntax_error(
+                "Expected at one token pair, got nothing!",
+                Position::Unknown,
+            ),
+        }
+    }
+}
+
+/// Extension methods for working with [`Pair`].
+pub(crate) trait PairExt<'val, R: RuleType> {
+    /// Translates the start position of the [`Pair`] into a [`LineAndColumn`].
+    fn start(&self) -> ParserResult<LineAndColumn>;
+
+    /// Translates the end position of the [`Pair`] into a [`LineAndColumn`].
+    fn end(&self) -> ParserResult<LineAndColumn>;
+}
+
+impl<'val, R: RuleType> PairExt<'val, R> for Pair<'val, R> {
+    #[inline]
+    fn start(&self) -> ParserResult<LineAndColumn> {
+        self.as_span().start_pos().line_col().try_into()
+    }
+
+    #[inline]
+    fn end(&self) -> ParserResult<LineAndColumn> {
+        self.as_span().end_pos().line_col().try_into()
+    }
+}
 
 /// Recognizer for PartiQL queries.
 ///
@@ -18,7 +69,7 @@ struct PartiQLParser;
 ///
 /// This API will be replaced with one that produces an AST in the future.
 pub fn recognize_partiql(input: &str) -> ParserResult<()> {
-    PartiQLParser::parse(Rule::Keywords, input)?;
+    PartiQLParser::parse(Rule::Query, input)?;
     Ok(())
 }
 
@@ -34,13 +85,9 @@ mod tests {
     #[test]
     fn error() -> ParserResult<()> {
         match recognize_partiql("SELECT FROM MOO") {
-            Err(ParserError::SyntaxError { position, .. }) => assert_eq!(
-                Position::At {
-                    line: 1,
-                    column: 13
-                },
-                position
-            ),
+            Err(ParserError::SyntaxError { position, .. }) => {
+                assert_eq!(Position::at(1, 13), position)
+            }
             _ => panic!("Expected Syntax Error"),
         };
         Ok(())

partiql-parser/src/prelude.rs

@@ -3,6 +3,10 @@
 //! Convenience export of common traits and basic types that are almost always
 //! needed when using the parser APIs.
 
+pub use crate::result::LineAndColumn;
 pub use crate::result::ParserError;
 pub use crate::result::ParserResult;
 pub use crate::result::Position;
+pub use crate::scanner::Scanner;
+pub use std::convert::TryFrom;
+pub use std::convert::TryInto;