Implement the reference Fluent Syntax parser (#103)

This is the official reference implementation of the parser. The parser focuses
on strictness and correctness at a cost of reduced performance.

The ASDL description of the AST has been removed in favor of `syntax/ast.mjs`
which defines the actual AST nodes returned by the reference parser.

The EBNF is now auto-generated from the reference parser's `syntax/grammar.mjs`
file. It provides an easy to read overview of the grammar and will continue to
be updated in the future.

Going forward, all changes to the grammar will be implemented in the
reference parser first, which also ships with an extensive test suite.

Notable changes:

  - Added junk entries.

    The grammar now explicitly defines the `junk_line` production which is
    converted into `Junk` during the AST construction.

  - Comments may now be attached to Messages or Terms.

    The grammar now encodes the behavior for Comments preceding Messages and
    Terms. If there are no blank lines between the Comment and the Message or
    the Term, the Comment becomes part of the `Message` or the `Term` AST node.

  - Fixed many issues with the white-space grammar.

    The EBNF for Syntax 0.5 had many issues with its use of `inline-space` and
    `break-indent`. It defined undesirable or impossible parsing behavior.
    These issues have been fixed and the new test suite will help ensure the
    correctness of the grammar in the future.

Author: stasm

.eslintrc

@@ -0,0 +1,22 @@
+{
+    "parserOptions": {
+        "ecmaVersion": 2018,
+        "sourceType": "module"
+    },
+    "env": {
+        "es6": true,
+        "node": true
+    },
+    "rules": {
+        "eqeqeq": ["error", "always"],
+        "indent": ["error", 4, {
+            "CallExpression": {"arguments": 1},
+            "MemberExpression": "off",
+            "SwitchCase": 1
+        }],
+        "quotes": ["error", "double"],
+        "semi": ["error", "always"],
+        "no-tabs": "error",
+        "no-undef": "error"
+    }
+}

.travis.yml

@@ -0,0 +1,16 @@
+sudo: false
+language: node_js
+script: npm run ci
+node_js: 9
+cache:
+  directories: node_modules
+notifications:
+  irc:
+    channels:
+      - "irc.mozilla.org#l20n"
+    on_success: always
+    on_failure: always
+    use_notice: true
+    template:
+      - "#%{build_number} %{message} %{build_url}"
+      - "Change by %{author}: %{compare_url}"

README.md

@@ -3,17 +3,34 @@
 Fluent is a localization system designed to unleash the expressive power of
 the natural language.
 
-This repository contains specification, design and documentation for Fluent.
-
+This repository contains the specification, the reference implementation of the
+parser and the documentation for Fluent.
 
 ## Fluent Syntax (FTL)
 
 FTL is the syntax for describing translation resources in Project Fluent.  FTL
-stands for *Fluent Translation List*.
+stands for *Fluent Translation List*. Read the [Fluent Syntax Guide][] to get
+started learning Fluent.
+
+The `syntax/` directory contains the reference implementation of the syntax as
+a _LL(infinity)_ parser.
+
+The `spec/` directory contains the formal EBNF grammar, autogenerated from the
+reference implementation.
+
+## Development
+
+While working on the reference parser, use the following commands to test and
+validate your work:
+
+    npm test                # Test the parser against JSON AST fixtures.
+    npm run lint            # Lint the parser code.
 
-Read the [Fluent Syntax Guide][].  The `spec/` directory contains the formal
-grammar of the Fluent Syntax.
+    npm run build:ebnf      # Generate the EBNF from syntax/grammar.mjs.
+    npm run build:fixtures  # Generate test fixtures (FTL → JSON AST).
+    npm run build:guide     # Build the HTML version of the Guide.
 
+    npm run bench           # Run the performance benchmark on large FTL.
 
 ## Learn more
 

bin/ebnf.mjs

@@ -0,0 +1,72 @@
+import fs from "fs";
+import readline from "readline";
+import parse_args from "minimist";
+import ebnf from "../lib/ebnf.mjs";
+
+const argv = parse_args(process.argv.slice(2), {
+    boolean: ["help"],
+    alias: {
+        help: "h",
+    },
+});
+
+if (argv.help) {
+    exit_help(0);
+}
+
+const [file_path] = argv._;
+
+if (file_path === "-") {
+    from_stdin();
+} else if (file_path) {
+    from_file(file_path);
+} else {
+    exit_help(1);
+}
+
+function exit_help(exit_code) {
+    console.log(`
+    Usage: node --experimental-modules ebnf.mjs [OPTIONS] <FILE>
+
+    When FILE is "-", read text from stdin.
+
+    Examples:
+
+        node --experimental-modules ebnf.mjs path/to/grammar.mjs
+        cat path/to/grammar.mjs | node --experimental-modules ebnf.mjs -
+
+    Options:
+
+        -h, --help      Display help and quit.
+`);
+    process.exit(exit_code);
+}
+
+function from_stdin() {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+        prompt: "fluent>",
+    });
+
+    const lines = [];
+
+    rl.on("line", line => lines.push(line));
+    rl.on("close", () =>
+        print_ebnf(lines.join("\n") + "\n"));
+}
+
+function from_file(file_path) {
+    fs.readFile(file_path, "utf8", (err, content) => {
+        if (err) {
+            throw err;
+        }
+
+        print_ebnf(content);
+    });
+}
+
+function print_ebnf(source) {
+    // Each EBNF rule already ends with \n.
+    process.stdout.write(ebnf(source));
+}

bin/parse.mjs

@@ -0,0 +1,74 @@
+import fs from "fs";
+import readline from "readline";
+import parse_args from "minimist";
+import {Resource} from "../syntax/grammar.mjs";
+
+const argv = parse_args(process.argv.slice(2), {
+    boolean: ["help"],
+    alias: {
+        help: "h",
+    },
+});
+
+if (argv.help) {
+    exit_help(0);
+}
+
+const [file_path] = argv._;
+
+if (file_path === "-") {
+    parse_stdin();
+} else if (file_path) {
+    parse_file(file_path);
+} else {
+    exit_help(1);
+}
+
+function exit_help(exit_code) {
+    console.log(`
+    Usage: node --experimental-modules parse.mjs [OPTIONS] <FILE>
+
+    When FILE is "-", read text from stdin.
+
+    Examples:
+
+        node --experimental-modules parse.mjs path/to/file.ftl
+        cat path/to/file.ftl | node --experimental-modules parse.mjs -
+
+    Options:
+
+        -h, --help      Display help and quit.
+`);
+    process.exit(exit_code);
+}
+
+function parse_stdin() {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+        prompt: "fluent>",
+    });
+
+    const lines = [];
+
+    rl.on("line", line => lines.push(line));
+    rl.on("close", () =>
+        parse(lines.join("\n") + "\n"));
+}
+
+function parse_file(file_path) {
+    fs.readFile(file_path, "utf8", (err, content) => {
+        if (err) {
+            throw err;
+        }
+
+        parse(content);
+    });
+}
+
+
+function parse(ftl) {
+    Resource.run(ftl).fold(
+        ast => console.log(JSON.stringify(ast, null, 4)),
+        err => console.error(err));
+}

lib/combinators.mjs

@@ -0,0 +1,139 @@
+import Parser from "./parser.mjs";
+import {Success, Failure} from "./result.mjs";
+import {join} from "./mappers.mjs";
+
+export function defer(fn) {
+    // Parsers may be defined as defer(() => parser) to avoid cyclic
+    // dependecies.
+    return new Parser(stream =>
+        fn().run(stream));
+}
+
+export function char(c) {
+    return new Parser(stream =>
+        stream.head() === c
+            ? new Success(c, stream.move(1))
+            : new Failure(`${c} not found`, stream));
+}
+
+export function regex(re) {
+    return new Parser(stream => {
+        const result = stream.exec(re);
+
+        if (result === null) {
+            return new Failure("regex did not match", stream);
+        }
+
+        const [match] = result;
+
+        return new Success(match, stream.move(match.length));
+    });
+}
+
+export function charset(range) {
+    return regex(new RegExp(`[${range}]`));
+}
+
+export function string(str) {
+    return sequence(...str.split("").map(char)).map(join);
+}
+
+export function eof() {
+    return new Parser(stream =>
+        stream.head() === Symbol.for("eof")
+            ? new Success(stream.head(), stream.move(1))
+            : new Failure("not at EOF", stream));
+}
+
+export function lookahead(parser) {
+    return new Parser(stream =>
+        parser
+            .run(stream)
+            .fold(
+                value => new Success(value, stream),
+                value => new Failure(value, stream)));
+}
+
+export function not(parser) {
+    return new Parser(stream =>
+        parser
+            .run(stream)
+            .fold(
+                (value, tail) => new Failure("not failed", stream),
+                (value, tail) => new Success(null, stream)));
+}
+
+export function and(...parsers) {
+    const final = parsers.pop();
+    return sequence(...parsers.map(lookahead), final)
+        .map(results => results[results.length - 1]);
+}
+
+export function either(...parsers) {
+    return new Parser(stream => {
+        for (const parser of parsers) {
+            const result = parser.run(stream);
+            if (result instanceof Success) {
+                return result;
+            }
+        }
+        return new Failure("either failed", stream);
+    });
+}
+
+export function always(value) {
+    return new Parser(stream => new Success(value, stream));
+}
+
+export function never(value) {
+    return new Parser(stream => new Failure(value, stream));
+}
+
+export function maybe(parser) {
+    return new Parser(stream =>
+        parser
+            .run(stream)
+            .fold(
+                (value, tail) => new Success(value, tail),
+                (value, tail) => new Success(undefined, stream)));
+}
+
+export function append(p1, p2) {
+    return p1.chain(values =>
+        p2.map(value => values.concat([value])));
+}
+
+export function after(prefix, parser) {
+    return sequence(prefix, parser)
+        .map(([left, value]) => value);
+}
+
+export function sequence(...parsers) {
+    return parsers.reduce(
+        (acc, parser) => append(acc, parser),
+        always([]));
+}
+
+export function repeat(parser) {
+    return new Parser(stream =>
+        parser
+            .run(stream)
+            .fold(
+                (value, tail) =>
+                    repeat(parser)
+                        .map(rest => [value].concat(rest))
+                        .run(tail),
+                (value, tail) => new Success([], stream)));
+}
+
+export function repeat1(parser) {
+    return new Parser(stream =>
+        parser
+            .run(stream)
+            .fold(
+                (value, tail) =>
+                    repeat(parser)
+                        .map(rest => [value].concat(rest))
+                        .run(tail),
+                (value, tail) => new Failure("repeat1 failed", stream)));
+}

lib/ebnf.mjs

@@ -0,0 +1,18 @@
+import babylon from "babylon";
+import walk from "./walker.mjs";
+import visitor from "./visitor.mjs";
+import serialize from "./serializer.mjs";
+
+export default
+function ebnf(source) {
+    let grammar_ast = babylon.parse(source, {sourceType: "module"});
+    let rules = walk(grammar_ast, visitor);
+    let state = {
+        max_name_length: Math.max(
+            ...rules.map(rule => rule.name.length)),
+    };
+    return rules
+        .map(rule => serialize(rule, state))
+        .join("");
+}
+