From 53eec7c2b2f7732d79f3a865a124b96e7616e1b4 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 10 Oct 2019 15:19:59 +0300 Subject: [PATCH 1/4] Squish bugs in whitespace removal sequences --- fluent/resource.lua | 28 +++++++++++++++++----------- spec/fixtures_spec.lua | 2 +- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fluent/resource.lua b/fluent/resource.lua index 7d264a9..c2f56da 100644 --- a/fluent/resource.lua +++ b/fluent/resource.lua @@ -14,7 +14,7 @@ local FluentNode = class({ if key == "id" then self.type = value elseif key == "value" then - self[key] = string.gsub(string.gsub(value, "\r\n?","\n"), "^\n+ +", "") + self[key] = string.gsub(value, "\r\n?","\n") elseif key ~= "pos" and key ~= "sigil" then self[key] = value end @@ -136,27 +136,33 @@ FTL.Pattern = class({ return tablex.reduce(math.min, indents) or 0 end local striplen = tablex.reduce(math.min, tablex.imap(mindent, self.elements)) or 0 - local i, strippref = 1, "\n" + local i, common = 1, "" while i <= striplen do - strippref = strippref .. " " + common = common .. " " i = i + 1 end - local strip = function (node, key, len) + local strip = function (node, key, common) if type(node.value) == "string" then - local value = node.value - if len >= 1 then - value = string.gsub(value, strippref, "\n\n") - end - value = key == 1 and string.gsub(value, "^[\n ]+", "") or value + -- Strip spaces from otherwise empty lines + local value = string.gsub(node.value, "\n +\n", "\n\n") + -- Strip leading whitespace from first element + value = key == 1 and string.gsub(value, "^\n+", "") or value + -- Strip trailing whitespace from last element value = key == #self.elements and string.gsub(value, "[\n ]+$", "") or value + -- Strip lowest common donominator indent from all elements + if string.len(common) >= 1 then + value = string.gsub(value, "\n"..common, "\n") + end if string.len(value) == 0 then + -- Delete from elements if result is completely empty self.elements[key] = nil else + -- Otherwise update value self.elements[key].value = value end end end - tablex.foreachi(self.elements, strip, striplen) + tablex.foreachi(self.elements, strip, common) end, __mul = function (self, node) if node:is_a(FTL.Message) or node:is_a(FTL.Attribute) or node:is_a(FTL.Variant) then @@ -179,7 +185,7 @@ FTL.TextElement = class({ end, __add = function (self, node) if self:is_a(node:is_a()) and self.appendable and node.appendable then - node.value = (node.value or "") .. "\n" .. (self.value or "") + node.value = (node.value or "") .. (self.value or "") return node end end, diff --git a/spec/fixtures_spec.lua b/spec/fixtures_spec.lua index 50c6d4f..104b169 100644 --- a/spec/fixtures_spec.lua +++ b/spec/fixtures_spec.lua @@ -47,7 +47,7 @@ describe('upstream reference fixture', function () or fname:match("/reference_expressions.ftl$") or fname:match("/select_expressions.ftl$") or fname:match("/select_indent.ftl$") - -- or fname:match("/sparse_entries.ftl$") + or fname:match("/sparse_entries.ftl$") or fname:match("/tab.ftl$") -- or fname:match("/term_parameters.ftl$") or fname:match("/terms.ftl$") From a9868cb0ab3a38eb1884e422aa2feeea2d5c512e Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 11 Oct 2019 17:42:56 +0300 Subject: [PATCH 2/4] Handle bad EOF senarios so Junk doesn't end up with our PEG grammar hack --- fluent/resource.lua | 6 ++++++ spec/fixtures_spec.lua | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fluent/resource.lua b/fluent/resource.lua index c2f56da..b4745d4 100644 --- a/fluent/resource.lua +++ b/fluent/resource.lua @@ -2,6 +2,8 @@ local class = require("pl.class") local tablex = require("pl.tablex") +local nulleof = "NULL\000" + local FTL = {} local node_to_type @@ -80,6 +82,10 @@ end FTL.Junk = class({ _base = FluentNode, + _init = function (self, node, resource) + self:super(node, resource) + self.content = string.gsub(self.content, nulleof.."$", "") + end }) FTL.Message = class({ diff --git a/spec/fixtures_spec.lua b/spec/fixtures_spec.lua index 104b169..ca07051 100644 --- a/spec/fixtures_spec.lua +++ b/spec/fixtures_spec.lua @@ -29,9 +29,9 @@ describe('upstream reference fixture', function () or fname:match("/crlf.ftl$") or fname:match("/eof_comment.ftl$") or fname:match("/eof_empty.ftl$") - -- or fname:match("/eof_id_equals.ftl$") - -- or fname:match("/eof_id.ftl$") - -- or fname:match("/eof_junk.ftl$") + or fname:match("/eof_id_equals.ftl$") + or fname:match("/eof_id.ftl$") + or fname:match("/eof_junk.ftl$") or fname:match("/eof_value.ftl$") -- or fname:match("/escaped_characters.ftl$") or fname:match("/junk.ftl$") From 17fb72cc04bb9071cf6bc47b82e79439fdc1e8e6 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 11 Oct 2019 21:46:21 +0300 Subject: [PATCH 3/4] Handle inline vs. start of block placables differently on append() --- fluent/parser.lua | 2 +- fluent/resource.lua | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/fluent/parser.lua b/fluent/parser.lua index fd6881b..9891196 100644 --- a/fluent/parser.lua +++ b/fluent/parser.lua @@ -66,7 +66,7 @@ local ftl_grammar = epnf.define(function (_ENV) InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"MessageReference" + V"TermReference" + V"VariableReference" + inline_placeable _InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"_TermReference" + V"VariableReference" SelectExpression = V"_InlineExpression" * blank^-1 * P"->" * blank_inline^-1 * V"variant_list" - PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable + block_placeable, "expression") + PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable, "expression") + Cg(block_placeable, "block_expression") Pattern = V"PatternElement"^1 Attribute = line_end * blank^-1 * P"." * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern" local junk_line = (1-P"\n"-P(nulleof))^0 * (P"\n" + P(nulleof)) diff --git a/fluent/resource.lua b/fluent/resource.lua index b4745d4..9021c12 100644 --- a/fluent/resource.lua +++ b/fluent/resource.lua @@ -201,13 +201,21 @@ FTL.TextElement = class({ }) FTL.Placeable = class({ - appendable = true, _base = FluentNode, _init = function (self, node, resource) node.id = "Placeable" + if node.block_expression then + getmetatable(self).blockwise = true + node.expression = node.block_expression + node.block_expression = nil + end node.expression = node_to_type(node.expression, resource) self:super(node, resource) end, + __add = function (self, node) + if self.blockwise and node.value then node.value = node.value .. "\n" end + return nil -- don't acknowledge this even happened so we keep trying options + end, __mod = function (self, node) if node:is_a(FTL.Pattern) then table.insert(node.elements, self) From 2a2df8c0e5b3586fd6c8451e8a0ef00044bdc93d Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sat, 12 Oct 2019 17:47:55 +0300 Subject: [PATCH 4/4] Try another approach to blockwise Placeables --- fluent/parser.lua | 4 ++-- fluent/resource.lua | 22 ++++++++++++++-------- spec/fixtures_spec.lua | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/fluent/parser.lua b/fluent/parser.lua index 9891196..d07ff0f 100644 --- a/fluent/parser.lua +++ b/fluent/parser.lua @@ -49,7 +49,7 @@ local ftl_grammar = epnf.define(function (_ENV) VariantKey = P"[" * blank^-1 * (V"NumberLiteral" + V"Identifier") * blank^-1 * P"]" NumberLiteral = Cg(C(P"-"^-1 * digits * (P"." * digits)^-1), "value") local inline_placeable = P"{" * blank^-1 * (V"SelectExpression" + V"InlineExpression") * blank^-1 * P"}" - local block_placeable = blank_block * blank_inline^-1 * inline_placeable + local block_placeable = Cg(blank_block, "_blockwise") * blank_inline^-1 * Cg(inline_placeable, "expression") local inline_text = text_char^1 local block_text = blank_block * blank_inline * indented_char * inline_text^-1 StringLiteral = P'"' * Cg(C(quoted_char^0), "value") * P'"' @@ -66,7 +66,7 @@ local ftl_grammar = epnf.define(function (_ENV) InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"MessageReference" + V"TermReference" + V"VariableReference" + inline_placeable _InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"_TermReference" + V"VariableReference" SelectExpression = V"_InlineExpression" * blank^-1 * P"->" * blank_inline^-1 * V"variant_list" - PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable, "expression") + Cg(block_placeable, "block_expression") + PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable, "expression") + block_placeable Pattern = V"PatternElement"^1 Attribute = line_end * blank^-1 * P"." * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern" local junk_line = (1-P"\n"-P(nulleof))^0 * (P"\n" + P(nulleof)) diff --git a/fluent/resource.lua b/fluent/resource.lua index 9021c12..099528f 100644 --- a/fluent/resource.lua +++ b/fluent/resource.lua @@ -17,7 +17,7 @@ local FluentNode = class({ self.type = value elseif key == "value" then self[key] = string.gsub(value, "\r\n?","\n") - elseif key ~= "pos" and key ~= "sigil" then + elseif key ~= "pos" and key ~= "sigil" and key ~= "_blockwise" then self[key] = value end end @@ -204,17 +204,23 @@ FTL.Placeable = class({ _base = FluentNode, _init = function (self, node, resource) node.id = "Placeable" - if node.block_expression then - getmetatable(self).blockwise = true - node.expression = node.block_expression - node.block_expression = nil - end + D{nobe._blockwise} + if node._blockwise then getmetatable(self).blockwise = true end + -- if node.block_expression then + -- getmetatable(self).blockwise = true + -- node.expression = node.block_expression + -- node.block_expression = nil + -- end node.expression = node_to_type(node.expression, resource) self:super(node, resource) end, __add = function (self, node) - if self.blockwise and node.value then node.value = node.value .. "\n" end - return nil -- don't acknowledge this even happened so we keep trying options + -- D{self.type, self.blockwise, self.expression, node.value} + if node:is_a(FTL.TextElement) and self.blockwise then + node.value = node.value .. "\n" + -- don't acknowledge this even happened so we go on to attach self as well as modify node + return nil + end end, __mod = function (self, node) if node:is_a(FTL.Pattern) then diff --git a/spec/fixtures_spec.lua b/spec/fixtures_spec.lua index ca07051..4cb5428 100644 --- a/spec/fixtures_spec.lua +++ b/spec/fixtures_spec.lua @@ -40,7 +40,7 @@ describe('upstream reference fixture', function () -- or fname:match("/member_expressions.ftl$") or fname:match("/messages.ftl$") or fname:match("/mixed_entries.ftl$") - -- or fname:match("/multiline_values.ftl$") + or fname:match("/multiline_values.ftl$") or fname:match("/numbers.ftl$") or fname:match("/obsolete.ftl$") or fname:match("/placeables.ftl$")