From 8b5973292ca03fff580fa6272dbb54856fa1020a Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 13 Jun 2021 12:24:38 -0700 Subject: [PATCH 01/18] init repo From 6a076a418fb540e4e25f84fc02e9430350ab7162 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 13 Jun 2021 12:39:46 -0700 Subject: [PATCH 02/18] Initial import --- .editorconfig | 22 +++++++++ LICENSE | 13 +++++ README.md | 35 +++++++++++++ lib/jpipe/__init__.py | 112 ++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 7 +++ setup.py | 104 +++++++++++++++++++++++++++++++++++++++ test/test_jpipe.py | 59 ++++++++++++++++++++++ 7 files changed, 352 insertions(+) create mode 100644 .editorconfig create mode 100644 LICENSE create mode 100644 README.md create mode 100644 lib/jpipe/__init__.py create mode 100644 pyproject.toml create mode 100644 setup.py create mode 100644 test/test_jpipe.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..d16e0488 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,22 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 4 +tab_width = 4 + +[*.py] +indent_style = space +indent_size = 4 +tab_width = 4 +max_line_length = 120 +trim_trailing_whitespace = true + +[*.{ebuild,sh,bash,initd}] +indent_style = tab +indent_size = 4 +tab_width = 4 +trim_trailing_whitespace = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..24f28fff --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2021 Zac Medico + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..13d708db --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# jpipe + +A python implementation (using +[jmespath.py](https://github.com/jmespath/jmespath.py)) +of the [`jp` CLI](https://github.com/jmespath/jp) for the +[JMESPath](https://jmespath.org/) language (a query language for JSON). + +## Compatiblity + +The aim is for 100% compatiblity with the official +[`jp` CLI for JMESPath](https://github.com/jmespath/jp). +Please open an issue if an incompatibility is found. + +## Usage +``` +usage: jpipe [-h] [-e EXPR_FILE] [-f FILENAME] [-u] [--ast] [expression] + + jpipe - A python implementation of the jp cli for JMESPath + +positional arguments: + expression + +optional arguments: + -h, --help show this help message and exit + -e EXPR_FILE, --expr-file EXPR_FILE + Read JMESPath expression from the specified file. + -f FILENAME, --filename FILENAME + The filename containing the input data. If a filename + is not given then data is read from stdin. + -u, --unquoted If the final result is a string, it will be printed + without quotes. + --ast Only print the AST of the parsed expression. Do not + rely on this output, only useful for debugging + purposes. +``` diff --git a/lib/jpipe/__init__.py b/lib/jpipe/__init__.py new file mode 100644 index 00000000..d0bb1d36 --- /dev/null +++ b/lib/jpipe/__init__.py @@ -0,0 +1,112 @@ +import argparse +import json +import os +import pprint +import sys + +import jmespath + +__version__ = "VERSION" +__project__ = "jpipe" +__description__ = "A python implementation of the jp CLI for JMESPath" +__author__ = "Zac Medico" +__email__ = "" +__classifiers__ = ( + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", + "Programming Language :: Unix Shell", +) +__copyright__ = "Copyright 2021 Zac Medico" +__license__ = "Apache-2.0" +__url__ = "https://github.com/pipebus/jpipe" +__project_urls__ = (("Bug Tracker", "https://github.com/pipebus/jpipe/issues"),) + + +# This 2 space indent matches https://github.com/jmespath/jp behavior. +JP_COMPAT_DUMP_KWARGS = ( + ("indent", 2), + ("ensure_ascii", False), +) + + +def jpipe_main(argv=None): + argv = sys.argv if argv is None else argv + parser = argparse.ArgumentParser( + prog=os.path.basename(argv[0]), + ) + parser.add_argument("expression", nargs="?", default=None) + parser.add_argument( + "-e", + "--expr-file", + dest="expr_file", + default=None, + help=( + "Read JMESPath expression from the specified file." + ), + ) + parser.add_argument( + "-f", + "--filename", + dest="filename", + default=None, + help=( + "The filename containing the input data. " + "If a filename is not given then data is " + "read from stdin." + ), + ) + parser.add_argument( + "-u", + "--unquoted", + action="store_false", + dest="quoted", + default=True, + help=( + "If the final result is a string, it will be printed without quotes." + ), + ) + parser.add_argument( + "--ast", + action="store_true", + help=("Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes."), + ) + parser.usage = "{}\n {} - {}".format(parser.format_usage().partition("usage: ")[-1], __project__, __description__) + + args = parser.parse_args(argv[1:]) + expression = args.expression + if expression == "help": + parser.print_help() + return 1 + + if expression and args.expr_file: + parser.error("Only use one of the expression or --expr-file arguments.") + + if args.expr_file: + with open(args.expr_file, "rt") as f: + expression = f.read() + + if args.ast: + # Only print the AST + expression = jmespath.compile(args.expression) + sys.stdout.write(pprint.pformat(expression.parsed)) + sys.stdout.write("\n") + return 0 + if args.filename: + with open(args.filename, "rt") as f: + data = json.load(f) + else: + data = sys.stdin.read() + data = json.loads(data) + + result = jmespath.search(expression, data) + if args.quoted or not isinstance(result, str): + result = json.dumps(result, **dict(JP_COMPAT_DUMP_KWARGS)) + + sys.stdout.write(result) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(jpipe_main(argv=sys.argv)) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1f8457df --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "jmespath", + "setuptools", + "wheel", +] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..2d7b055d --- /dev/null +++ b/setup.py @@ -0,0 +1,104 @@ +import os +import shutil +import subprocess +import sys + +from setuptools import ( + Command, + setup, +) + +__version__ = "0.0.1" + +sys.path.insert(0, "lib") +from jpipe import ( + __author__, + __classifiers__, + __description__, + __email__, + __project__, + __project_urls__, + __url__, +) + +sys.path.remove("lib") + + +class PyTest(Command): + user_options = [ + ("match=", "k", "Run only tests that match the provided expressions") + ] + + def initialize_options(self): + self.match = None + + def finalize_options(self): + pass + + def run(self): + testpath = "./test" + pytest_exe = shutil.which("py.test") + if pytest_exe is not None: + test_cmd = ( + [ + pytest_exe, + "-s", + "-v", + testpath, + "--cov-report=xml", + "--cov-report=term-missing", + ] + + (["-k", self.match] if self.match else []) + + ["--cov=jpipe"] + ) + else: + test_cmd = ["python", "test/test_jpipe.py"] + subprocess.check_call(test_cmd) + + +def version_subst(filename): + with open(filename, "rt") as f: + content = f.read() + if "VERSION" in content: + content = content.replace("VERSION", __version__) + with open(filename, "wt") as f: + f.write(content) + + +def find_packages(): + for dirpath, _dirnames, filenames in os.walk("lib"): + if "__init__.py" in filenames: + for filename in filenames: + if filename.endswith(".py"): + version_subst(os.path.join(dirpath, filename)) + yield os.path.relpath(dirpath, "lib") + + +with open(os.path.join(os.path.dirname(__file__), "README.md"), "rt") as f: + long_description = f.read() + + +setup( + name=__project__, + version=__version__, + description=__description__, + long_description=long_description, + long_description_content_type="text/markdown", + author=__author__, + author_email=__email__, + url=__url__, + project_urls=dict(__project_urls__), + classifiers=list(__classifiers__), + cmdclass={ + "test": PyTest, + }, + install_requires=["jmespath"], + package_dir={"": "lib"}, + packages=list(find_packages()), + entry_points={ + "console_scripts": [ + "jpipe = jpipe:jpipe_main", + ] + }, + python_requires=">=3.6", +) diff --git a/test/test_jpipe.py b/test/test_jpipe.py new file mode 100644 index 00000000..008a097d --- /dev/null +++ b/test/test_jpipe.py @@ -0,0 +1,59 @@ +import io +import os +import sys +import tempfile +import unittest.mock + +try: + import jpipe +except ImportError: + sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "lib" + ) + ) + import jpipe + + +class JPipeTest(unittest.TestCase): + def testJPipe(self): + for input_args, input_json, input_expr, expected_output, expected_retval in ( + ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), + ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), + (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), + ): + self._test_inputs( + input_args, input_json, input_expr, expected_output, expected_retval + ) + + def _test_inputs( + self, input_args, input_json, input_expr, expected_output, expected_retval + ): + with tempfile.NamedTemporaryFile( + mode="wt" + ) as expr_file, tempfile.NamedTemporaryFile(mode="wt") as filename: + + expr_file.write(input_expr) + expr_file.flush() + + filename.write(input_json) + filename.flush() + + mock_stdout = io.StringIO() + with unittest.mock.patch("sys.stdout", new=mock_stdout): + retval = jpipe.jpipe_main( + [ + "jpipe", + "--expr-file", + expr_file.name, + "--filename", + filename.name, + ] + + list(input_args) + ) + self.assertEqual(retval, expected_retval) + self.assertEqual(mock_stdout.getvalue(), expected_output) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 4ca9dd3bdf36cf16b1a10a670da8f3eafd7d3388 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Mon, 14 Jun 2021 23:38:25 -0700 Subject: [PATCH 03/18] jpp: fork jpp command from jpipe This is the first commit of a series that will port the golang jpp implementation to python. The jpp implementation is in these commits which begin with the 0.1.3 tag and end with the 0.1.3.1 tag: https://github.com/pipebus/jpp/compare/2e59b07d58b76dd7e8c1a26d5815b8eb0d3717a3%5E...a8ea7c998c07baafd8f239cfbe6a418619e5e01f --- lib/jpipe/jpp/__init__.py | 0 lib/jpipe/jpp/main.py | 100 ++++++++++++++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 lib/jpipe/jpp/__init__.py create mode 100644 lib/jpipe/jpp/main.py diff --git a/lib/jpipe/jpp/__init__.py b/lib/jpipe/jpp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py new file mode 100644 index 00000000..d178a9cc --- /dev/null +++ b/lib/jpipe/jpp/main.py @@ -0,0 +1,100 @@ +import argparse +import json +import os +import pprint +import sys + +import jmespath + +from .. import __version__ +from .. import __project__ + +__description__ = "jpp is an extended superset of the jp CLI for JMESPath" + +# This 2 space indent matches https://github.com/jmespath/jp behavior. +JP_COMPAT_DUMP_KWARGS = ( + ("indent", 2), + ("ensure_ascii", False), +) + + +def jpp_main(argv=None): + argv = sys.argv if argv is None else argv + parser = argparse.ArgumentParser( + prog=os.path.basename(argv[0]), + ) + parser.add_argument("expression", nargs="?", default=None) + parser.add_argument( + "-e", + "--expr-file", + dest="expr_file", + default=None, + help=( + "Read JMESPath expression from the specified file." + ), + ) + parser.add_argument( + "-f", + "--filename", + dest="filename", + default=None, + help=( + "The filename containing the input data. " + "If a filename is not given then data is " + "read from stdin." + ), + ) + parser.add_argument( + "-u", + "--unquoted", + action="store_false", + dest="quoted", + default=True, + help=( + "If the final result is a string, it will be printed without quotes." + ), + ) + parser.add_argument( + "--ast", + action="store_true", + help=("Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes."), + ) + parser.usage = "{}\n {} - {}".format(parser.format_usage().partition("usage: ")[-1], __project__, __description__) + + args = parser.parse_args(argv[1:]) + expression = args.expression + if expression == "help": + parser.print_help() + return 1 + + if expression and args.expr_file: + parser.error("Only use one of the expression or --expr-file arguments.") + + if args.expr_file: + with open(args.expr_file, "rt") as f: + expression = f.read() + + if args.ast: + # Only print the AST + expression = jmespath.compile(args.expression) + sys.stdout.write(pprint.pformat(expression.parsed)) + sys.stdout.write("\n") + return 0 + if args.filename: + with open(args.filename, "rt") as f: + data = json.load(f) + else: + data = sys.stdin.read() + data = json.loads(data) + + result = jmespath.search(expression, data) + if args.quoted or not isinstance(result, str): + result = json.dumps(result, **dict(JP_COMPAT_DUMP_KWARGS)) + + sys.stdout.write(result) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(jpp_main(argv=sys.argv)) diff --git a/setup.py b/setup.py index 2d7b055d..d7cc2440 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.0.1" +__version__ = "0.1.3" sys.path.insert(0, "lib") from jpipe import ( @@ -98,6 +98,7 @@ def find_packages(): entry_points={ "console_scripts": [ "jpipe = jpipe:jpipe_main", + "jpp = jpipe.jpp.main:jpp_main", ] }, python_requires=">=3.6", From 90682799322b58db5ad2864a6a9fcc5cd879bf7d Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 00:05:47 -0700 Subject: [PATCH 04/18] jpp: copy jpipe unit tests --- test/test_jpp.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/test_jpp.py diff --git a/test/test_jpp.py b/test/test_jpp.py new file mode 100644 index 00000000..219380ca --- /dev/null +++ b/test/test_jpp.py @@ -0,0 +1,59 @@ +import io +import os +import sys +import tempfile +import unittest.mock + +try: + import jpipe.jpp.main +except ImportError: + sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "lib" + ) + ) + import jpipe.jpp.main + + +class JPPTest(unittest.TestCase): + def testJPP(self): + for input_args, input_json, input_expr, expected_output, expected_retval in ( + ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), + ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), + (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), + ): + self._test_inputs( + input_args, input_json, input_expr, expected_output, expected_retval + ) + + def _test_inputs( + self, input_args, input_json, input_expr, expected_output, expected_retval + ): + with tempfile.NamedTemporaryFile( + mode="wt" + ) as expr_file, tempfile.NamedTemporaryFile(mode="wt") as filename: + + expr_file.write(input_expr) + expr_file.flush() + + filename.write(input_json) + filename.flush() + + mock_stdout = io.StringIO() + with unittest.mock.patch("sys.stdout", new=mock_stdout): + retval = jpipe.jpp.main.jpp_main( + [ + "jpp", + "--expr-file", + expr_file.name, + "--filename", + filename.name, + ] + + list(input_args) + ) + self.assertEqual(retval, expected_retval) + self.assertEqual(mock_stdout.getvalue(), expected_output) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From ef5e7e33e155c66da3a8a7b02ad52382aacfb3b0 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Mon, 14 Jun 2021 23:54:46 -0700 Subject: [PATCH 05/18] jpp: Add --compact, -c bool flag to omit nonessential whitespace Reference golang implementation: https://github.com/pipebus/jpp/commit/ee7042022fccd01a7494b2fff239894428d24770 --- lib/jpipe/jpp/main.py | 17 ++++++++++++++++- test/test_jpp.py | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index d178a9cc..0df0203d 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -24,6 +24,16 @@ def jpp_main(argv=None): prog=os.path.basename(argv[0]), ) parser.add_argument("expression", nargs="?", default=None) + parser.add_argument( + "-c", + "--compact", + action="store_true", + dest="compact", + default=False, + help=( + "Produce compact JSON output that omits nonessential whitespace." + ), + ) parser.add_argument( "-e", "--expr-file", @@ -70,6 +80,11 @@ def jpp_main(argv=None): if expression and args.expr_file: parser.error("Only use one of the expression or --expr-file arguments.") + dump_kwargs = dict(JP_COMPAT_DUMP_KWARGS) + if args.compact: + dump_kwargs.pop("indent", None) + dump_kwargs["separators"] = (',', ':') + if args.expr_file: with open(args.expr_file, "rt") as f: expression = f.read() @@ -89,7 +104,7 @@ def jpp_main(argv=None): result = jmespath.search(expression, data) if args.quoted or not isinstance(result, str): - result = json.dumps(result, **dict(JP_COMPAT_DUMP_KWARGS)) + result = json.dumps(result, **dump_kwargs) sys.stdout.write(result) sys.stdout.write("\n") diff --git a/test/test_jpp.py b/test/test_jpp.py index 219380ca..339f7d05 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -19,6 +19,7 @@ class JPPTest(unittest.TestCase): def testJPP(self): for input_args, input_json, input_expr, expected_output, expected_retval in ( ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), + (("-c",), """{"hello": "world"}""", "@", '{"hello":"world"}\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), ): From b64e300b0986d3cbb31a89a95a24a681694815ab Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 01:12:03 -0700 Subject: [PATCH 06/18] jpp: decode all objects from the input stream Reference golang implementation: https://github.com/pipebus/jpp/commit/ba4e8833f8bf3f6c85810319a3753008e72d5620 https://github.com/pipebus/jpp/commit/d75cf886203343d6614a8a722e035cf1fe3d1144 --- lib/jpipe/jpp/main.py | 60 ++++++++++++++++++++++++++++++++++++------- test/test_jpp.py | 1 + 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 0df0203d..e3ea2ad1 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -18,6 +18,47 @@ ) +def decode_json_stream(stream): + """ + Decode a text JSON input stream and generate objects until EOF. + """ + eof = False + line_buffer = [] + while line_buffer or not eof: + progress = False + if not eof: + line = stream.readline() + if line: + progress = True + line_buffer.append(line) + else: + eof = True + + if line_buffer: + chunk = "".join(line_buffer) + del line_buffer[:] + + try: + yield json.loads(chunk) + progress = True + except json.JSONDecodeError as e: + if e.pos > 0: + try: + yield json.loads(chunk[:e.pos]) + progress = True + except json.JSONDecodeError: + # Raise if there's no progress, since a given + # chunk should be growning if it is not yet + # decodable. + if not progress: + raise + line_buffer.append(chunk) + else: + line_buffer.append(chunk[e.pos:]) + else: + raise + + def jpp_main(argv=None): argv = sys.argv if argv is None else argv parser = argparse.ArgumentParser( @@ -95,19 +136,20 @@ def jpp_main(argv=None): sys.stdout.write(pprint.pformat(expression.parsed)) sys.stdout.write("\n") return 0 + if args.filename: - with open(args.filename, "rt") as f: - data = json.load(f) + f = open(args.filename, "rt") else: - data = sys.stdin.read() - data = json.loads(data) + f = sys.stdin - result = jmespath.search(expression, data) - if args.quoted or not isinstance(result, str): - result = json.dumps(result, **dump_kwargs) + with f: + for data in decode_json_stream(f): + result = jmespath.search(expression, data) + if args.quoted or not isinstance(result, str): + result = json.dumps(result, **dump_kwargs) - sys.stdout.write(result) - sys.stdout.write("\n") + sys.stdout.write(result) + sys.stdout.write("\n") return 0 diff --git a/test/test_jpp.py b/test/test_jpp.py index 339f7d05..987bc46c 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -20,6 +20,7 @@ def testJPP(self): for input_args, input_json, input_expr, expected_output, expected_retval in ( ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), (("-c",), """{"hello": "world"}""", "@", '{"hello":"world"}\n', 0), + (("-c",), """{\n "foo": "bar"\n}{ \n"foo": "x"\n}""", "@", '{"foo":"bar"}\n{"foo":"x"}\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), ): From f51b65f143b54b84c1a76b0207f9c2e897548d60 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 01:55:58 -0700 Subject: [PATCH 07/18] jpp: Add --slurp, -s bool flag like jq has Read one or more input JSON objects into an array and apply the JMESPath expression to the resulting array. Reference golang implementation: https://github.com/pipebus/jpp/commit/8a08979e22529be138623cec18b65184ba932331 --- lib/jpipe/jpp/main.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index e3ea2ad1..c662959c 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -95,6 +95,16 @@ def jpp_main(argv=None): "read from stdin." ), ) + parser.add_argument( + "-s", + "--slurp", + action="store_true", + dest="slurp", + default=False, + help=( + "Read one or more input JSON objects into an array and apply the JMESPath expression to the resulting array." + ), + ) parser.add_argument( "-u", "--unquoted", @@ -143,13 +153,26 @@ def jpp_main(argv=None): f = sys.stdin with f: - for data in decode_json_stream(f): + stream_iter = decode_json_stream(f) + while True: + if args.slurp: + data = list(stream_iter) + if not data: + break + else: + try: + data = next(stream_iter) + except StopIteration: + break + result = jmespath.search(expression, data) if args.quoted or not isinstance(result, str): result = json.dumps(result, **dump_kwargs) sys.stdout.write(result) sys.stdout.write("\n") + if args.slurp: + break return 0 From 0bbc90f4cfe219339c8b2ecfccff5932565334fd Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 02:50:51 -0700 Subject: [PATCH 08/18] jpp: Add --accumulate, -a option which accumulates all output objects into a single recursively merged object This option will accumulate all output objects into a single recursively merged output object. Reference golang implementation: https://github.com/pipebus/jpp/commit/e7989df1056486dbf3e1c3b53a2f208a2023efc9 --- lib/jpipe/jpp/main.py | 80 +++++++++++++++++++++++++++++++++++++------ test/test_jpp.py | 3 ++ 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index c662959c..f9ae1054 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -3,6 +3,7 @@ import os import pprint import sys +import itertools import jmespath @@ -65,6 +66,16 @@ def jpp_main(argv=None): prog=os.path.basename(argv[0]), ) parser.add_argument("expression", nargs="?", default=None) + parser.add_argument( + "-a", + "--accumulate", + action="store_true", + dest="accumulate", + default=False, + help=( + "Accumulate all output objects into a single recursively merged output object." + ), + ) parser.add_argument( "-c", "--compact", @@ -152,29 +163,78 @@ def jpp_main(argv=None): else: f = sys.stdin + accumulator = None + eof = False + with f: stream_iter = decode_json_stream(f) while True: - if args.slurp: - data = list(stream_iter) - if not data: - break - else: - try: - data = next(stream_iter) - except StopIteration: + while True: + if args.slurp: + data = list(stream_iter) + if not data: + eof = True + break + else: + try: + data = next(stream_iter) + except StopIteration: + eof = True + break + + result = jmespath.search(expression, data) + + if args.accumulate: + if accumulator is None: + accumulator = result + else: + accumulator = merge(accumulator, result) + else: break - result = jmespath.search(expression, data) + if args.accumulate: + result = accumulator + elif eof: + break + if args.quoted or not isinstance(result, str): result = json.dumps(result, **dump_kwargs) sys.stdout.write(result) sys.stdout.write("\n") - if args.slurp: + if eof or args.accumulate or args.slurp: break return 0 +def merge(base, head): + """ + Recursively merge head onto base. + """ + if isinstance(head, dict): + if not isinstance(base, dict): + return head + + result = {} + for k in itertools.chain(head, base): + try: + result[k] = merge(base[k], head[k]) + except KeyError: + try: + result[k] = head[k] + except KeyError: + result[k] = base[k] + + elif isinstance(head, list): + result = [] + if isinstance(base, list): + result.extend(base) + result.extend(head) + else: + result = head + + return result + + if __name__ == "__main__": sys.exit(jpp_main(argv=sys.argv)) diff --git a/test/test_jpp.py b/test/test_jpp.py index 987bc46c..dec72c75 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -21,6 +21,9 @@ def testJPP(self): ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), (("-c",), """{"hello": "world"}""", "@", '{"hello":"world"}\n', 0), (("-c",), """{\n "foo": "bar"\n}{ \n"foo": "x"\n}""", "@", '{"foo":"bar"}\n{"foo":"x"}\n', 0), + (("-a", "-c",), """{"foo": ["a"]}{"foo": ["x"]}""", "@", '{"foo":["a","x"]}\n', 0), + (("-a", "-c",), """["a"]["x"]""", "@", '["a","x"]\n', 0), + (("-c", "-s",), """"a"\n"x"\n""", "@", '["a","x"]\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), ): From 8acf6dcfae4cec9f9943ceb660aa5537fe8e4aa2 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 03:19:51 -0700 Subject: [PATCH 09/18] version 0.1.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7cc2440..edefe137 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.1.3" +__version__ = "0.1.3.1" sys.path.insert(0, "lib") from jpipe import ( From 72e273e216b79ab5d1a079c7f130034e95baf59a Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 15 Jun 2021 13:33:10 -0700 Subject: [PATCH 10/18] jpp: fix --accumulate array merge to coalesce duplicates Reference golang implementation: https://github.com/pipebus/jpp/commit/0abe226af789977087407929fba0e4cc85f9c5a1 Fixes: 0bbc90f4cfe2 ("jpp: Add --accumulate, -a option which accumulates all output objects into a single recursively merged object") --- lib/jpipe/jpp/main.py | 6 +++++- setup.py | 2 +- test/test_jpp.py | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index f9ae1054..c2f30634 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -229,7 +229,11 @@ def merge(base, head): result = [] if isinstance(base, list): result.extend(base) - result.extend(head) + for node in head: + if node not in result: + result.append(node) + else: + result.extend(head) else: result = head diff --git a/setup.py b/setup.py index edefe137..9e24500e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.1.3.1" +__version__ = "0.1.3.2" sys.path.insert(0, "lib") from jpipe import ( diff --git a/test/test_jpp.py b/test/test_jpp.py index dec72c75..a02d471a 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -21,7 +21,10 @@ def testJPP(self): ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), (("-c",), """{"hello": "world"}""", "@", '{"hello":"world"}\n', 0), (("-c",), """{\n "foo": "bar"\n}{ \n"foo": "x"\n}""", "@", '{"foo":"bar"}\n{"foo":"x"}\n', 0), - (("-a", "-c",), """{"foo": ["a"]}{"foo": ["x"]}""", "@", '{"foo":["a","x"]}\n', 0), + (("-a", "-c",), """{"foo": ["a"]}{"foo": ["a", "x"]}""", "@", '{"foo":["a","x"]}\n', 0), + (("-a", "-c",), """{"foo": ["a"]}{"foo": ["a"]}""", "@", '{"foo":["a"]}\n', 0), + (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a"]}""", "@", '{"foo":["a","a"]}\n', 0), + (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a", "b"]}""", "@", '{"foo":["a","a","b"]}\n', 0), (("-a", "-c",), """["a"]["x"]""", "@", '["a","x"]\n', 0), (("-c", "-s",), """"a"\n"x"\n""", "@", '["a","x"]\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), From fbd37512a21e6c18c4746da1117b9b62e1e25387 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 09:49:42 -0700 Subject: [PATCH 11/18] jpp: default to the identity expression @ if no expression is given Reference golang implementation: https://github.com/pipebus/jpp/commit/74c0bbd7ccd1d325a0d2b12ff18251c431fd0636 --- lib/jpipe/jpp/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index c2f30634..39370970 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -5,7 +5,7 @@ import sys import itertools -import jmespath +import jmespath.exceptions from .. import __version__ from .. import __project__ @@ -150,6 +150,10 @@ def jpp_main(argv=None): if args.expr_file: with open(args.expr_file, "rt") as f: expression = f.read() + if not expression: + raise jmespath.exceptions.EmptyExpressionError() + elif not expression: + expression = "@" if args.ast: # Only print the AST From a61cdea8b16b602a3836628b1c08a628e5727315 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 11:39:11 -0700 Subject: [PATCH 12/18] jpp: Add --read-raw, -R bool flag like jq has Reference golang implementation: https://github.com/pipebus/jpp/commit/b025412072e6fa1029a5ebc1be92915978ef4813 --- lib/jpipe/jpp/main.py | 33 ++++++++++++++++++++++++++------- test/test_jpp.py | 3 +++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 39370970..16f65d66 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -106,6 +106,16 @@ def jpp_main(argv=None): "read from stdin." ), ) + parser.add_argument( + "-R", + "--read-raw", + action="store_true", + dest="read_raw", + default=False, + help=( + "Read raw string input and box it as JSON strings." + ), + ) parser.add_argument( "-s", "--slurp", @@ -168,23 +178,30 @@ def jpp_main(argv=None): f = sys.stdin accumulator = None + stream_iter = None eof = False with f: - stream_iter = decode_json_stream(f) + if not args.read_raw: + stream_iter = decode_json_stream(f) while True: while True: if args.slurp: - data = list(stream_iter) - if not data: - eof = True - break + if stream_iter is None: + data = f.read() + else: + data = list(stream_iter) + elif stream_iter is None: + data = f.readline() else: try: data = next(stream_iter) except StopIteration: - eof = True - break + data = None + + if not data: + eof = True + break result = jmespath.search(expression, data) @@ -193,6 +210,8 @@ def jpp_main(argv=None): accumulator = result else: accumulator = merge(accumulator, result) + if args.slurp: + break else: break diff --git a/test/test_jpp.py b/test/test_jpp.py index a02d471a..9796d025 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -29,6 +29,9 @@ def testJPP(self): (("-c", "-s",), """"a"\n"x"\n""", "@", '["a","x"]\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), + (("-R", "-a", "-c",), "hello world", "@", '"hello world"\n', 0), + (("-R", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\n"\n"line 2\\n"\n"line 3\\n"\n', 0), + (("-R", "-s", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\nline 2\\nline 3\\n"\n', 0), ): self._test_inputs( input_args, input_json, input_expr, expected_output, expected_retval From 073bb8828adcf5c64b292d4ca997ca6c3f909ef0 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 12:30:55 -0700 Subject: [PATCH 13/18] jpp: Add --raw, -r bool flag like jq has (an alias for unquoted) Reference golang implementation: https://github.com/pipebus/jpp/commit/39fd7916d58b950ce1dd706978c60f10e1525a9c --- lib/jpipe/jpp/main.py | 17 ++++++++++++++++- setup.py | 2 +- test/test_jpp.py | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 16f65d66..9e93c8a0 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -106,6 +106,16 @@ def jpp_main(argv=None): "read from stdin." ), ) + parser.add_argument( + "-r", + "--raw", + action="store_false", + dest="quoted", + default=True, + help=( + "If the final result is a string, it will be printed without quotes (an alias for unquoted)." + ), + ) parser.add_argument( "-R", "--read-raw", @@ -224,7 +234,12 @@ def jpp_main(argv=None): result = json.dumps(result, **dump_kwargs) sys.stdout.write(result) - sys.stdout.write("\n") + + if args.quoted or ( + not args.quoted and isinstance(result, str) and result[-1:] != "\n" + ): + sys.stdout.write("\n") + if eof or args.accumulate or args.slurp: break return 0 diff --git a/setup.py b/setup.py index 9e24500e..6675ce99 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.1.3.2" +__version__ = "0.1.3.3" sys.path.insert(0, "lib") from jpipe import ( diff --git a/test/test_jpp.py b/test/test_jpp.py index 9796d025..936fce80 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -30,6 +30,8 @@ def testJPP(self): ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), (("-R", "-a", "-c",), "hello world", "@", '"hello world"\n', 0), + (("-R", "-r",), "hello world\n", "@", 'hello world\n', 0), + (("-R", "-u",), "hello world\n", "@", 'hello world\n', 0), (("-R", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\n"\n"line 2\\n"\n"line 3\\n"\n', 0), (("-R", "-s", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\nline 2\\nline 3\\n"\n', 0), ): From 6b5b6110718aba4e8f6296442e9a8b67aa6ac31d Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 21:13:19 -0700 Subject: [PATCH 14/18] jpp: rename --read-raw to --raw-input for consistency with jq Reference golang implementation: https://github.com/pipebus/jpp/commit/fbd178fb35f5b859ebc34a8e65fe7ebe5389b3ce --- lib/jpipe/jpp/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 9e93c8a0..0033d45e 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -118,9 +118,9 @@ def jpp_main(argv=None): ) parser.add_argument( "-R", - "--read-raw", + "--raw-input", action="store_true", - dest="read_raw", + dest="raw_input", default=False, help=( "Read raw string input and box it as JSON strings." @@ -192,7 +192,7 @@ def jpp_main(argv=None): eof = False with f: - if not args.read_raw: + if not args.raw_input: stream_iter = decode_json_stream(f) while True: while True: From 60cd963bcca6df6300f96441f99ee647c7caf1e9 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 22:22:58 -0700 Subject: [PATCH 15/18] jpp: Add --unbox, -u flag (and drop --unquoted to reduce clutter) If the final result is a list, unbox it into a stream of output objects that is suitable for consumption by --slurp mode. Reference golang implementation: https://github.com/pipebus/jpp/commit/9caa261b30d3440cd930f76c938fae92244d9c85 See: https://github.com/stedolan/jq/issues/878 --- lib/jpipe/jpp/main.py | 38 +++++++++++++++++++++++--------------- test/test_jpp.py | 4 ++-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 0033d45e..59e74b74 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -113,7 +113,7 @@ def jpp_main(argv=None): dest="quoted", default=True, help=( - "If the final result is a string, it will be printed without quotes (an alias for unquoted)." + "If the final result is a string, it will be printed without quotes." ), ) parser.add_argument( @@ -138,12 +138,12 @@ def jpp_main(argv=None): ) parser.add_argument( "-u", - "--unquoted", - action="store_false", - dest="quoted", - default=True, + "--unbox", + action="store_true", + dest="unbox", + default=False, help=( - "If the final result is a string, it will be printed without quotes." + "If the final result is a list, unbox it into a stream of output objects that is suitable for consumption by --slurp mode." ), ) parser.add_argument( @@ -230,21 +230,29 @@ def jpp_main(argv=None): elif eof: break - if args.quoted or not isinstance(result, str): - result = json.dumps(result, **dump_kwargs) - - sys.stdout.write(result) - - if args.quoted or ( - not args.quoted and isinstance(result, str) and result[-1:] != "\n" - ): - sys.stdout.write("\n") + if args.unbox and isinstance(result, list): + for element in result: + output_result(args, dump_kwargs, element) + else: + output_result(args, dump_kwargs, result) if eof or args.accumulate or args.slurp: break return 0 +def output_result(args, dump_kwargs, result): + if args.quoted or not isinstance(result, str): + result = json.dumps(result, **dump_kwargs) + + sys.stdout.write(result) + + if args.quoted or ( + not args.quoted and isinstance(result, str) and result[-1:] != "\n" + ): + sys.stdout.write("\n") + + def merge(base, head): """ Recursively merge head onto base. diff --git a/test/test_jpp.py b/test/test_jpp.py index 936fce80..ab8b7604 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -26,12 +26,12 @@ def testJPP(self): (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a"]}""", "@", '{"foo":["a","a"]}\n', 0), (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a", "b"]}""", "@", '{"foo":["a","a","b"]}\n', 0), (("-a", "-c",), """["a"]["x"]""", "@", '["a","x"]\n', 0), + (("-a", "-c", "-u"), """["a"]["x"]""", "@", '"a"\n"x"\n', 0), (("-c", "-s",), """"a"\n"x"\n""", "@", '["a","x"]\n', 0), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), - (("-u",), """{"hello": "world"}""", "@.hello", "world\n", 0), + (("-r",), """{"hello": "world"}""", "@.hello", "world\n", 0), (("-R", "-a", "-c",), "hello world", "@", '"hello world"\n', 0), (("-R", "-r",), "hello world\n", "@", 'hello world\n', 0), - (("-R", "-u",), "hello world\n", "@", 'hello world\n', 0), (("-R", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\n"\n"line 2\\n"\n"line 3\\n"\n', 0), (("-R", "-s", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\nline 2\\nline 3\\n"\n', 0), ): From d811157ef8bd0f1a7fe04f402ea9d46833c7a756 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 16 Jun 2021 22:26:50 -0700 Subject: [PATCH 16/18] jpp: rename --raw to --raw-output for consistency with jq Reference golang implementation: https://github.com/pipebus/jpp/commit/f574aa57ce4f68fc3d6892302dc6e69bf3640fc6 --- lib/jpipe/jpp/main.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 59e74b74..df377aa4 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -108,7 +108,7 @@ def jpp_main(argv=None): ) parser.add_argument( "-r", - "--raw", + "--raw-output", action="store_false", dest="quoted", default=True, diff --git a/setup.py b/setup.py index 6675ce99..8c98ad11 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.1.3.3" +__version__ = "0.1.3.4" sys.path.insert(0, "lib") from jpipe import ( From 87036fde2bfabaf89fa2268e47c9207ea8528e96 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 17 Jun 2021 15:07:26 -0700 Subject: [PATCH 17/18] 3 files reformatted with black --- lib/jpipe/__init__.py | 16 +++--- lib/jpipe/jpp/main.py | 44 +++++++--------- test/test_jpp.py | 120 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 137 insertions(+), 43 deletions(-) diff --git a/lib/jpipe/__init__.py b/lib/jpipe/__init__.py index d0bb1d36..1d56778d 100644 --- a/lib/jpipe/__init__.py +++ b/lib/jpipe/__init__.py @@ -41,9 +41,7 @@ def jpipe_main(argv=None): "--expr-file", dest="expr_file", default=None, - help=( - "Read JMESPath expression from the specified file." - ), + help=("Read JMESPath expression from the specified file."), ) parser.add_argument( "-f", @@ -62,16 +60,18 @@ def jpipe_main(argv=None): action="store_false", dest="quoted", default=True, - help=( - "If the final result is a string, it will be printed without quotes." - ), + help=("If the final result is a string, it will be printed without quotes."), ) parser.add_argument( "--ast", action="store_true", - help=("Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes."), + help=( + "Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes." + ), + ) + parser.usage = "{}\n {} - {}".format( + parser.format_usage().partition("usage: ")[-1], __project__, __description__ ) - parser.usage = "{}\n {} - {}".format(parser.format_usage().partition("usage: ")[-1], __project__, __description__) args = parser.parse_args(argv[1:]) expression = args.expression diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index df377aa4..39e9c90e 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -45,7 +45,7 @@ def decode_json_stream(stream): except json.JSONDecodeError as e: if e.pos > 0: try: - yield json.loads(chunk[:e.pos]) + yield json.loads(chunk[: e.pos]) progress = True except json.JSONDecodeError: # Raise if there's no progress, since a given @@ -55,7 +55,7 @@ def decode_json_stream(stream): raise line_buffer.append(chunk) else: - line_buffer.append(chunk[e.pos:]) + line_buffer.append(chunk[e.pos :]) else: raise @@ -82,18 +82,14 @@ def jpp_main(argv=None): action="store_true", dest="compact", default=False, - help=( - "Produce compact JSON output that omits nonessential whitespace." - ), + help=("Produce compact JSON output that omits nonessential whitespace."), ) parser.add_argument( "-e", "--expr-file", dest="expr_file", default=None, - help=( - "Read JMESPath expression from the specified file." - ), + help=("Read JMESPath expression from the specified file."), ) parser.add_argument( "-f", @@ -112,9 +108,7 @@ def jpp_main(argv=None): action="store_false", dest="quoted", default=True, - help=( - "If the final result is a string, it will be printed without quotes." - ), + help=("If the final result is a string, it will be printed without quotes."), ) parser.add_argument( "-R", @@ -122,9 +116,7 @@ def jpp_main(argv=None): action="store_true", dest="raw_input", default=False, - help=( - "Read raw string input and box it as JSON strings." - ), + help=("Read raw string input and box it as JSON strings."), ) parser.add_argument( "-s", @@ -149,9 +141,13 @@ def jpp_main(argv=None): parser.add_argument( "--ast", action="store_true", - help=("Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes."), + help=( + "Only print the AST of the parsed expression. Do not rely on this output, only useful for debugging purposes." + ), + ) + parser.usage = "{}\n {} - {}".format( + parser.format_usage().partition("usage: ")[-1], __project__, __description__ ) - parser.usage = "{}\n {} - {}".format(parser.format_usage().partition("usage: ")[-1], __project__, __description__) args = parser.parse_args(argv[1:]) expression = args.expression @@ -165,7 +161,7 @@ def jpp_main(argv=None): dump_kwargs = dict(JP_COMPAT_DUMP_KWARGS) if args.compact: dump_kwargs.pop("indent", None) - dump_kwargs["separators"] = (',', ':') + dump_kwargs["separators"] = (",", ":") if args.expr_file: with open(args.expr_file, "rt") as f: @@ -242,15 +238,15 @@ def jpp_main(argv=None): def output_result(args, dump_kwargs, result): - if args.quoted or not isinstance(result, str): - result = json.dumps(result, **dump_kwargs) + if args.quoted or not isinstance(result, str): + result = json.dumps(result, **dump_kwargs) - sys.stdout.write(result) + sys.stdout.write(result) - if args.quoted or ( - not args.quoted and isinstance(result, str) and result[-1:] != "\n" - ): - sys.stdout.write("\n") + if args.quoted or ( + not args.quoted and isinstance(result, str) and result[-1:] != "\n" + ): + sys.stdout.write("\n") def merge(base, head): diff --git a/test/test_jpp.py b/test/test_jpp.py index ab8b7604..5f8ef8a5 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -20,20 +20,118 @@ def testJPP(self): for input_args, input_json, input_expr, expected_output, expected_retval in ( ((), """{"hello": "world"}""", "@", '{\n "hello": "world"\n}\n', 0), (("-c",), """{"hello": "world"}""", "@", '{"hello":"world"}\n', 0), - (("-c",), """{\n "foo": "bar"\n}{ \n"foo": "x"\n}""", "@", '{"foo":"bar"}\n{"foo":"x"}\n', 0), - (("-a", "-c",), """{"foo": ["a"]}{"foo": ["a", "x"]}""", "@", '{"foo":["a","x"]}\n', 0), - (("-a", "-c",), """{"foo": ["a"]}{"foo": ["a"]}""", "@", '{"foo":["a"]}\n', 0), - (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a"]}""", "@", '{"foo":["a","a"]}\n', 0), - (("-a", "-c",), """{"foo": ["a", "a"]}{"foo": ["a", "b"]}""", "@", '{"foo":["a","a","b"]}\n', 0), - (("-a", "-c",), """["a"]["x"]""", "@", '["a","x"]\n', 0), + ( + ("-c",), + """{\n "foo": "bar"\n}{ \n"foo": "x"\n}""", + "@", + '{"foo":"bar"}\n{"foo":"x"}\n', + 0, + ), + ( + ( + "-a", + "-c", + ), + """{"foo": ["a"]}{"foo": ["a", "x"]}""", + "@", + '{"foo":["a","x"]}\n', + 0, + ), + ( + ( + "-a", + "-c", + ), + """{"foo": ["a"]}{"foo": ["a"]}""", + "@", + '{"foo":["a"]}\n', + 0, + ), + ( + ( + "-a", + "-c", + ), + """{"foo": ["a", "a"]}{"foo": ["a"]}""", + "@", + '{"foo":["a","a"]}\n', + 0, + ), + ( + ( + "-a", + "-c", + ), + """{"foo": ["a", "a"]}{"foo": ["a", "b"]}""", + "@", + '{"foo":["a","a","b"]}\n', + 0, + ), + ( + ( + "-a", + "-c", + ), + """["a"]["x"]""", + "@", + '["a","x"]\n', + 0, + ), (("-a", "-c", "-u"), """["a"]["x"]""", "@", '"a"\n"x"\n', 0), - (("-c", "-s",), """"a"\n"x"\n""", "@", '["a","x"]\n', 0), + ( + ( + "-c", + "-s", + ), + """"a"\n"x"\n""", + "@", + '["a","x"]\n', + 0, + ), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-r",), """{"hello": "world"}""", "@.hello", "world\n", 0), - (("-R", "-a", "-c",), "hello world", "@", '"hello world"\n', 0), - (("-R", "-r",), "hello world\n", "@", 'hello world\n', 0), - (("-R", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\n"\n"line 2\\n"\n"line 3\\n"\n', 0), - (("-R", "-s", "-c",), "line 1\nline 2\nline 3\n", "@", '"line 1\\nline 2\\nline 3\\n"\n', 0), + ( + ( + "-R", + "-a", + "-c", + ), + "hello world", + "@", + '"hello world"\n', + 0, + ), + ( + ( + "-R", + "-r", + ), + "hello world\n", + "@", + "hello world\n", + 0, + ), + ( + ( + "-R", + "-c", + ), + "line 1\nline 2\nline 3\n", + "@", + '"line 1\\n"\n"line 2\\n"\n"line 3\\n"\n', + 0, + ), + ( + ( + "-R", + "-s", + "-c", + ), + "line 1\nline 2\nline 3\n", + "@", + '"line 1\\nline 2\\nline 3\\n"\n', + 0, + ), ): self._test_inputs( input_args, input_json, input_expr, expected_output, expected_retval From ff641ef7c4172cf691a1d2cb774fd488349ca262 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 17 Jun 2021 15:15:47 -0700 Subject: [PATCH 18/18] jpp: re-add --unquoted (a short -u means --unbox now) Reference golang implementation: https://github.com/pipebus/jpp/commit/6a504e8305e06a5f96c4c39f5f0864bc54689d26 --- lib/jpipe/jpp/main.py | 11 ++++++++++- setup.py | 2 +- test/test_jpp.py | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/jpipe/jpp/main.py b/lib/jpipe/jpp/main.py index 39e9c90e..c792734a 100644 --- a/lib/jpipe/jpp/main.py +++ b/lib/jpipe/jpp/main.py @@ -108,7 +108,9 @@ def jpp_main(argv=None): action="store_false", dest="quoted", default=True, - help=("If the final result is a string, it will be printed without quotes."), + help=( + "If the final result is a string, it will be printed without quotes (an alias for --unquoted)." + ), ) parser.add_argument( "-R", @@ -138,6 +140,13 @@ def jpp_main(argv=None): "If the final result is a list, unbox it into a stream of output objects that is suitable for consumption by --slurp mode." ), ) + parser.add_argument( + "--unquoted", + action="store_false", + dest="quoted", + default=True, + help=("If the final result is a string, it will be printed without quotes."), + ) parser.add_argument( "--ast", action="store_true", diff --git a/setup.py b/setup.py index 8c98ad11..df5d0e87 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup, ) -__version__ = "0.1.3.4" +__version__ = "0.1.3.7" sys.path.insert(0, "lib") from jpipe import ( diff --git a/test/test_jpp.py b/test/test_jpp.py index 5f8ef8a5..67003ba6 100644 --- a/test/test_jpp.py +++ b/test/test_jpp.py @@ -90,6 +90,7 @@ def testJPP(self): ), ((), """{"hello": "world"}""", "@.hello", '"world"\n', 0), (("-r",), """{"hello": "world"}""", "@.hello", "world\n", 0), + (("--unquoted",), """{"hello": "world"}""", "@.hello", "world\n", 0), ( ( "-R",