Streaming JSON-LD parser.

* Non-event based, but processes input in strict order (with a few exceptions). Passes 468/483 tests, with some arcane features not easily implemented using this approach.
* Available through `JSON::LD::Reader` interface, with the `:stream` option set to `true`.

Author: gkellogg

README.md

@@ -19,6 +19,12 @@ JSON::LD can now be used to create a _context_ from an RDFS/OWL definition, and
 
 Install with `gem install json-ld`
 
+### JSON-LD Streaming Profile
+This gem implements an optimized streaming reader used for generating RDF from large dataset dumps formatted as JSON-LD. Such documents must correspond to the [JSON-LD Streaming Profile](https://w3c.github.io/json-ld-streaming/):
+
+* Keys in JSON objects must be ordered with any of `@context`, and/or `@type` coming before any other keys, in that order. This includes aliases of those keys. It is strongly encouraged that `@id` be present, and come immediately after.
+* JSON-LD documents can be signaled or requested in [streaming document form](https://w3c.github.io/json-ld-streaming/#dfn-streaming-document-form). The profile URI identifying the [streaming document form](https://w3c.github.io/json-ld-streaming/#dfn-streaming-document-form) is `http://www.w3.org/ns/json-ld#streaming`.
+
 ### MultiJson parser
 The [MultiJson](https://rubygems.org/gems/multi_json) gem is used for parsing JSON; this defaults to the native JSON parser, but will use a more performant parser if one is available. A specific parser can be specified by adding the `:adapter` option to any API call. See [MultiJson](https://rubygems.org/gems/multi_json) for more information.
 

json-ld.gemspec

@@ -30,7 +30,8 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency     'link_header',     '~> 0.0', '>= 0.0.8'
   gem.add_runtime_dependency     'lru_redux',       '~> 1.1'
   gem.add_runtime_dependency     'json-canonicalization', '~> 0.2'
-  gem.add_runtime_dependency     'htmlentities',     '~> 4.3'
+  gem.add_runtime_dependency     'json-stream',     '~> 0.2'
+  gem.add_runtime_dependency     'htmlentities',    '~> 4.3'
   gem.add_runtime_dependency     'rack',            '~> 2.0'
   gem.add_development_dependency 'sinatra-linkeddata','~> 3.1'
   gem.add_development_dependency 'jsonlint',        '~> 0.3'  unless is_java

lib/json/ld.rb

@@ -34,6 +34,8 @@ module LD
     autoload :Normalize,          'json/ld/normalize'
     autoload :Reader,             'json/ld/reader'
     autoload :Resource,           'json/ld/resource'
+    autoload :StreamingReader,    'json/ld/streaming_reader'
+    autoload :StreamingWriter,    'json/ld/streaming_writer'
     autoload :VERSION,            'json/ld/version'
     autoload :Writer,             'json/ld/writer'
 
@@ -143,6 +145,7 @@ class InvalidReverseValue < JsonLdError; @code = "invalid @reverse value"; end
       class InvalidScopedContext < JsonLdError; @code = "invalid scoped context"; end
       class InvalidScriptElement < JsonLdError; @code = "invalid script element"; end
       class InvalidSetOrListObject < JsonLdError; @code = "invalid set or list object"; end
+      class InvalidStreamingKeyOrder < JsonLdError; @code = 'invalid streaming key order' end
       class InvalidTermDefinition < JsonLdError; @code = "invalid term definition"; end
       class InvalidBaseDirection < JsonLdError; @code = "invalid base direction"; end
       class InvalidTypedValue < JsonLdError; @code = "invalid typed value"; end

lib/json/ld/context.rb

@@ -1517,8 +1517,8 @@ def expand_value(property, value, useNativeTypes: false, rdfDirection: nil, base
         if value.datatype == RDF::URI(RDF.to_uri + "JSON") && processingMode('json-ld-1.1')
           # Value parsed as JSON
           # FIXME: MultiJson
-          res['@value'] = ::JSON.parse(value.object)
           res['@type'] = '@json'
+          res['@value'] = ::JSON.parse(value.object)
         elsif value.datatype.start_with?("https://www.w3.org/ns/i18n#") && rdfDirection == 'i18n-datatype' && processingMode('json-ld-1.1')
           lang, dir = value.datatype.fragment.split('_')
           res['@value'] = value.to_s
@@ -1534,24 +1534,23 @@ def expand_value(property, value, useNativeTypes: false, rdfDirection: nil, base
           end
           res['@direction'] = dir
         elsif useNativeTypes && RDF_LITERAL_NATIVE_TYPES.include?(value.datatype)
-          res['@value'] = value.object
           res['@type'] = uri(coerce(property)) if coerce(property)
+          res['@value'] = value.object
         else
           value.canonicalize! if value.datatype == RDF::XSD.double
-          res['@value'] = value.to_s
           if coerce(property)
             res['@type'] = uri(coerce(property)).to_s
           elsif value.has_datatype?
             res['@type'] = uri(value.datatype).to_s
           elsif value.has_language? || language(property)
             res['@language'] = (value.language || language(property)).to_s
-            # FIXME: direction
           end
+          res['@value'] = value.to_s
         end
         res
       else
         # Otherwise, initialize result to a JSON object with an @value member whose value is set to value.
-        res = {'@value' => value}
+        res = {}
 
         if td.type_mapping && !CONTAINERS_ID_VOCAB.include?(td.type_mapping.to_s)
           res['@type'] = td.type_mapping.to_s
@@ -1562,7 +1561,7 @@ def expand_value(property, value, useNativeTypes: false, rdfDirection: nil, base
           res['@direction'] = direction if direction
         end
 
-        res
+        res.merge('@value' => value)
       end
 
       result

lib/json/ld/reader.rb

@@ -1,12 +1,13 @@
 # -*- encoding: utf-8 -*-
-# frozen_string_literal: true
+
 module JSON::LD
   ##
   # A JSON-LD parser in Ruby.
   #
   # @see https://www.w3.org/TR/json-ld11-api
   # @author [Gregg Kellogg](http://greggkellogg.net/)
   class Reader < RDF::Reader
+    include StreamingReader
     format Format
 
     ##
@@ -46,6 +47,12 @@ def self.options
           control: :select,
           on: ["--rdf-direction DIR", %w(i18n-datatype compound-literal)],
           description: "How to serialize literal direction (i18n-datatype compound-literal)") {|arg| RDF::URI(arg)},
+        RDF::CLI::Option.new(
+          symbol: :stream,
+          datatype: TrueClass,
+          control: :checkbox,
+          on: ["--[no-]stream"],
+          description: "Optimize for streaming JSON-LD to RDF.") {|arg| arg},
       ]
     end
 
@@ -63,13 +70,11 @@ def initialize(input = $stdin, **options, &block)
       options[:base_uri] ||= options[:base]
       super do
         @options[:base] ||= base_uri.to_s if base_uri
-        begin
-          # Trim non-JSON stuff in script.
-          @doc = if input.respond_to?(:read)
-            input
-          else
-            StringIO.new(input.to_s.sub(%r(\A[^{\[]*)m, '').sub(%r([^}\]]*\Z)m, ''))
-          end
+        # Trim non-JSON stuff in script.
+        @doc = if input.respond_to?(:read)
+          input
+        else
+          StringIO.new(input.to_s.sub(%r(\A[^{\[]*)m, '').sub(%r([^}\]]*\Z)m, ''))
         end
 
         if block_given?
@@ -85,7 +90,11 @@ def initialize(input = $stdin, **options, &block)
     # @private
     # @see   RDF::Reader#each_statement
     def each_statement(&block)
-      JSON::LD::API.toRdf(@doc, **@options, &block)
+      if @options[:stream]
+        stream_statement(&block)
+      else
+        API.toRdf(@doc, **@options, &block)
+      end
     rescue ::JSON::ParserError, ::JSON::LD::JsonLdError => e
       log_fatal("Failed to parse input document: #{e.message}", exception: RDF::ReaderError)
     end
@@ -95,7 +104,7 @@ def each_statement(&block)
     # @see   RDF::Reader#each_triple
     def each_triple(&block)
       if block_given?
-        JSON::LD::API.toRdf(@doc, **@options) do |statement|
+        each_statement do |statement|
           yield(*statement.to_triple)
         end
       end