Release 1.1.8

* Use RDF::Util::File.open_file for retrieving remote contexts; this enables the use of HTTP caching if RestClient is included.
* When opening a context, make sure options are passed to documentLoader, but exclude 'Cache-Control' directive. This ensures that frequently accessed contexts can be cached, even if the referencing document isn't cached.
* Added :stream option to JSON::LD::Writer to write out blocks where context and subject are the same. If quads come sorted by context and subject, this dramatically improves write performance by avoiding core algorithms. (use --stream option to jsonld CLI).
* For API#initialize, set context to link header only if the document doesn't define a context, and one is not passed.
* Define Context#merge and Context#merge!
* Relax absolute IRI validation in context unless :validate flag is set.
* Recover from bad JSON/RDF serialization in debug messages, typically due to UTF-8 errors.
* Ignore input header options when requesting remote files to ensure property Accept sequence is used.

Author: gkellogg

.travis.yml

@@ -7,6 +7,11 @@ rvm:
   - 1.9.3
   - 2.0
   - 2.1
-  - jruby-19mode
+  - 2.2
+  - jruby
   - rbx-2
 cache: bundler
+matrix:
+  allow_failures:
+      - rvm: 2.2
+

Gemfile

@@ -1,18 +1,18 @@
 source "https://rubygems.org"
 
 gemspec
-gem 'rdf',              :git => "git://github.com/ruby-rdf/rdf.git", :branch => "develop"
-gem 'rdf-spec',         :git => "git://github.com/ruby-rdf/rdf-spec.git", :branch => "develop"
+gem 'rdf',              git: "git://github.com/ruby-rdf/rdf.git", branch: "develop"
+gem 'rdf-spec',         git: "git://github.com/ruby-rdf/rdf-spec.git", branch: "develop"
 
 group :development do
-  gem 'rdf-turtle',     :git => "git://github.com/ruby-rdf/rdf-turtle.git", :branch => "develop"
-  gem 'rdf-trig',       :git => "git://github.com/ruby-rdf/rdf-trig.git", :branch => "develop"
+  gem 'rdf-turtle',     git: "git://github.com/ruby-rdf/rdf-turtle.git", branch: "develop"
+  gem 'rdf-trig',       git: "git://github.com/ruby-rdf/rdf-trig.git", branch: "develop"
 end
 
 group :debug do
   gem "wirble"
   gem "linkeddata"
-  gem "byebug", :platforms => [:mri_20, :mri_21]
+  gem "byebug", platforms: [:mri_20, :mri_21]
 end
 
 platforms :rbx do

README.md

@@ -13,6 +13,12 @@ JSON::LD can now be used to create a _context_ from an RDFS/OWL definition, and
 
 Install with `gem install json-ld`
 
+### JSON-LD Streaming Profile
+This gem implements an optimized streaming writer used for generating JSON-LD from large repositories. Such documents result in the JSON-LD Streaming Profile:
+
+* Each statement written as a separate node in expanded/flattened form.
+* RDF Lists are written as separate nodes using `rdf:first` and `rdf:rest` properties.
+
 ## Examples
 
     require 'rubygems'
@@ -165,7 +171,7 @@ Install with `gem install json-ld`
     graph = RDF::Graph.new << JSON::LD::API.toRdf(input)
 
     require 'rdf/turtle'
-    graph.dump(:ttl, :prefixes => {:foaf => "http://xmlns.com/foaf/0.1/"})
+    graph.dump(:ttl, prefixes: {foaf: "http://xmlns.com/foaf/0.1/"})
     @prefix foaf: <http://xmlns.com/foaf/0.1/> .
 
     <http://example.org/people#joebob> a foaf:Person;
@@ -213,8 +219,8 @@ Install with `gem install json-ld`
 ## RDF Reader and Writer
 {JSON::LD} also acts as a normal RDF reader and writer, using the standard RDF.rb reader/writer interfaces:
 
-    graph = RDF::Graph.load("etc/doap.jsonld", :format => :jsonld)
-    graph.dump(:jsonld, :standard_prefixes => true)
+    graph = RDF::Graph.load("etc/doap.jsonld", format: :jsonld)
+    graph.dump(:jsonld, standard_prefixes: true)
 
 `RDF::GRAPH#dump` can also take a `:context` option to use a separately defined context
 

Rakefile

@@ -1,6 +1,6 @@
 require 'rubygems'
 
-task :default => [ :spec ]
+task default: [ :spec ]
 
 namespace :gem do
   desc "Build the json-ld-#{File.read('VERSION').chomp}.gem file"
@@ -50,7 +50,7 @@ namespace :presentation do
   end
 
   desc "Build presentation files"
-  task :build => %w(
+  task build: %w(
     presentation/dbpedia/expanded.jsonld
     presentation/dbpedia/compacted.jsonld
     presentation/dbpedia/framed.jsonld
@@ -95,5 +95,5 @@ namespace :doc do
   YARD::Rake::YardocTask.new
 end
 
-task :default => :spec
-task :specs => :spec
+task default: :spec
+task specs: :spec

VERSION

@@ -1 +1 @@
-1.1.7
+1.1.8

bin/jsonld

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
 require 'rubygems'
-$:.unshift(File.expand_path("../../lib", __FILE__))
 begin
   require 'linkeddata'
 rescue LoadError
 end
+$:.unshift(File.expand_path("../../lib", __FILE__))
 require 'json/ld'
 require 'getoptlong'
 require 'open-uri'
@@ -17,42 +17,49 @@ def run(input, options)
   options[:format] = :jsonld if options[:compact] || options[:frame]
 
   # If input format is not JSON-LD, transform input to JSON-LD first
-  if options[:format] == :jsonld && options[:input_format] != :jsonld
-    r = reader_class.new(input, options[:parser_options])
-    g = RDF::Repository.new << r
-    input = JSON::LD::API.fromRdf(g)
+  reader = if options[:input_format] != :jsonld
+    reader_class.new(input, options[:parser_options])
   end
 
   prefixes = {}
   start = Time.new
   if options[:expand]
-    options = options.merge(:expandContext => options.delete(:context)) if options.has_key?(:context)
+    options = options.merge(expandContext: options.delete(:context)) if options.has_key?(:context)
+    input = JSON::LD::API.fromRdf(reader) if reader
     output = JSON::LD::API.expand(input, options)
     secs = Time.new - start
     options[:output].puts output.to_json(JSON::LD::JSON_STATE)
     STDERR.puts "Expanded in #{secs} seconds." unless options[:quiet]
   elsif options[:compact]
+    input = JSON::LD::API.fromRdf(reader) if reader
     output = JSON::LD::API.compact(input, options[:context], options)
     secs = Time.new - start
     options[:output].puts output.to_json(JSON::LD::JSON_STATE)
     STDERR.puts "Compacted in #{secs} seconds." unless options[:quiet]
   elsif options[:flatten]
+    input = JSON::LD::API.fromRdf(reader) if reader
     output = JSON::LD::API.flatten(input, options[:context], options)
     secs = Time.new - start
     options[:output].puts output.to_json(JSON::LD::JSON_STATE)
     STDERR.puts "Flattened in #{secs} seconds." unless options[:quiet]
   elsif options[:frame]
+    input = JSON::LD::API.fromRdf(reader) if reader
     output = JSON::LD::API.frame(input, options[:frame], options)
     secs = Time.new - start
     options[:output].puts output.to_json(JSON::LD::JSON_STATE)
     STDERR.puts "Framed in #{secs} seconds." unless options[:quiet]
   else
-    options = options.merge(:expandContext => options.delete(:context)) if options.has_key?(:context)
-    g = JSON::LD::API.toRdf(input, options)
+    options = options.merge(expandContext: options.delete(:context)) if options.has_key?(:context)
+    parser_options = options[:parser_options].merge(standard_prefixes: true)
+    reader ||= JSON::LD::Reader.new(input, parser_options)
+    num = 0
+    RDF::Writer.for(options[:output_format]).new(options[:output], parser_options) do |w|
+      reader.each do |statement|
+        num += 1
+        w << statement
+      end
+    end
     secs = Time.new - start
-    num = g.count
-    parser_options = options[:parser_options].merge(:standard_prefixes => true)
-    options[:output].puts g.dump(options[:output_format], parser_options)
     STDERR.puts "\nParsed #{num} statements in #{secs} seconds @ #{num/secs} statements/second." unless options[:quiet]
   end
 rescue
@@ -62,17 +69,18 @@ rescue
 end
 
 parser_options = {
-  :base     => nil,
-  :progress => false,
-  :validate => false,
-  :strict   => false,
+  base:     nil,
+  progress: false,
+  validate: false,
+  stream:   false,
+  strict:   false,
 }
 
 options = {
-  :parser_options => parser_options,
-  :output        => STDOUT,
-  :output_format => :turtle,
-  :input_format  => :jsonld,
+  parser_options: parser_options,
+  output:         STDOUT,
+  output_format:  :jsonld,
+  input_format:   :jsonld,
 }
 input = nil
 
@@ -89,8 +97,9 @@ OPT_ARGS = [
   ["--output", "-o",  GetoptLong::REQUIRED_ARGUMENT,"Output to the specified file path"],
   ["--parse-only",    GetoptLong::NO_ARGUMENT,      "Parse the document for well-formedness only"],
   ["--quiet",         GetoptLong::NO_ARGUMENT,      "Supress most output other than progress indicators"],
+  ["--stream",        GetoptLong::NO_ARGUMENT,      "Use Streaming reader/writer"],
   ["--uri",           GetoptLong::REQUIRED_ARGUMENT,"URI to be used as the document base"],
-  ["--verbose",       GetoptLong::NO_ARGUMENT,      "Detail on execution"],
+  ["--validate",      GetoptLong::NO_ARGUMENT,      "Validate while processing"],
   ["--help", "-?",    GetoptLong::NO_ARGUMENT,      "This message"]
 ]
 def usage
@@ -108,7 +117,6 @@ def usage
   exit(1)
 end
 
-
 opts = GetoptLong.new(*OPT_ARGS.map {|o| o[0..-2]})
 
 opts.each do |opt, arg|
@@ -125,18 +133,29 @@ opts.each do |opt, arg|
   when '--output'       then options[:output] = File.open(arg, "w")
   when '--parse-only'   then options[:parse_only] = true
   when '--quiet'        then options[:quiet] = true
+  when '--stream'       then parser_options[:stream] = true
   when '--uri'          then parser_options[:base] = arg
-  when '--verbose'      then $verbose = true
+  when '--validate'     then parser_options[:validate] = true
   when '--help'         then usage
   end
 end
 
+# Hack
+options[:parser_options][:context] = options[:context] if parser_options[:stream]
+
+if !(options.keys & [:expand, :compact, :flatten, :frame]).empty? &&
+   (parser_options[:stream] || options[:output_format] != :jsonld)
+   STDERR.puts "Incompatible options"
+   exit(1)
+end
+
 if ARGV.empty?
   s = input ? input : $stdin.read
   run(StringIO.new(s), options)
 else
   ARGV.each do |file|
-    run(file, options)
+    # Call with opened files
+    RDF::Util::File.open_file(file, options) {|f| run(f, options)}
   end
 end
 puts

example-files/bench.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+$:.unshift(File.expand_path("../../lib", __FILE__))
+require 'json/ld'
+require 'getoptlong'
+
+def run(options)
+  parser_options = options[:parser_options].merge(standard_prefixes: true)
+  statement = RDF::Statement(RDF::URI("http://example/a"), RDF::URI("http://example/b"), RDF::Literal("c"))
+  start = Time.new
+  num = 100_000
+  RDF::Writer.for(options[:output_format]).new(options[:output], parser_options) do |w|
+    (1..num).each do
+      w << statement
+    end
+  end
+  secs = Time.new - start
+  STDERR.puts "\nProcessed #{num} statements in #{secs} seconds @ #{num/secs} statements/second." unless options[:quiet]
+end
+
+parser_options = {
+  base:     nil,
+  progress: false,
+  validate: false,
+  stream:   false,
+  strict:   false,
+}
+
+options = {
+  parser_options: parser_options,
+  output:         STDOUT,
+  output_format:  :jsonld,
+  input_format:   :jsonld,
+}
+input = nil
+
+OPT_ARGS = [
+  ["--output", "-o",  GetoptLong::REQUIRED_ARGUMENT,"Output to the specified file path"],
+  ["--stream",        GetoptLong::NO_ARGUMENT,      "Use Streaming reader/writer"],
+  ["--help", "-?",    GetoptLong::NO_ARGUMENT,      "This message"]
+]
+def usage
+  STDERR.puts %{Usage: #{$0} [options] file ...}
+  width = OPT_ARGS.map do |o|
+    l = o.first.length
+    l += o[1].length + 2 if o[1].is_a?(String)
+    l
+  end.max
+  OPT_ARGS.each do |o|
+    s = "  %-*s  " % [width, (o[1].is_a?(String) ? "#{o[0,2].join(', ')}" : o[0])]
+    s += o.last
+    STDERR.puts s
+  end
+  exit(1)
+end
+
+
+opts = GetoptLong.new(*OPT_ARGS.map {|o| o[0..-2]})
+
+opts.each do |opt, arg|
+  case opt
+  when '--output'       then options[:output] = File.open(arg, "w")
+  when '--quiet'        then options[:quiet] = true
+  when '--stream'       then parser_options[:stream] = true
+  when '--help'         then usage
+  end
+end
+
+run(options)

json-ld.gemspec

@@ -27,9 +27,11 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.2'
   gem.requirements          = []
-  gem.add_runtime_dependency     'rdf',             '~> 1.1', '>= 1.1.4'
-  gem.add_development_dependency 'open-uri-cached', '~> 0.0', '>= 0.0.5'
+  gem.add_runtime_dependency     'rdf',             '~> 1.1', '>= 1.1.7'
   gem.add_development_dependency 'yard' ,           '~> 0.8'
+  gem.add_development_dependency "rack-cache",      '~> 1.2'
+  gem.add_development_dependency "rest-client",     '~> 1.7'
+  gem.add_development_dependency "rest-client-components", '~> 1.3'
   gem.add_development_dependency 'rspec',           '~> 3.0'
   gem.add_development_dependency 'rspec-its',       '~> 1.0'
   gem.add_development_dependency 'rdf-spec',        '~> 1.1'

lib/json/ld.rb

@@ -25,7 +25,7 @@ module LD
     require 'json/ld/format'
     require 'json/ld/utils'
     autoload :API,                'json/ld/api'
-    autoload :Context,  'json/ld/context'
+    autoload :Context,            'json/ld/context'
     autoload :Normalize,          'json/ld/normalize'
     autoload :Reader,             'json/ld/reader'
     autoload :Resource,           'json/ld/resource'
@@ -77,11 +77,11 @@ module LD
     NATIVE_DATATYPES = [RDF::XSD.integer.to_s, RDF::XSD.boolean.to_s, RDF::XSD.double.to_s]
 
     JSON_STATE = JSON::State.new(
-      :indent       => "  ",
-      :space        => " ",
-      :space_before => "",
-      :object_nl    => "\n",
-      :array_nl     => "\n"
+      indent:       "  ",
+      space:        " ",
+      space_before: "",
+      object_nl:    "\n",
+      array_nl:     "\n"
     )
 
     def self.debug?; @debug; end