Merge pull request #45 from SpringMT/fix/fix-SEGV-when-gc-at-streaming

fix segv bugs when gc at streaming

Author: SpringMT

benchmarks/zstd_streaming_compress_memory.rb

@@ -0,0 +1,27 @@
+require 'benchmark/ips'
+
+$LOAD_PATH.unshift '../lib'
+
+require 'json'
+require 'objspace'
+require 'zstd-ruby'
+
+p "#{ObjectSpace.memsize_of_all/1000} #{ObjectSpace.count_objects} #{`ps -o rss= -p #{Process.pid}`.to_i}"
+
+sample_file_name = ARGV[0]
+
+json_string = IO.read("./samples/#{sample_file_name}")
+
+i = 0
+while true do
+  stream = Zstd::StreamingCompress.new
+  stream << json_string[0, 5]
+  res = stream.flush
+  stream << json_string[5..-1]
+  res << stream.finish
+  if ((i % 1000) == 0 )
+    GC.start
+    puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+  end
+  i += 1
+end

benchmarks/zstd_streaming_decompress_memory.rb

@@ -0,0 +1,26 @@
+require 'benchmark/ips'
+
+$LOAD_PATH.unshift '../lib'
+
+require 'json'
+require 'zstd-ruby'
+require 'objspace'
+
+p "#{ObjectSpace.memsize_of_all/1000} #{ObjectSpace.count_objects} #{`ps -o rss= -p #{Process.pid}`.to_i}"
+
+sample_file_name = ARGV[0]
+
+cstr = IO.read("./results/#{sample_file_name}.zstd")
+i = 0
+while true do
+  stream = Zstd::StreamingDecompress.new
+  result = ''
+  result << stream.decompress(cstr[0, 10])
+  result << stream.decompress(cstr[10..-1])
+
+  if ((i % 1000) == 0 )
+    GC.start
+    puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+  end
+  i += 1
+end

ext/zstdruby/streaming_compress.c

@@ -58,7 +58,7 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
 
   int compression_level;
   if (NIL_P(compression_level_value)) {
-    compression_level = 0; // The default. See ZSTD_CLEVEL_DEFAULT in zstd_compress.c
+    compression_level = ZSTD_CLEVEL_DEFAULT;
   } else {
     compression_level = NUM2INT(compression_level_value);
   }

ext/zstdruby/streaming_decompress.c

@@ -6,15 +6,6 @@ struct streaming_decompress_t {
   size_t buf_size;
 };
 
-static void
-streaming_decompress_mark(void *p)
-{
-  struct streaming_decompress_t *sd = p;
-  // rb_gc_mark((VALUE)sd->ctx);
-  rb_gc_mark(sd->buf);
-  rb_gc_mark(sd->buf_size);
-}
-
 static void
 streaming_decompress_free(void *p)
 {
@@ -34,7 +25,7 @@ streaming_decompress_memsize(const void *p)
 
 static const rb_data_type_t streaming_decompress_type = {
     "streaming_decompress",
-    { streaming_decompress_mark, streaming_decompress_free, streaming_decompress_memsize, },
+    { 0, streaming_decompress_free, streaming_decompress_memsize, },
      0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };