compact marshal encoding scheme

Author: bobzhang

jscomp/core/bs_cmi_load.ml

@@ -38,8 +38,8 @@ let load_cmi ~unit_name : Env.Persistent_signature.t option =
         if Js_config.get_diagnose () then
           Format.fprintf Format.err_formatter ">Cmi: %s@." unit_name;
         let cmi : Cmi_format.cmi_infos = 
-            Marshal.from_string 
-            Builtin_cmi_datasets.module_data.(cmi) 0 in   
+            Ext_marshal.from_string_uncheck
+            Builtin_cmi_datasets.module_data.(cmi)  in   
         if Js_config.get_diagnose () then
           Format.fprintf Format.err_formatter "<Cmi: %s@." unit_name;
         Some {filename = Sys.executable_name ; 

jscomp/core/js_cmj_load.ml

@@ -42,7 +42,7 @@ then
     if Js_config.get_diagnose () then
       Format.fprintf Format.err_formatter ">Cmj: %s@." unit_name;
     let cmj_table : Js_cmj_format.t = 
-       let values, pure =  Marshal.from_string Builtin_cmj_datasets.module_data.(i) 0 in   
+       let values, pure =  Ext_marshal.from_string_uncheck Builtin_cmj_datasets.module_data.(i)  in   
        {values; pure; package_spec = Js_packages_info.runtime_package_specs;js_file_kind = Little_js} 
     in 
     if Js_config.get_diagnose () then

jscomp/ext/ext_marshal.ml

@@ -28,7 +28,7 @@
 
 
 
-
+(* 
 let to_file filename v = 
   let chan = open_out_bin filename in
   Marshal.to_channel chan v  [];
@@ -39,5 +39,10 @@ let from_file filename =
   let chan = open_in_bin filename in 
   let v = Marshal.from_channel chan in
   close_in chan; 
-  v 
-  
+  v  *)
+
+external from_bytes_unsafe: string -> int -> 'a
+  = "caml_input_value_from_string"
+
+let from_string_uncheck (s:string) = 
+  from_bytes_unsafe s 0  

jscomp/ext/ext_marshal.mli

@@ -32,6 +32,8 @@
 (** Extension to the standard library [Marshall] module 
  *)
 
-val to_file : string -> 'a -> unit
+(* val to_file : string -> 'a -> unit *)
 
-val from_file : string -> 'a
+(* val from_file : string -> 'a *)
+
+val from_string_uncheck : string -> 'a

jscomp/main/builtin_cmi_datasets.ml

@@ -56,7 +56,7 @@ let toString (x : t) =
 
 (* exception handling*)
 let fromString (x : string) : t =   
-    Marshal.from_string x 0
+    Ext_marshal.from_string_uncheck x 
 
 let handle_extension record_as_js_object e (self : Bs_ast_mapper.mapper)
     (({txt ; loc} as lid , payload) : Parsetree.extension) = 

jscomp/syntax/ast_exp_extension.ml

@@ -235,31 +235,32 @@ let check_ffi ?loc ffi : bool =
   end; 
   !xrelative
 
-let bs_prefix = "BS:"
+(* let bs_prefix = "BS:"
 let bs_prefix_length = String.length bs_prefix
-
+ *)
 
 (** TODO: Make sure each version is not prefix of each other
     Solution:
     1. fixed length
     2. non-prefix approach
 *)
-let bs_external = bs_prefix 
+(* let bs_external = bs_prefix  *)
 
 
-let bs_external_length = String.length bs_external
+(* let bs_external_length = String.length bs_external *)
 
 
 let to_string  (t : t) =
-  bs_external ^ Marshal.to_string t []
-
-let is_bs_primitive s =
-  let s_len = String.length s in
-   s_len >= bs_prefix_length &&
-     String.unsafe_get s 0 = 'B' &&
-     String.unsafe_get s 1 = 'S' &&
-     String.unsafe_get s 2 = ':' 
-     
+  Marshal.to_string t []
+
+(* \132\149\166\190 
+   0x84 95 A6 BE Intext_magic_small intext.h
+   https://github.com/ocaml/merlin/commit/b094c937c3a360eb61054f7652081b88e4f3612f
+*)
+let is_bs_primitive s =  
+   String.length s >= 20 (* Marshal.header_size*) &&
+     String.unsafe_get s 0 = '\132' &&
+     String.unsafe_get s 1 = '\149' 
 
 let () = Oprint.map_primitive_name := 
 #if BS_RELEASE_BUILD then  
@@ -273,7 +274,7 @@ let () = Oprint.map_primitive_name :=
 (* TODO:  better error message when version mismatch *)
 let from_string s : t =
   if is_bs_primitive s  then   
-    Marshal.from_string s bs_external_length
+    Ext_marshal.from_string_uncheck s
   else Ffi_normal
 
 

jscomp/syntax/external_ffi_types.ml

@@ -1 +1 @@
-../lib/4.06.1/bsdep.ml: ../ocaml/driver/compdynlink.mli ../ocaml/driver/compenv.ml ../ocaml/driver/compenv.mli ../ocaml/driver/compmisc.ml ../ocaml/driver/compmisc.mli ../ocaml/driver/compplugin.ml ../ocaml/driver/compplugin.mli ../ocaml/driver/makedepend.ml ../ocaml/driver/makedepend.mli ../ocaml/driver/pparse.ml ../ocaml/driver/pparse.mli ../ocaml/parsing/ast_helper.ml ../ocaml/parsing/ast_helper.mli ../ocaml/parsing/ast_invariants.ml ../ocaml/parsing/ast_invariants.mli ../ocaml/parsing/ast_iterator.ml ../ocaml/parsing/ast_iterator.mli ../ocaml/parsing/ast_mapper.ml ../ocaml/parsing/ast_mapper.mli ../ocaml/parsing/asttypes.mli ../ocaml/parsing/builtin_attributes.ml ../ocaml/parsing/builtin_attributes.mli ../ocaml/parsing/depend.ml ../ocaml/parsing/depend.mli ../ocaml/parsing/docstrings.ml ../ocaml/parsing/docstrings.mli ../ocaml/parsing/lexer.ml ../ocaml/parsing/lexer.mli ../ocaml/parsing/location.ml ../ocaml/parsing/location.mli ../ocaml/parsing/longident.ml ../ocaml/parsing/longident.mli ../ocaml/parsing/parse.ml ../ocaml/parsing/parse.mli ../ocaml/parsing/parser.ml ../ocaml/parsing/parser.mli ../ocaml/parsing/parsetree.mli ../ocaml/parsing/syntaxerr.ml ../ocaml/parsing/syntaxerr.mli ../ocaml/tools/ocamldep.ml ../ocaml/utils/arg_helper.ml ../ocaml/utils/arg_helper.mli ../ocaml/utils/ccomp.ml ../ocaml/utils/ccomp.mli ../ocaml/utils/clflags.ml ../ocaml/utils/clflags.mli ../ocaml/utils/identifiable.ml ../ocaml/utils/identifiable.mli ../ocaml/utils/misc.ml ../ocaml/utils/misc.mli ../ocaml/utils/numbers.ml ../ocaml/utils/numbers.mli ../ocaml/utils/profile.ml ../ocaml/utils/profile.mli ../ocaml/utils/terminfo.ml ../ocaml/utils/terminfo.mli ../ocaml/utils/warnings.ml ../ocaml/utils/warnings.mli ./common/bs_loc.ml ./common/bs_loc.mli ./common/bs_version.ml ./common/bs_version.mli ./common/bs_warnings.ml ./common/bs_warnings.mli ./common/js_config.ml ./common/js_config.mli ./common/lam_methname.ml ./common/lam_methname.mli ./core/bs_cmi_load.ml ./core/bs_conditional_initial.ml ./core/bs_conditional_initial.mli ./core/config_util.ml ./core/config_util.mli ./core/config_whole_compiler.ml ./core/config_whole_compiler.mli ./core/js_raw_info.ml ./core/lam_constant.ml ./core/lam_constant.mli ./core/lam_pointer_info.ml ./core/lam_pointer_info.mli ./core/lam_tag_info.ml ./core/matching_polyfill.ml ./core/matching_polyfill.mli ./core/record_attributes_check.ml ./ext/bsc_warnings.ml ./ext/ext_array.ml ./ext/ext_array.mli ./ext/ext_bytes.ml ./ext/ext_bytes.mli ./ext/ext_char.ml ./ext/ext_char.mli ./ext/ext_filename.ml ./ext/ext_filename.mli ./ext/ext_json_parse.ml ./ext/ext_json_parse.mli ./ext/ext_json_types.ml ./ext/ext_list.ml ./ext/ext_list.mli ./ext/ext_option.ml ./ext/ext_option.mli ./ext/ext_position.ml ./ext/ext_position.mli ./ext/ext_ref.ml ./ext/ext_ref.mli ./ext/ext_string.ml ./ext/ext_string.mli ./ext/ext_string_array.ml ./ext/ext_string_array.mli ./ext/ext_utf8.ml ./ext/ext_utf8.mli ./ext/ext_util.ml ./ext/ext_util.mli ./ext/hash_set_gen.ml ./ext/hash_set_poly.ml ./ext/hash_set_poly.mli ./ext/hash_set_string.ml ./ext/hash_set_string.mli ./ext/literals.ml ./ext/literals.mli ./ext/map_gen.ml ./ext/map_string.ml ./ext/map_string.mli ./ext/set_gen.ml ./ext/set_string.ml ./ext/set_string.mli ./main/builtin_cmi_datasets.ml ./main/builtin_cmi_datasets.mli ./stubs/bs_hash_stubs.ml ./syntax/ast_attributes.ml ./syntax/ast_attributes.mli ./syntax/ast_bs_open.ml ./syntax/ast_bs_open.mli ./syntax/ast_comb.ml ./syntax/ast_comb.mli ./syntax/ast_compatible.ml ./syntax/ast_compatible.mli ./syntax/ast_config.ml ./syntax/ast_config.mli ./syntax/ast_core_type.ml ./syntax/ast_core_type.mli ./syntax/ast_core_type_class_type.ml ./syntax/ast_core_type_class_type.mli ./syntax/ast_derive.ml ./syntax/ast_derive.mli ./syntax/ast_derive_abstract.ml ./syntax/ast_derive_abstract.mli ./syntax/ast_derive_js_mapper.ml ./syntax/ast_derive_js_mapper.mli ./syntax/ast_derive_projector.ml ./syntax/ast_derive_projector.mli ./syntax/ast_derive_util.ml ./syntax/ast_derive_util.mli ./syntax/ast_exp.ml ./syntax/ast_exp.mli ./syntax/ast_exp_apply.ml ./syntax/ast_exp_apply.mli ./syntax/ast_exp_extension.ml ./syntax/ast_exp_extension.mli ./syntax/ast_exp_handle_external.ml ./syntax/ast_exp_handle_external.mli ./syntax/ast_external.ml ./syntax/ast_external.mli ./syntax/ast_external_mk.ml ./syntax/ast_external_mk.mli ./syntax/ast_external_process.ml ./syntax/ast_external_process.mli ./syntax/ast_literal.ml ./syntax/ast_literal.mli ./syntax/ast_open_cxt.ml ./syntax/ast_open_cxt.mli ./syntax/ast_pat.ml ./syntax/ast_pat.mli ./syntax/ast_payload.ml ./syntax/ast_payload.mli ./syntax/ast_polyvar.ml ./syntax/ast_polyvar.mli ./syntax/ast_signature.ml ./syntax/ast_signature.mli ./syntax/ast_structure.ml ./syntax/ast_structure.mli ./syntax/ast_tdcls.ml ./syntax/ast_tdcls.mli ./syntax/ast_tuple_pattern_flatten.ml ./syntax/ast_tuple_pattern_flatten.mli ./syntax/ast_typ_uncurry.ml ./syntax/ast_typ_uncurry.mli ./syntax/ast_utf8_string.ml ./syntax/ast_utf8_string.mli ./syntax/ast_utf8_string_interp.ml ./syntax/ast_utf8_string_interp.mli ./syntax/ast_util.ml ./syntax/ast_util.mli ./syntax/bs_ast_invariant.ml ./syntax/bs_ast_invariant.mli ./syntax/bs_ast_mapper.ml ./syntax/bs_ast_mapper.mli ./syntax/bs_builtin_ppx.ml ./syntax/bs_builtin_ppx.mli ./syntax/bs_syntaxerr.ml ./syntax/bs_syntaxerr.mli ./syntax/external_arg_spec.ml ./syntax/external_arg_spec.mli ./syntax/external_ffi_types.ml ./syntax/external_ffi_types.mli ./syntax/ppx_entry.ml ./syntax/reactjs_jsx_ppx_v2.ml ./syntax/reactjs_jsx_ppx_v3.ml 
+../lib/4.06.1/bsdep.ml: ../ocaml/driver/compdynlink.mli ../ocaml/driver/compenv.ml ../ocaml/driver/compenv.mli ../ocaml/driver/compmisc.ml ../ocaml/driver/compmisc.mli ../ocaml/driver/compplugin.ml ../ocaml/driver/compplugin.mli ../ocaml/driver/makedepend.ml ../ocaml/driver/makedepend.mli ../ocaml/driver/pparse.ml ../ocaml/driver/pparse.mli ../ocaml/parsing/ast_helper.ml ../ocaml/parsing/ast_helper.mli ../ocaml/parsing/ast_invariants.ml ../ocaml/parsing/ast_invariants.mli ../ocaml/parsing/ast_iterator.ml ../ocaml/parsing/ast_iterator.mli ../ocaml/parsing/ast_mapper.ml ../ocaml/parsing/ast_mapper.mli ../ocaml/parsing/asttypes.mli ../ocaml/parsing/builtin_attributes.ml ../ocaml/parsing/builtin_attributes.mli ../ocaml/parsing/depend.ml ../ocaml/parsing/depend.mli ../ocaml/parsing/docstrings.ml ../ocaml/parsing/docstrings.mli ../ocaml/parsing/lexer.ml ../ocaml/parsing/lexer.mli ../ocaml/parsing/location.ml ../ocaml/parsing/location.mli ../ocaml/parsing/longident.ml ../ocaml/parsing/longident.mli ../ocaml/parsing/parse.ml ../ocaml/parsing/parse.mli ../ocaml/parsing/parser.ml ../ocaml/parsing/parser.mli ../ocaml/parsing/parsetree.mli ../ocaml/parsing/syntaxerr.ml ../ocaml/parsing/syntaxerr.mli ../ocaml/tools/ocamldep.ml ../ocaml/utils/arg_helper.ml ../ocaml/utils/arg_helper.mli ../ocaml/utils/ccomp.ml ../ocaml/utils/ccomp.mli ../ocaml/utils/clflags.ml ../ocaml/utils/clflags.mli ../ocaml/utils/identifiable.ml ../ocaml/utils/identifiable.mli ../ocaml/utils/misc.ml ../ocaml/utils/misc.mli ../ocaml/utils/numbers.ml ../ocaml/utils/numbers.mli ../ocaml/utils/profile.ml ../ocaml/utils/profile.mli ../ocaml/utils/terminfo.ml ../ocaml/utils/terminfo.mli ../ocaml/utils/warnings.ml ../ocaml/utils/warnings.mli ./common/bs_loc.ml ./common/bs_loc.mli ./common/bs_version.ml ./common/bs_version.mli ./common/bs_warnings.ml ./common/bs_warnings.mli ./common/js_config.ml ./common/js_config.mli ./common/lam_methname.ml ./common/lam_methname.mli ./core/bs_cmi_load.ml ./core/bs_conditional_initial.ml ./core/bs_conditional_initial.mli ./core/config_util.ml ./core/config_util.mli ./core/config_whole_compiler.ml ./core/config_whole_compiler.mli ./core/js_raw_info.ml ./core/lam_constant.ml ./core/lam_constant.mli ./core/lam_pointer_info.ml ./core/lam_pointer_info.mli ./core/lam_tag_info.ml ./core/matching_polyfill.ml ./core/matching_polyfill.mli ./core/record_attributes_check.ml ./ext/bsc_warnings.ml ./ext/ext_array.ml ./ext/ext_array.mli ./ext/ext_bytes.ml ./ext/ext_bytes.mli ./ext/ext_char.ml ./ext/ext_char.mli ./ext/ext_filename.ml ./ext/ext_filename.mli ./ext/ext_json_parse.ml ./ext/ext_json_parse.mli ./ext/ext_json_types.ml ./ext/ext_list.ml ./ext/ext_list.mli ./ext/ext_marshal.ml ./ext/ext_marshal.mli ./ext/ext_option.ml ./ext/ext_option.mli ./ext/ext_position.ml ./ext/ext_position.mli ./ext/ext_ref.ml ./ext/ext_ref.mli ./ext/ext_string.ml ./ext/ext_string.mli ./ext/ext_string_array.ml ./ext/ext_string_array.mli ./ext/ext_utf8.ml ./ext/ext_utf8.mli ./ext/ext_util.ml ./ext/ext_util.mli ./ext/hash_set_gen.ml ./ext/hash_set_poly.ml ./ext/hash_set_poly.mli ./ext/hash_set_string.ml ./ext/hash_set_string.mli ./ext/literals.ml ./ext/literals.mli ./ext/map_gen.ml ./ext/map_string.ml ./ext/map_string.mli ./ext/set_gen.ml ./ext/set_string.ml ./ext/set_string.mli ./main/builtin_cmi_datasets.ml ./main/builtin_cmi_datasets.mli ./stubs/bs_hash_stubs.ml ./syntax/ast_attributes.ml ./syntax/ast_attributes.mli ./syntax/ast_bs_open.ml ./syntax/ast_bs_open.mli ./syntax/ast_comb.ml ./syntax/ast_comb.mli ./syntax/ast_compatible.ml ./syntax/ast_compatible.mli ./syntax/ast_config.ml ./syntax/ast_config.mli ./syntax/ast_core_type.ml ./syntax/ast_core_type.mli ./syntax/ast_core_type_class_type.ml ./syntax/ast_core_type_class_type.mli ./syntax/ast_derive.ml ./syntax/ast_derive.mli ./syntax/ast_derive_abstract.ml ./syntax/ast_derive_abstract.mli ./syntax/ast_derive_js_mapper.ml ./syntax/ast_derive_js_mapper.mli ./syntax/ast_derive_projector.ml ./syntax/ast_derive_projector.mli ./syntax/ast_derive_util.ml ./syntax/ast_derive_util.mli ./syntax/ast_exp.ml ./syntax/ast_exp.mli ./syntax/ast_exp_apply.ml ./syntax/ast_exp_apply.mli ./syntax/ast_exp_extension.ml ./syntax/ast_exp_extension.mli ./syntax/ast_exp_handle_external.ml ./syntax/ast_exp_handle_external.mli ./syntax/ast_external.ml ./syntax/ast_external.mli ./syntax/ast_external_mk.ml ./syntax/ast_external_mk.mli ./syntax/ast_external_process.ml ./syntax/ast_external_process.mli ./syntax/ast_literal.ml ./syntax/ast_literal.mli ./syntax/ast_open_cxt.ml ./syntax/ast_open_cxt.mli ./syntax/ast_pat.ml ./syntax/ast_pat.mli ./syntax/ast_payload.ml ./syntax/ast_payload.mli ./syntax/ast_polyvar.ml ./syntax/ast_polyvar.mli ./syntax/ast_signature.ml ./syntax/ast_signature.mli ./syntax/ast_structure.ml ./syntax/ast_structure.mli ./syntax/ast_tdcls.ml ./syntax/ast_tdcls.mli ./syntax/ast_tuple_pattern_flatten.ml ./syntax/ast_tuple_pattern_flatten.mli ./syntax/ast_typ_uncurry.ml ./syntax/ast_typ_uncurry.mli ./syntax/ast_utf8_string.ml ./syntax/ast_utf8_string.mli ./syntax/ast_utf8_string_interp.ml ./syntax/ast_utf8_string_interp.mli ./syntax/ast_util.ml ./syntax/ast_util.mli ./syntax/bs_ast_invariant.ml ./syntax/bs_ast_invariant.mli ./syntax/bs_ast_mapper.ml ./syntax/bs_ast_mapper.mli ./syntax/bs_builtin_ppx.ml ./syntax/bs_builtin_ppx.mli ./syntax/bs_syntaxerr.ml ./syntax/bs_syntaxerr.mli ./syntax/external_arg_spec.ml ./syntax/external_arg_spec.mli ./syntax/external_ffi_types.ml ./syntax/external_ffi_types.mli ./syntax/ppx_entry.ml ./syntax/reactjs_jsx_ppx_v2.ml ./syntax/reactjs_jsx_ppx_v3.ml