[MERGE #5982 @MikeHolman] Add JSRT API for externalizing ArrayBuffers

Merge pull request #5982 from MikeHolman:externalizearraypr

This API gives the host control of an ArrayBuffer's lifetime

Author: MikeHolman

bin/ch/ChakraRtInterface.cpp

@@ -200,6 +200,7 @@ bool ChakraRTInterface::LoadChakraDll(ArgInfo* argInfo, HINSTANCE *outLibrary)
 
     m_jsApiHooks.pfJsrtDetachArrayBuffer = (JsAPIHooks::JsrtDetachArrayBufferPtr)GetChakraCoreSymbol(library, "JsDetachArrayBuffer");
     m_jsApiHooks.pfJsrtGetArrayBufferFreeFunction = (JsAPIHooks::JsrtGetArrayBufferFreeFunction)GetChakraCoreSymbol(library, "JsGetArrayBufferFreeFunction");
+    m_jsApiHooks.pfJsrtExternalizeArrayBuffer = (JsAPIHooks::JsrtExternalizeArrayBufferPtr)GetChakraCoreSymbol(library, "JsExternalizeArrayBuffer");
 
 #ifdef _WIN32
     m_jsApiHooks.pfJsrtConnectJITProcess = (JsAPIHooks::JsrtConnectJITProcess)GetChakraCoreSymbol(library, "JsConnectJITProcess");

bin/ch/ChakraRtInterface.h

@@ -138,7 +138,8 @@ struct JsAPIHooks
     typedef JsErrorCode(WINAPI *JsrtVarDeserializerFreePtr)(JsVarDeserializerHandle deserializerHandle);
 
     typedef JsErrorCode(WINAPI *JsrtDetachArrayBufferPtr)(JsValueRef buffer);
-    typedef JsErrorCode(WINAPI* JsrtGetArrayBufferFreeFunction)(JsValueRef buffer, ArrayBufferFreeFn** freeFn);
+    typedef JsErrorCode(WINAPI* JsrtGetArrayBufferFreeFunction)(JsValueRef buffer, ArrayBufferFreeFn* freeFn);
+    typedef JsErrorCode(WINAPI* JsrtExternalizeArrayBufferPtr)(JsValueRef buffer);
 
     JsrtCreateRuntimePtr pfJsrtCreateRuntime;
     JsrtCreateContextPtr pfJsrtCreateContext;
@@ -270,6 +271,7 @@ struct JsAPIHooks
 
     JsrtDetachArrayBufferPtr pfJsrtDetachArrayBuffer;
     JsrtGetArrayBufferFreeFunction pfJsrtGetArrayBufferFreeFunction;
+    JsrtExternalizeArrayBufferPtr pfJsrtExternalizeArrayBuffer;
 #ifdef _WIN32
     JsrtConnectJITProcess pfJsrtConnectJITProcess;
 #endif
@@ -512,7 +514,8 @@ class ChakraRTInterface
     static JsErrorCode WINAPI JsConnectJITProcess(HANDLE processHandle, void* serverSecurityDescriptor, UUID connectionId) { return HOOK_JS_API(ConnectJITProcess(processHandle, serverSecurityDescriptor, connectionId)); }
 #endif
 
-    static JsErrorCode WINAPI JsGetArrayBufferFreeFunction(JsValueRef buffer, ArrayBufferFreeFn** freeFn) { return HOOK_JS_API(GetArrayBufferFreeFunction(buffer, freeFn)); }
+    static JsErrorCode WINAPI JsGetArrayBufferFreeFunction(JsValueRef buffer, ArrayBufferFreeFn* freeFn) { return HOOK_JS_API(GetArrayBufferFreeFunction(buffer, freeFn)); }
+    static JsErrorCode WINAPI JsExternalizeArrayBuffer(JsValueRef buffer) { return HOOK_JS_API(ExternalizeArrayBuffer(buffer)); }
 };
 
 class AutoRestoreContext

bin/ch/WScriptJsrt.cpp

@@ -44,11 +44,16 @@
 #define INTL_LIBRARY_TEXT ""
 #endif
 
+struct ArrayBufferTransferInfo {
+    byte* buffer;
+    uint length;
+    ArrayBufferFreeFn freeFn;
+};
 struct SerializerBlob
 {
     void *data;
     size_t dataLength;
-    std::vector< std::pair<void *, uint32> > transferableArrays;
+    std::vector<ArrayBufferTransferInfo> transferableArrays;
 };
 
 MessageQueue* WScriptJsrt::messageQueue = nullptr;
@@ -371,13 +376,11 @@ JsValueRef __stdcall WScriptJsrt::SerializeObject(JsValueRef callee, bool isCons
         for (int i = 0; i < transferVarsCount; i++)
         {
             JsValueRef arrayBuffer = transferVarsArray[i];
-            BYTE *buffer = nullptr;
-            uint32 bufferLength = 0;
-            IfJsrtErrorSetGo(ChakraRTInterface::JsGetArrayBufferStorage(arrayBuffer, &buffer, &bufferLength));
-            ArrayBufferFreeFn* freeFn = nullptr;
-            IfJsrtErrorSetGo(ChakraRTInterface::JsGetArrayBufferFreeFunction(arrayBuffer, &freeFn));
-
-            blob->transferableArrays.push_back(std::make_pair((void*)buffer, bufferLength));
+            ArrayBufferTransferInfo bufferInfo;
+            IfJsrtErrorSetGo(ChakraRTInterface::JsGetArrayBufferStorage(arrayBuffer, &bufferInfo.buffer, &bufferInfo.length));
+            IfJsrtErrorSetGo(ChakraRTInterface::JsExternalizeArrayBuffer(arrayBuffer));
+            IfJsrtErrorSetGo(ChakraRTInterface::JsGetArrayBufferFreeFunction(arrayBuffer, &bufferInfo.freeFn));
+            blob->transferableArrays.push_back(bufferInfo);
             IfJsrtErrorSetGo(ChakraRTInterface::JsDetachArrayBuffer(arrayBuffer));
         }
 
@@ -412,6 +415,25 @@ JsValueRef CHAKRA_CALLBACK GetWasmModuleFromId(void * state, uint32_t transfer_i
     return nullptr;
 }
 
+struct BufferFreeFunctionState {
+    ArrayBufferFreeFn freeFn;
+    void* buffer;
+};
+
+void CHAKRA_CALLBACK BufferFreeFunction(void * state)
+{
+    BufferFreeFunctionState* bufferState = (BufferFreeFunctionState*)state;
+    if (!bufferState)
+    {
+        return;
+    }
+    if (bufferState->freeFn)
+    {
+        bufferState->freeFn(bufferState->buffer);
+    }
+    delete bufferState;
+}
+
 JsValueRef __stdcall WScriptJsrt::Deserialize(JsValueRef callee, bool isConstructCall, JsValueRef *arguments, unsigned short argumentCount, void *callbackState)
 {
     JsErrorCode errorCode = JsNoError;
@@ -442,7 +464,10 @@ JsValueRef __stdcall WScriptJsrt::Deserialize(JsValueRef callee, bool isConstruc
             for (size_t i = 0; i < arraySize; ++i)
             {
                 JsValueRef result = nullptr;
-                IfJsrtErrorSetGo(ChakraRTInterface::JsCreateExternalArrayBuffer(blob->transferableArrays[i].first, blob->transferableArrays[i].second, nullptr, nullptr, &result));
+                BufferFreeFunctionState* bufferFreeState = new BufferFreeFunctionState();
+                bufferFreeState->buffer = blob->transferableArrays[i].buffer;
+                bufferFreeState->freeFn = blob->transferableArrays[i].freeFn;
+                IfJsrtErrorSetGo(ChakraRTInterface::JsCreateExternalArrayBuffer(blob->transferableArrays[i].buffer, blob->transferableArrays[i].length, BufferFreeFunction, bufferFreeState, &result));
                 transferables[i] = result;
             }
 

lib/Jsrt/ChakraCore.h

@@ -1887,7 +1887,7 @@ CHAKRA_API
 JsDetachArrayBuffer(
     _In_ JsValueRef arrayBuffer);
 
-typedef void __cdecl ArrayBufferFreeFn(void* ptr);
+typedef void(__cdecl *ArrayBufferFreeFn)(void*);
 
 /// <summary>
 ///     Returns the function which free the underlying buffer of ArrayBuffer
@@ -1901,8 +1901,18 @@ typedef void __cdecl ArrayBufferFreeFn(void* ptr);
 CHAKRA_API
 JsGetArrayBufferFreeFunction(
     _In_ JsValueRef arrayBuffer,
-    _Out_ ArrayBufferFreeFn** freeFn);
+    _Out_ ArrayBufferFreeFn* freeFn);
 
+/// <summary>
+///     Take ownership of current ArrayBuffer
+/// </summary>
+/// <param name="arrayBuffer">An ArrayBuffer to take ownership of</param>
+/// <returns>
+///     The code <c>JsNoError</c> if the operation succeeded, a failure code otherwise.
+/// </returns>
+CHAKRA_API
+JsExternalizeArrayBuffer(
+    _In_ JsValueRef arrayBuffer);
 
 #ifdef _WIN32
 #include "ChakraCoreWindows.h"

lib/Jsrt/Core/JsrtCore.cpp

@@ -497,6 +497,25 @@ JsSetArrayBufferExtraInfo(
     END_JSRT_NO_EXCEPTION
 }
 
+CHAKRA_API
+JsExternalizeArrayBuffer(
+    _In_ JsValueRef arrayBufferVar)
+{
+    VALIDATE_JSREF(arrayBufferVar);
+    return ContextAPINoScriptWrapper_NoRecord([&](Js::ScriptContext *scriptContext) -> JsErrorCode {
+
+        Js::ArrayBuffer* arrayBuffer = Js::JavascriptOperators::TryFromVar<Js::ArrayBuffer>(arrayBufferVar);
+        if (!arrayBuffer)
+        {
+            return JsErrorInvalidArgument;
+        }
+
+        arrayBuffer->Externalize();
+
+        return JsNoError;
+    });
+}
+
 CHAKRA_API
 JsDetachArrayBuffer(
     _In_ JsValueRef arrayBuffer)
@@ -1419,7 +1438,7 @@ JsConnectJITProcess(_In_ HANDLE processHandle, _In_opt_ void* serverSecurityDesc
 CHAKRA_API
 JsGetArrayBufferFreeFunction(
     _In_ JsValueRef arrayBuffer,
-    _Out_ ArrayBufferFreeFn** freeFn)
+    _Out_ ArrayBufferFreeFn* freeFn)
 {
   VALIDATE_JSREF(arrayBuffer);
   PARAM_NOT_NULL(freeFn);
@@ -1430,7 +1449,7 @@ JsGetArrayBufferFreeFunction(
           return JsErrorInvalidArgument;
         }
 
-        *freeFn = (ArrayBufferFreeFn*)Js::VarTo<Js::ArrayBuffer>(arrayBuffer)->GetArrayBufferFreeFn();
+        *freeFn = Js::VarTo<Js::ArrayBuffer>(arrayBuffer)->GetArrayBufferFreeFn();
         return JsNoError;
       });
 }

lib/Jsrt/JsrtCommonExports.inc

@@ -122,7 +122,9 @@
     JsCreateWeakReference
     JsDetachArrayBuffer
     JsGetArrayBufferExtraInfo
+    JsExternalizeArrayBuffer
     JsGetAndClearExceptionWithMetadata
+    JsGetArrayBufferFreeFunction
     JsGetArrayEntriesFunction
     JsGetArrayForEachFunction
     JsGetArrayKeysFunction
@@ -168,5 +170,4 @@
     JsVarSerializerSetTransferableVars
     JsVarSerializerWriteRawBytes
     JsVarSerializerWriteValue
-    JsGetArrayBufferFreeFunction
 #endif

lib/Runtime/Library/ArrayBuffer.cpp

@@ -637,7 +637,7 @@ namespace Js
     ArrayBufferContentForDelayedFreeBase* ArrayBuffer::CopyBufferContentForDelayedFree(RefCountedBuffer * content, DECLSPEC_GUARD_OVERFLOW uint32 bufferLength)
     {
         Assert(content != nullptr);
-        FreeFn* freeFn = nullptr;
+        FreeFn freeFn = nullptr;
 #if ENABLE_FAST_ARRAYBUFFER
         if (IsValidVirtualBufferLength(bufferLength))
         {
@@ -831,7 +831,7 @@ namespace Js
         return result;
     }
 
-    ArrayBuffer::FreeFn* JavascriptArrayBuffer::GetArrayBufferFreeFn()
+    ArrayBuffer::FreeFn JavascriptArrayBuffer::GetArrayBufferFreeFn()
     {
 #if ENABLE_FAST_ARRAYBUFFER
         if (IsValidVirtualBufferLength(bufferLength))
@@ -847,7 +847,7 @@ namespace Js
 
     ArrayBufferDetachedStateBase* JavascriptArrayBuffer::CreateDetachedState(RefCountedBuffer * content, uint32 bufferLength)
     {
-        FreeFn* freeFn = nullptr;
+        FreeFn freeFn = nullptr;
         ArrayBufferAllocationType allocationType;
 #if ENABLE_FAST_ARRAYBUFFER
         if (IsValidVirtualBufferLength(bufferLength))
@@ -914,14 +914,14 @@ namespace Js
         if (refCount == 0)
         {
             BYTE * buffer = content->GetBuffer();
-            if (buffer)
+            if (buffer && !this->externalized)
             {
                 // Recycler may not be available at Dispose. We need to
                 // free the memory and report that it has been freed at the same
                 // time. Otherwise, AllocationPolicyManager is unable to provide correct feedback
 #if ENABLE_FAST_ARRAYBUFFER
         //AsmJS Virtual Free
-                if (buffer && IsValidVirtualBufferLength(this->bufferLength))
+                if (IsValidVirtualBufferLength(this->bufferLength))
                 {
                     FreeMemAlloc(buffer);
                 }

lib/Runtime/Library/ArrayBuffer.h

@@ -60,13 +60,13 @@ namespace Js
             return AllocWrapper(length, MaxVirtualSize);
         }
 
-        static void FreeMemAlloc(Var ptr)
+        static void __cdecl FreeMemAlloc(Var ptr)
         {
             BOOL fSuccess = VirtualFree((LPVOID)ptr, 0, MEM_RELEASE);
             Assert(fSuccess);
         }
 #else
-        static void FreeMemAlloc(Var ptr)
+        static void __cdecl FreeMemAlloc(Var ptr)
         {
             // This free function should never be used
             Js::Throw::FatalInternalError();
@@ -77,7 +77,12 @@ namespace Js
 
         virtual void MarshalToScriptContext(Js::ScriptContext * scriptContext) = 0;
 
-        ArrayBufferBase(DynamicType *type) : DynamicObject(type), isDetached(false), infoBits(0) { }
+        ArrayBufferBase(DynamicType *type) :
+            DynamicObject(type),
+            isDetached(false),
+            infoBits(0),
+            externalized(false) { }
+
         bool IsDetached() { return isDetached; }
 
 #if ENABLE_TTD
@@ -99,8 +104,10 @@ namespace Js
 
         static int GetIsDetachedOffset() { return offsetof(ArrayBufferBase, isDetached); }
 
+        void Externalize() { this->externalized = true; }
     protected:
         Field(bool) isDetached;
+        Field(bool) externalized;
         Field(char) infoBits;
     };
 
@@ -142,9 +149,9 @@ namespace Js
         class ArrayBufferDetachedState : public ArrayBufferDetachedStateBase
         {
         public:
-            FreeFN* freeFunction;
+            FreeFN freeFunction;
             Recycler* recycler;
-            ArrayBufferDetachedState(RefCountedBuffer* buffer, uint32 bufferLength, FreeFN* freeFunction, Recycler* r, ArrayBufferAllocationType allocationType)
+            ArrayBufferDetachedState(RefCountedBuffer* buffer, uint32 bufferLength, FreeFN freeFunction, Recycler* r, ArrayBufferAllocationType allocationType)
                 : ArrayBufferDetachedStateBase(TypeIds_ArrayBuffer, buffer, bufferLength, allocationType),
                 recycler(r),
                 freeFunction(freeFunction)
@@ -206,8 +213,8 @@ namespace Js
         RefCountedBuffer *GetBufferContent() { return bufferContent;  }
         static int GetBufferContentsOffset() { return offsetof(ArrayBuffer, bufferContent); }
 
-        typedef void __cdecl FreeFn(void* ptr);
-        virtual FreeFn* GetArrayBufferFreeFn() { return nullptr; }
+        typedef void(__cdecl *FreeFn)(void*);
+        virtual FreeFn GetArrayBufferFreeFn() { return nullptr; }
         static int GetByteLengthOffset() { return offsetof(ArrayBuffer, bufferLength); }
         virtual void AddParent(ArrayBufferParent* parent) override;
 
@@ -310,7 +317,7 @@ namespace Js
         virtual bool IsValidAsmJsBufferLength(uint length, bool forceCheck = false) override;
         virtual bool IsValidVirtualBufferLength(uint length) const override;
 
-        virtual FreeFn* GetArrayBufferFreeFn() override;
+        virtual FreeFn GetArrayBufferFreeFn() override;
 
        protected:
         JavascriptArrayBuffer(DynamicType * type);
@@ -358,7 +365,7 @@ namespace Js
         DEFINE_VTABLE_CTOR(ProjectionArrayBuffer, ArrayBuffer);
         DEFINE_MARSHAL_OBJECT_TO_SCRIPT_CONTEXT(ProjectionArrayBuffer);
 
-        typedef void __stdcall FreeFn(LPVOID ptr);
+        typedef void (__stdcall *FreeFn)(LPVOID ptr);
         virtual ArrayBufferDetachedStateBase* CreateDetachedState(RefCountedBuffer * content, DECLSPEC_GUARD_OVERFLOW uint32 bufferLength) override
         {
             return HeapNew(ArrayBufferDetachedState<FreeFn>, content, bufferLength, CoTaskMemFree, GetScriptContext()->GetRecycler(), ArrayBufferAllocationType::CoTask);

lib/Runtime/Library/DelayFreeArrayBufferHelper.h

@@ -93,10 +93,10 @@ namespace Js
     template <typename FreeFN>
     class ArrayBufferContentForDelayedFree : public ArrayBufferContentForDelayedFreeBase
     {
-        FreeFN* freeFunction;
+        FreeFN freeFunction;
 
     public:
-        ArrayBufferContentForDelayedFree(RefCountedBuffer *content, uint32 len, Recycler *r, FreeFN * freeFunction)
+        ArrayBufferContentForDelayedFree(RefCountedBuffer *content, uint32 len, Recycler *r, FreeFN freeFunction)
             : ArrayBufferContentForDelayedFreeBase( content, len, r), freeFunction(freeFunction)
         {}