Handle binary (base64-encoded) files (#315)

Author: davidbrochart

jupyverse_api/jupyverse_api/contents/__init__.py

@@ -113,7 +113,7 @@ def file_id_manager(self) -> FileIdManager:
 
     @abstractmethod
     async def read_content(
-        self, path: Union[str, Path], get_content: bool, as_json: bool = False
+        self, path: Union[str, Path], get_content: bool, file_format: Optional[str] = None
     ) -> Content:
         ...
 

plugins/contents/fps_contents/routes.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import os
 import shutil
@@ -137,7 +138,7 @@ async def rename_content(
         return await self.read_content(rename_content.path, False)
 
     async def read_content(
-        self, path: Union[str, Path], get_content: bool, as_json: bool = False
+        self, path: Union[str, Path], get_content: bool, file_format: Optional[str] = None
     ) -> Content:
         if isinstance(path, str):
             path = Path(path)
@@ -151,10 +152,14 @@ async def read_content(
                 ]
             elif path.is_file() or path.is_symlink():
                 try:
-                    async with await open_file(path) as f:
-                        content = await f.read()
-                    if as_json:
-                        content = json.loads(content)
+                    async with await open_file(path, mode="rb") as f:
+                        content_bytes = await f.read()
+                    if file_format == "base64":
+                        content = base64.b64encode(content_bytes).decode("ascii")
+                    elif file_format == "json":
+                        content = json.loads(content_bytes)
+                    else:
+                        content = content_bytes.decode()
                 except Exception:
                     raise HTTPException(status_code=404, detail="Item not found")
         format: Optional[str] = None
@@ -171,7 +176,7 @@ async def read_content(
                 mimetype = None
                 if content is not None:
                     nb: dict
-                    if as_json:
+                    if file_format == "json":
                         content = cast(Dict, content)
                         nb = content
                     else:
@@ -181,7 +186,7 @@ async def read_content(
                         if "metadata" not in cell:
                             cell["metadata"] = {}
                         cell["metadata"].update({"trusted": False})
-                    if not as_json:
+                    if file_format != "json":
                         content = json.dumps(nb)
             elif path.suffix == ".json":
                 type = "json"
@@ -212,24 +217,33 @@ async def read_content(
     async def write_content(self, content: Union[SaveContent, Dict]) -> None:
         if not isinstance(content, SaveContent):
             content = SaveContent(**content)
-        async with await open_file(content.path, "w") as f:
-            if content.format == "json":
-                dict_content = cast(Dict, content.content)
-                if content.type == "notebook":
-                    # see https://github.com/jupyterlab/jupyterlab/issues/11005
-                    if "metadata" in dict_content and "orig_nbformat" in dict_content["metadata"]:
-                        del dict_content["metadata"]["orig_nbformat"]
-                await f.write(json.dumps(dict_content, indent=2))
-            else:
+        if content.format == "base64":
+            async with await open_file(content.path, "wb") as f:
                 content.content = cast(str, content.content)
-                await f.write(content.content)
+                content_bytes = content.content.encode("ascii")
+                await f.write(content_bytes)
+        else:
+            async with await open_file(content.path, "wt") as f:
+                if content.format == "json":
+                    dict_content = cast(Dict, content.content)
+                    if content.type == "notebook":
+                        # see https://github.com/jupyterlab/jupyterlab/issues/11005
+                        if (
+                            "metadata" in dict_content
+                            and "orig_nbformat" in dict_content["metadata"]
+                        ):
+                            del dict_content["metadata"]["orig_nbformat"]
+                    await f.write(json.dumps(dict_content, indent=2))
+                else:
+                    content.content = cast(str, content.content)
+                    await f.write(content.content)
 
     @property
     def file_id_manager(self):
         return FileIdManager()
 
 
-def get_available_path(path: Path, sep: str = ""):
+def get_available_path(path: Path, sep: str = "") -> Path:
     directory = path.parent
     name = Path(path.name)
     i = None

plugins/webdav/fps_webdav/py.typed

@@ -175,9 +175,8 @@ async def serve(self, websocket: YpyWebsocket, permissions) -> None:
                 logger.info(f"Opening collaboration room: {websocket.path} ({file_path})")
                 document = YDOCS.get(file_type, YFILE)(room.ydoc)
                 self.documents[websocket.path] = document
-                is_notebook = file_type == "notebook"
                 async with self.lock:
-                    model = await self.contents.read_content(file_path, True, as_json=is_notebook)
+                    model = await self.contents.read_content(file_path, True, file_format)
                 assert model.last_modified is not None
                 self.last_modified[file_id] = to_datetime(model.last_modified)
                 if not room.ready:
@@ -201,11 +200,13 @@ async def serve(self, websocket: YpyWebsocket, permissions) -> None:
                     document.dirty = False
                     room.ready = True
                     # save the document to file when changed
-                    document.observe(partial(self.on_document_change, file_id, file_type, document))
+                    document.observe(
+                        partial(self.on_document_change, file_id, file_type, file_format, document)
+                    )
                     # update the document when file changes
                     if file_id not in self.watchers:
                         self.watchers[file_id] = asyncio.create_task(
-                            self.watch_file(file_type, file_id, document)
+                            self.watch_file(file_format, file_id, document)
                         )
 
         await self.websocket_server.serve(websocket)
@@ -245,7 +246,7 @@ async def get_file_path(self, file_id: str, document) -> str | None:
             document.path = file_path
         return file_path
 
-    async def watch_file(self, file_type: str, file_id: str, document: YBaseDoc) -> None:
+    async def watch_file(self, file_format: str, file_id: str, document: YBaseDoc) -> None:
         file_path = await self.get_file_path(file_id, document)
         assert file_path is not None
         logger.debug(f"Watching file: {file_path}")
@@ -260,26 +261,25 @@ async def watch_file(self, file_type: str, file_id: str, document: YBaseDoc) ->
                     self.contents.file_id_manager.unwatch(file_path, watcher)
                     file_path = new_file_path
                     # break
-                await self.maybe_load_file(file_type, file_path, file_id)
+                await self.maybe_load_file(file_format, file_path, file_id)
 
-    async def maybe_load_file(self, file_type: str, file_path: str, file_id: str) -> None:
+    async def maybe_load_file(self, file_format: str, file_path: str, file_id: str) -> None:
         async with self.lock:
             model = await self.contents.read_content(file_path, False)
         # do nothing if the file was saved by us
         assert model.last_modified is not None
         if self.last_modified[file_id] < to_datetime(model.last_modified):
             # the file was not saved by us, update the shared document(s)
-            is_notebook = file_type == "notebook"
             async with self.lock:
-                model = await self.contents.read_content(file_path, True, as_json=is_notebook)
+                model = await self.contents.read_content(file_path, True, file_format)
             assert model.last_modified is not None
             documents = [v for k, v in self.documents.items() if k.split(":", 2)[2] == file_id]
             for document in documents:
                 document.source = model.content
             self.last_modified[file_id] = to_datetime(model.last_modified)
 
     def on_document_change(
-        self, file_id: str, file_type: str, document: YBaseDoc, target, event
+        self, file_id: str, file_type: str, file_format: str, document: YBaseDoc, target, event
     ) -> None:
         if target == "state" and "dirty" in event.keys:
             dirty = event.keys["dirty"]["newValue"]
@@ -289,14 +289,18 @@ def on_document_change(
         # unobserve and observe again because the structure of the document may have changed
         # e.g. a new cell added to a notebook
         document.unobserve()
-        document.observe(partial(self.on_document_change, file_id, file_type, document))
+        document.observe(
+            partial(self.on_document_change, file_id, file_type, file_format, document)
+        )
         if file_id in self.savers:
             self.savers[file_id].cancel()
         self.savers[file_id] = asyncio.create_task(
-            self.maybe_save_document(file_id, file_type, document)
+            self.maybe_save_document(file_id, file_type, file_format, document)
         )
 
-    async def maybe_save_document(self, file_id: str, file_type: str, document: YBaseDoc) -> None:
+    async def maybe_save_document(
+        self, file_id: str, file_type: str, file_format: str, document: YBaseDoc
+    ) -> None:
         # save after 1 second of inactivity to prevent too frequent saving
         await asyncio.sleep(1)  # FIXME: pass in config
         # if the room cannot be found, don't save
@@ -305,9 +309,8 @@ async def maybe_save_document(self, file_id: str, file_type: str, document: YBas
         except BaseException:
             return
         assert file_path is not None
-        is_notebook = file_type == "notebook"
         async with self.lock:
-            model = await self.contents.read_content(file_path, True, as_json=is_notebook)
+            model = await self.contents.read_content(file_path, True, file_format)
         assert model.last_modified is not None
         if self.last_modified[file_id] < to_datetime(model.last_modified):
             # file changed on disk, let's revert
@@ -318,10 +321,9 @@ async def maybe_save_document(self, file_id: str, file_type: str, document: YBas
             # don't save if not needed
             # this also prevents the dirty flag from bouncing between windows of
             # the same document opened as different types (e.g. notebook/text editor)
-            format = "json" if file_type == "notebook" else "text"
             content = {
                 "content": document.source,
-                "format": format,
+                "format": file_format,
                 "path": file_path,
                 "type": file_type,
             }