Fixex to \FORMATBYTESTRING and \FORMATBYTESTREAM (#1618)

Motivated by the fact that FILEPOS was not being properly informed that the bytes of the encoded stream are stable (unlike XCCS) so it can run at the byte level without character decoding.

Author: rmkaplan

sources/EXTERNALFORMAT

@@ -1,12 +1,12 @@
 (DEFINE-FILE-INFO PACKAGE "INTERLISP" READTABLE "INTERLISP" BASE 10)
 
-
(FILECREATED "12-Jan-2024 10:59:18" {DSK}<home>larry>il>medley>sources>EXTERNALFORMAT.;3
 38380  
+
(FILECREATED "19-Mar-2024 18:24:39" {WMEDLEY}<sources>EXTERNALFORMAT.;88
 38921  
 
-      :EDIT-BY "lmm"
+      :EDIT-BY rmk
 
-      :CHANGES-TO (FNS \BACKCCODE)
+      :CHANGES-TO (FNS \FORMATBYTESTRING \FORMATBYTESTREAM)
 
-      :PREVIOUS-DATE " 8-Dec-2023 22:02:21" {DSK}<home>larry>il>medley>sources>EXTERNALFORMAT.;1)
+      :PREVIOUS-DATE "12-Jan-2024 10:59:18" {WMEDLEY}<sources>EXTERNALFORMAT.;86)
 
 
 (PRETTYCOMPRINT EXTERNALFORMATCOMS)
@@ -472,6 +472,10 @@
 (\FORMATBYTESTREAM

   [LAMBDA (STREAM BYTESTREAM)
 
+    (* ;; "Edited 19-Mar-2024 15:57 by rmk: Remove installation of STREAM's EXTERNALFORMAT into BYTESTREAM.  That should be done by the formats FORMATBYTESTREAMFN.  In particular,")

+
+    (* ;; "Edited 19-Mar-2024 14:31 by rmk")

+
     (* ;; "Edited 24-Jul-2022 08:30 by rmk: STREAM can be the external format to be used for BYTESTREAM, not just a carrier of that format")

 
     (* ;; "Edited 22-Jun-2022 11:09 by rmk")

@@ -495,27 +499,33 @@
            ELSEIF
 (TYPE?
 EXTERNALFORMAT STREAM)
              THEN
 (SETQ FORMAT STREAM)
                   (SETQ EOLC (FFETCH
 (EXTERNALFORMAT EOL) OF
 FORMAT)))
-         (\EXTERNALFORMAT
 BYTESTREAM FORMAT)
          (REPLACE
 (STREAM EOLCONVENTION) OF
 BYTESTREAM WITH
 EOLC)
          (\SETFILEPTR BYTESTREAM 0)
-         (freplace
 (STREAM ENDOFSTREAMOP) of
 BYTESTREAM with
 (FUNCTION NILL))
 
          (* ;; "Presumably any format-specific cleanup function will know what to do if it receives a format instead of a stream.")

 
          (CL:WHEN (FFETCH
 (EXTERNALFORMAT FORMATBYTESTREAMFN) OF
 FORMAT)
-             (APPLY* (FFETCH
 (EXTERNALFORMAT FORMATBYTESTREAMFN) OF
 FORMAT)
-                    STREAM BYTESTREAM))
+             (SETQ BYTESTREAM (APPLY* (FFETCH
 (EXTERNALFORMAT FORMATBYTESTREAMFN) OF
 FORMAT)
+                                     STREAM BYTESTREAM)))
+
+         (* ;; "Maybe the function said no?")

+
+         (CL:WHEN BYTESTREAM
+             (freplace
 (STREAM ENDOFSTREAMOP) of
 BYTESTREAM with
 (FUNCTION NILL)))
          BYTESTREAM])
 
 (\FORMATBYTESTRING

-  [LAMBDA (STREAM STRING)
                                    (* ; "Edited 10-Jul-2022 16:39 by rmk")


+  [LAMBDA (STREAM STRING)
                                    (* ; "Edited 19-Mar-2024 18:24 by rmk")


+                                                             (* ; "Edited 10-Jul-2022 16:39 by rmk")


                                                              (* ; "Edited 22-Jun-2022 11:07 by rmk")


                                                              (* ; "Edited 18-Jun-2022 22:04 by rmk")

     (WITH-RESOURCE \FORMATBYTESTRING.STREAM (\SETFILEPTR \FORMATBYTESTRING.STREAM 0)
            (LET [FSTRING NBYTES (BYTESTRINGFN (FETCH
 (EXTERNALFORMAT FORMATBYTESTRINGFN)
                                                  OF
 (FETCH
 (STREAM EXTERNALFORMAT) OF
 STREAM]
                 (IF
 BYTESTRINGFN
-                    THEN
 (APPLY* BYTESTRINGFN STREAM STRING \FORMATBYTESTRING.STREAM)
+                    THEN
 (CL:WHEN (SETQ FSTRING (APPLY* BYTESTRINGFN STREAM STRING 
+                                                       \FORMATBYTESTRING.STREAM))
+                                (MKSTRING FSTRING))
                   ELSE
 (\FORMATBYTESTREAM
 STREAM \FORMATBYTESTRING.STREAM)
                        (FOR
 C INPNAME
 STRING DO
 (\OUTCHAR
 \FORMATBYTESTRING.STREAM C))
                        (SETQ NBYTES (\GETFILEPTR \FORMATBYTESTRING.STREAM))
@@ -727,13 +737,13 @@
 (\CREATE.THROUGH.EXTERNALFORMAT
)
 )
 (DECLARE%: DONTCOPY
-  (FILEMAP (NIL (6735 13568 (\EXTERNALFORMAT 6745 . 10523) (MAKE-EXTERNALFORMAT 10525 . 13095) (
-\EXTERNALFORMAT.DEFPRINT 13097 . 13566)) (13569 16610 (\INSTALL.EXTERNALFORMAT 13579 . 15028) (
-\REMOVE.EXTERNALFORMAT 15030 . 15861) (FIND-FORMAT 15863 . 16608)) (16611 17023 (SYSTEM-EXTERNALFORMAT
- 16621 . 17021)) (17372 32799 (\OUTCHAR 17382 . 18599) (\INCCODE 18601 . 19754) (\BACKCCODE 19756 . 
-21435) (\BACKCCODE.EOLC 21437 . 23627) (\PEEKCCODE 23629 . 23954) (\PEEKCCODE.EOLC 23956 . 24335) (
-\INCCODE.EOLC 24337 . 26136) (\FORMATBYTESTREAM 26138 . 28273) (\FORMATBYTESTRING 28275 . 29734) (
-\CHECKEOLC.CRLF 29736 . 32797)) (34081 36317 (\NULLDEVICE 34091 . 35993) (\NULL.OPENFILE 35995 . 36315
-)) (36457 38284 (\CREATE.THROUGH.EXTERNALFORMAT 36467 . 37253) (\THROUGHIN 37255 . 37675) (
-\THROUGHBACKCCODE 37677 . 37944) (\THROUGHOUTCHARFN 37946 . 38282)))))
+  (FILEMAP (NIL (6726 13559 (\EXTERNALFORMAT 6736 . 10514) (MAKE-EXTERNALFORMAT 10516 . 13086) (
+\EXTERNALFORMAT.DEFPRINT 13088 . 13557)) (13560 16601 (\INSTALL.EXTERNALFORMAT 13570 . 15019) (
+\REMOVE.EXTERNALFORMAT 15021 . 15852) (FIND-FORMAT 15854 . 16599)) (16602 17014 (SYSTEM-EXTERNALFORMAT
+ 16612 . 17012)) (17363 33340 (\OUTCHAR 17373 . 18590) (\INCCODE 18592 . 19745) (\BACKCCODE 19747 . 
+21426) (\BACKCCODE.EOLC 21428 . 23618) (\PEEKCCODE 23620 . 23945) (\PEEKCCODE.EOLC 23947 . 24326) (
+\INCCODE.EOLC 24328 . 26127) (\FORMATBYTESTREAM 26129 . 28573) (\FORMATBYTESTRING 28575 . 30275) (
+\CHECKEOLC.CRLF 30277 . 33338)) (34622 36858 (\NULLDEVICE 34632 . 36534) (\NULL.OPENFILE 36536 . 36856
+)) (36998 38825 (\CREATE.THROUGH.EXTERNALFORMAT 37008 . 37794) (\THROUGHIN 37796 . 38216) (
+\THROUGHBACKCCODE 38218 . 38485) (\THROUGHOUTCHARFN 38487 . 38823)))))
 STOP

sources/EXTERNALFORMAT.LCOM

@@ -1,12 +1,12 @@
 (DEFINE-FILE-INFO PACKAGE "INTERLISP" READTABLE "INTERLISP" BASE 10)
 
-
(FILECREATED " 9-Dec-2023 11:42:55" {WMEDLEY}<sources>XCCS.;66
 14365  
+
(FILECREATED "26-Mar-2024 11:00:37" {WMEDLEY}<sources>XCCS.;70
 14862  
 
       :EDIT-BY rmk
 
-      :CHANGES-TO (FNS \XCCSCHARSETFN)
+      :CHANGES-TO (FNS \XCCSFORMATBYTESTREAM)
 
-      :PREVIOUS-DATE " 8-Dec-2023 15:34:50" {WMEDLEY}<sources>XCCS.;65)
+      :PREVIOUS-DATE "19-Mar-2024 16:02:36" {WMEDLEY}<sources>XCCS.;68)
 
 
 (PRETTYCOMPRINT XCCSCOMS)
@@ -19,6 +19,7 @@
         (INITVARS (*SIGNAL-24BIT-NSENCODING-ERROR*))
         (DECLARE%: EVAL@COMPILE DONTCOPY (EXPORT (CONSTANTS (\NORUNCODE 255)
                                                         (NSCHARSETSHIFT 255))
+                                                (ALISTS (CHARACTERNAMES \NORUNCODE NSCHARSETSHIFT))
                                                 (MACROS \RUNCODED)))
         (DECLARE%: DONTEVAL@LOAD DOCOPY (P (\CREATE.XCCS.EXTERNALFORMAT])
 (DEFINEQ
@@ -200,9 +201,14 @@
                            BYTE)))])
 
 (\XCCSFORMATBYTESTREAM

-  [LAMBDA (STREAM BYTESTREAM)
                           (* ; "Edited 24-Jun-2021 16:47 by rmk:")

-    (REPLACE
 (STREAM CHARSET) OF
 BYTESTREAM WITH
 (FETCH
 (STREAM CHARSET) OF

-                                                                                         STREAM])
+  [LAMBDA (STREAM BYTESTREAM)
                                (* ; "Edited 26-Mar-2024 11:00 by rmk")


+                                                             (* ; "Edited 19-Mar-2024 16:02 by rmk")

+    (\EXTERNALFORMAT BYTESTREAM (\EXTERNALFORMAT STREAM))
+
+    (* ;; "This stream may be read as a continuation of STREAM (TTYIN, LAFITE?), and we want to make sure that the bytes are encoded properly.  So let's assert (and possibly mark) that that's its current situation.")

+
+    (\XCCSCHARSETFN
 BYTESTREAM (fetch
 (STREAM CHARSET) of
 STREAM))
+    BYTESTREAM])
 
 (\XCCSCHARSETFN

   [LAMBDA (STREAM CHARSET DONTMARKSTREAM)
                    (* ; "Edited  9-Dec-2023 11:18 by rmk")

@@ -270,6 +276,9 @@
 (CONSTANTS (\NORUNCODE 255)
        (NSCHARSETSHIFT 255))
 )
+
+(ADDTOVAR CHARACTERNAMES
 (\NORUNCODE 255)
+                         (NSCHARSETSHIFT 255))
 (DECLARE%: EVAL@COMPILE 
 
 (PUTPROPS \RUNCODED
 MACRO
 (OPENLAMBDA (STREAM)
@@ -289,8 +298,8 @@
 (\CREATE.XCCS.EXTERNALFORMAT
)
 )
 (DECLARE%: DONTCOPY
-  (FILEMAP (NIL (886 11846 (\XCCSINCCODE 896 . 3875) (\XCCSPEEKCCODE 3877 . 6546) (\XCCSOUTCHAR 6548 . 
-8768) (\XCCSBACKCCODE 8770 . 10314) (\XCCSFORMATBYTESTREAM 10316 . 10641) (\XCCSCHARSETFN 10643 . 
-11844)) (11847 12620 (\CREATE.XCCS.EXTERNALFORMAT 11857 . 12618)) (12621 13452 (
-\NSIN.24BITENCODING.ERROR 12631 . 13450)))))
+  (FILEMAP (NIL (993 12249 (\XCCSINCCODE 1003 . 3982) (\XCCSPEEKCCODE 3984 . 6653) (\XCCSOUTCHAR 6655 . 
+8875) (\XCCSBACKCCODE 8877 . 10421) (\XCCSFORMATBYTESTREAM 10423 . 11044) (\XCCSCHARSETFN 11046 . 
+12247)) (12250 13023 (\CREATE.XCCS.EXTERNALFORMAT 12260 . 13021)) (13024 13855 (
+\NSIN.24BITENCODING.ERROR 13034 . 13853)))))
 STOP