tests

Author: nhirschey

src/FSharp.Formatting.Literate/Literate.fs

@@ -513,6 +513,61 @@ type Literate private () =
         let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx
         docModel
 
+    /// Parse and transform a pynb document
+    static member internal ParseAndTransformPynbFile
+        (
+            input,
+            output,
+            outputKind,
+            prefix,
+            fscOptions,
+            lineNumbers,
+            references,
+            substitutions,
+            generateAnchors,
+            imageSaver,
+            rootInputFolder,
+            crefResolver,
+            mdlinkResolver,
+            parseOptions,
+            onError,
+            filesWithFrontMatter: FrontMatterFile array
+        ) =
+
+        let parseOptions =
+            match outputKind with
+            | OutputKind.Markdown
+            | OutputKind.Fsx
+            | OutputKind.Pynb -> parseOptions ||| MarkdownParseOptions.ParseCodeAsOther
+            //||| MarkdownParseOptions.ParseNonCodeAsOther
+            | _ -> parseOptions
+
+        let md = ParsePynb.pynbToMarkdown input
+        let doc =
+            Literate.ParseMarkdownString(
+                md,
+                ?fscOptions = fscOptions,
+                ?references = references,
+                parseOptions = parseOptions,
+                ?rootInputFolder = rootInputFolder,
+                ?onError = onError
+            )
+
+        let ctx =
+            makeFormattingContext
+                outputKind
+                prefix
+                lineNumbers
+                generateAnchors
+                substitutions
+                crefResolver
+                mdlinkResolver
+                None
+
+        let doc = downloadImagesForDoc imageSaver doc
+        let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx
+        docModel
+
     /// Convert a markdown file into HTML or another output kind
     static member ConvertMarkdownFile
         (

src/FSharp.Formatting.Literate/ParsePynb.fs

@@ -2,6 +2,7 @@ namespace FSharp.Formatting.Literate
 
 open System.IO
 open System.Text.Json
+open FSharp.Formatting.Templating
 
 module internal ParsePynb =
 
@@ -21,17 +22,15 @@ module internal ParsePynb =
                 match code.outputs with
                 | None -> codeBlock
                 | Some outputs ->
-                    let outputsString = outputs |> String.concat "\n\n"
-                    sprintf $"{codeBlock}\n\n{outputsString}\n\n"
+                    let outputsString = outputs |> String.concat "\n"
+                    sprintf $"{codeBlock}\n{outputsString}"
 
     module Output =
         let (|TextHtml|_|) (x: JsonElement) =
             match x.TryGetProperty("text/html") with
             | true, html ->
-                html.EnumerateArray()
-                |> Seq.map (fun x -> x.GetString())
-                |> String.concat ""
-                |> Some
+                let html = html.EnumerateArray() |> Seq.map (fun x -> x.GetString()) |> String.concat "\n"
+                Some("""<p>""" + html + """</p>""")
             | _ -> None
 
         let (|TextPlain|_|) (x: JsonElement) =
@@ -40,9 +39,9 @@ module internal ParsePynb =
                 let text = text.EnumerateArray() |> Seq.map (fun x -> x.GetString()) |> String.concat ""
 
                 Some(
-                    """<table class="pre"><tr><td><pre><code>"""
+                    """<table class="pre"><tbody><tr><td><pre><code>"""
                     + text
-                    + """</code></pre></td></tr></table>"""
+                    + """</code></pre></td></tr></tbody></table>"""
                 )
             | _ -> None
 
@@ -64,9 +63,9 @@ module internal ParsePynb =
                     |> String.concat ""
 
                 Some(
-                    """<table class="pre"><tr><td><pre><code>"""
+                    """<table class="pre"><tbody><tr><td><pre><code>"""
                     + text
-                    + """</code></pre></td></tr></table>"""
+                    + """</code></pre></td></tr></tbody></table>"""
                 )
             else
                 None
@@ -143,6 +142,20 @@ module internal ParsePynb =
 
         json.RootElement.GetProperty("cells").EnumerateArray()
         |> Seq.map (parseCell >> (fun x -> x.ToMarkdown()))
-        |> String.concat ""
+        |> String.concat "\n\n"
+
+    let pynbToMarkdown ipynbFile =
+        ipynbFile |> File.ReadAllText |> pynbStringToMarkdown
 
-    let pynbToMarkdown ipynbFile = ipynbFile |> File.ReadAllText |> pynbStringToMarkdown
+    let parseFrontMatter ipynbFile =
+        let json = JsonDocument.Parse(ipynbFile |> File.ReadAllText)
+
+        json.RootElement.GetProperty("cells").EnumerateArray()
+        |> Seq.map parseCell
+        |> Seq.choose (fun cell ->
+            match cell with
+            | Code _ -> None
+            | Markdown source ->
+                let lines = source.Split([| '\n'; '\r' |], System.StringSplitOptions.RemoveEmptyEntries)
+                FrontMatterFile.ParseFromLines ipynbFile lines)
+        |> Seq.tryHead

src/fsdocs-tool/BuildCommand.fs

@@ -74,10 +74,11 @@ type internal DocContent
         let inputFileName = Path.GetFileName(inputFileFullPath)
         let isFsx = inputFileFullPath.EndsWith(".fsx", true, CultureInfo.InvariantCulture)
         let isMd = inputFileFullPath.EndsWith(".md", true, CultureInfo.InvariantCulture)
+        let isPynb = inputFileFullPath.EndsWith(".ipynb", true, CultureInfo.InvariantCulture)
         let ext = outputKind.Extension
 
         let outputFileRelativeToRoot =
-            if isFsx || isMd then
+            if isFsx || isMd || isPynb then
                 let basename = Path.GetFileNameWithoutExtension(inputFileFullPath)
 
                 Path.Combine(outputFolderRelativeToRoot, sprintf "%s.%s" basename ext)
@@ -189,6 +190,8 @@ type internal DocContent
 
               let isMd = inputFileFullPath.EndsWith(".md", true, CultureInfo.InvariantCulture)
 
+              let isPynb = inputFileFullPath.EndsWith(".ipynb", true, CultureInfo.InvariantCulture)
+
               // A _template.tex or _template.pynb is needed to generate those files
               match outputKind, template with
               | OutputKind.Pynb, None -> ()
@@ -344,7 +347,43 @@ type internal DocContent
                                        template,
                                        outputFileFullPath
                                    )))
+                      elif isPynb then
+                          printfn " preparing %s --> %s" inputFileFullPath outputFileRelativeToRoot
+                          let model =
+                              Literate.ParseAndTransformPynbFile(
+                                  inputFileFullPath,
+                                  output = outputFileRelativeToRoot,
+                                  outputKind = outputKind,
+                                  prefix = None,
+                                  fscOptions = None,
+                                  lineNumbers = lineNumbers,
+                                  references = Some false,
+                                  substitutions = substitutions,
+                                  generateAnchors = Some true,
+                                  imageSaver = imageSaverOpt,
+                                  rootInputFolder = rootInputFolder,
+                                  crefResolver = crefResolver,
+                                  mdlinkResolver = mdlinkResolver,
+                                  parseOptions = MarkdownParseOptions.AllowYamlFrontMatter,
+                                  onError = Some onError,
+                                  filesWithFrontMatter = filesWithFrontMatter
+                              )
+
+                          yield
+                              ((if mainRun then
+                                    Some(inputFileFullPath, isOtherLang, model)
+                                else
+                                    None),
+                               (fun p ->
+                                   printfn "  writing %s --> %s" inputFileFullPath outputFileRelativeToRoot
+                                   ensureDirectory (Path.GetDirectoryName(outputFileFullPath))
 
+                                   SimpleTemplating.UseFileAsSimpleTemplate(
+                                       p @ model.Substitutions,
+                                       template,
+                                       outputFileFullPath
+                                   )))
+                        
                       else if mainRun then
                           yield
                               (None,
@@ -558,6 +597,8 @@ type internal DocContent
                     ParseScript.ParseFrontMatter(fileName)
                 elif ext = ".md" then
                     File.ReadLines fileName |> FrontMatterFile.ParseFromLines fileName
+                elif ext = ".ipynb" then
+                    ParsePynb.parseFrontMatter fileName
                 else
                     None)
             |> Seq.sortBy (fun { Index = idx; CategoryIndex = cIdx } -> cIdx, idx)

tests/FSharp.Literate.Tests/DocContentTests.fs

@@ -45,33 +45,48 @@ let ``Can build doc content`` () =
     // Check simple2.md --> simple2.html substititions
     let html1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.html")
     let html2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.html")
+    let html3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.html")
     html1 |> shouldContainText """href="simple1.html">"""
     html1 |> shouldContainText """href="simple2.html">"""
+    html1 |> shouldContainText """href="simple3.html">"""
     html2 |> shouldContainText """href="simple1.html">"""
     html2 |> shouldContainText """href="simple2.html">"""
+    html2 |> shouldContainText """href="simple3.html">"""
+    html3 |> shouldContainText """href="simple1.html">"""
+    html3 |> shouldContainText """href="simple2.html">"""
+    html3 |> shouldContainText """href="simple3.html">"""
 
     // Check simple1.fsx --> simple1.ipynb substititions
     // Check simple2.md --> simple1.ipynb substititions
     let ipynb1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.ipynb")
     let ipynb2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.ipynb")
+    let ipynb3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.ipynb")
     ipynb1 |> shouldContainText "simple2.ipynb"
+    ipynb1 |> shouldContainText "simple3.ipynb"
     ipynb2 |> shouldContainText "simple1.ipynb"
+    ipynb3 |> shouldContainText "simple1.ipynb"
 
     // Check fsx exists
     // Check simple1.fsx --> simple1.fsx substititions
     // Check simple2.md --> simple1.fsx substititions
     let fsx1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.fsx")
     let fsx2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.fsx")
+    let fsx3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.fsx")
     fsx1 |> shouldContainText "simple2.fsx"
+    fsx1 |> shouldContainText "simple3.fsx"
     fsx2 |> shouldContainText "simple1.fsx"
+    fsx3 |> shouldContainText "simple1.fsx"
 
     // Check md contents
     // Check simple1.fsx --> simple1.md substititions
     // Check simple2.md --> simple1.md substititions
     let md1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.md")
     let md2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.md")
+    let md3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.md")
     md1 |> shouldContainText "simple2.md"
+    md1 |> shouldContainText "simple3.md"
     md2 |> shouldContainText "simple1.md"
+    md3 |> shouldContainText "simple1.md"
 
 
     // Check in-folder1.fsx --> in-folder1.html substititions
@@ -134,34 +149,48 @@ let ``Can build doc content using relative input path`` () =
     // Check simple2.md --> simple2.html substititions
     let html1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.html")
     let html2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.html")
+    let html3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.html")
     html1 |> shouldContainText """href="simple1.html">"""
     html1 |> shouldContainText """href="simple2.html">"""
+    html1 |> shouldContainText """href="simple3.html">"""
     html2 |> shouldContainText """href="simple1.html">"""
     html2 |> shouldContainText """href="simple2.html">"""
+    html2 |> shouldContainText """href="simple3.html">"""
+    html3 |> shouldContainText """href="simple1.html">"""
+    html3 |> shouldContainText """href="simple2.html">"""
+    html3 |> shouldContainText """href="simple3.html">"""
 
     // Check simple1.fsx --> simple1.ipynb substititions
     // Check simple2.md --> simple1.ipynb substititions
     let ipynb1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.ipynb")
     let ipynb2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.ipynb")
+    let ipynb3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.ipynb")
     ipynb1 |> shouldContainText "simple2.ipynb"
+    ipynb1 |> shouldContainText "simple3.ipynb"
     ipynb2 |> shouldContainText "simple1.ipynb"
+    ipynb3 |> shouldContainText "simple1.ipynb"
 
     // Check fsx exists
     // Check simple1.fsx --> simple1.fsx substititions
     // Check simple2.md --> simple1.fsx substititions
     let fsx1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.fsx")
     let fsx2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.fsx")
+    let fsx3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.fsx")
     fsx1 |> shouldContainText "simple2.fsx"
+    fsx1 |> shouldContainText "simple3.fsx"
     fsx2 |> shouldContainText "simple1.fsx"
+    fsx3 |> shouldContainText "simple1.fsx"
 
     // Check md contents
     // Check simple1.fsx --> simple1.md substititions
     // Check simple2.md --> simple1.md substititions
     let md1 = File.ReadAllText(rootOutputFolderAsGiven </> "simple1.md")
     let md2 = File.ReadAllText(rootOutputFolderAsGiven </> "simple2.md")
+    let md3 = File.ReadAllText(rootOutputFolderAsGiven </> "simple3.md")
     md1 |> shouldContainText "simple2.md"
+    md1 |> shouldContainText "simple3.md"
     md2 |> shouldContainText "simple1.md"
-
+    md3 |> shouldContainText "simple1.md"
 
     // Check in-folder1.fsx --> in-folder1.html substititions
     let f1html1 = File.ReadAllText(rootOutputFolderAsGiven </> "folder1" </> "in-folder1.html")
@@ -225,3 +254,38 @@ let ``Parses frontmatter correctly `` () =
     twoTowersHtml |> shouldContainText "<a href=\"fellowship.html\">Previous</a>"
     twoTowersHtml |> shouldContainText "<a href=\"return.html\">Next</a>"
     returnHtml |> shouldContainText "<a href=\"two-tower.html\">Previous</a>"
+
+
+(* Cannot get this test to evaluate the notebook
+[<Test>]
+let ``ipynb notebook evaluates`` () =
+    let rootOutputFolderAsGiven = __SOURCE_DIRECTORY__ </> "ipynb-eval-output"
+    let rootInputFolderAsGiven = __SOURCE_DIRECTORY__ </> "ipynb-eval"
+
+    if Directory.Exists(rootOutputFolderAsGiven) then
+        Directory.Delete(rootOutputFolderAsGiven, true)
+
+    let content =
+        DocContent(
+            rootOutputFolderAsGiven,
+            Map.empty,
+            lineNumbers = None,
+            evaluate = true,
+            substitutions = [],
+            saveImages = None,
+            watch = false,
+            root = "https://github.com",
+            crefResolver = (fun _ -> None),
+            onError = failwith
+        )
+
+    let docModels = content.Convert(rootInputFolderAsGiven, None, [])
+    let globals = []
+
+    for (_thing, action) in docModels do
+        action globals  
+
+    let ipynbOut = rootOutputFolderAsGiven </> "eval.html" |> File.ReadAllText
+
+    ipynbOut |> shouldContainText "10007"
+*)

tests/FSharp.Literate.Tests/files/simple1.fsx

@@ -13,6 +13,7 @@ substitute-in-markdown: {{fsdocs-source-basename}}
 
 Another [hyperlink](simple1.fsx)
 Another [hyperlink](simple2.md)
+And another [hyperlink](simple3.ipynb)
 
 *)
 let hello = "Code sample"

tests/FSharp.Literate.Tests/files/simple2.md

@@ -7,3 +7,4 @@ With some [hyperlink](http://tomasp.net)
 
 Another [hyperlink](simple1.fsx)
 Yet another [hyperlink](simple2.md)
+And another [hyperlink](simple3.ipynb)