grouping report results into folders (#150)

marcinapostoluk · web-flow · commit 962598d03ca2 · 2023-08-31T07:45:01.000+01:00
grouping report results into folders
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 * Feature: Allow configuring error email addresses and email subject via UI. 
 * Bugfix: . and .. should now be allowed to be used when specifying the templates directory.
 * Bugfix: corrected cron schedule incorrectly shifting back one day upon save.
+* Feature: Grouping reports into nested folders
 
 0.5.1 (2023-02-22)
 ------------------
diff --git a/notebooker/notebook_templates_example/other_folder/other_slideshow_test.py b/notebooker/notebook_templates_example/other_folder/other_slideshow_test.py
@@ -0,0 +1,33 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
+#   kernelspec:
+#     display_name: py3-local-pegasus
+#     language: python
+#     name: env
+# ---
+
+# + tags=["parameters"]
+the_range = 10
+# -
+
+# + [markdown] slideshow={"slide_type": "slide"}
+# # This is slide number one
+
+# + [markdown] slideshow={"slide_type": "subslide"}
+# ## This is slide two
+#
+# - I have some
+# - things to talk
+# - about.
+
+# + [markdown] slideshow={"slide_type": "slide"}
+# # Okay onto the good stuff!
+# -
+
+list(range(the_range))
diff --git a/notebooker/serialization/mongo.py b/notebooker/serialization/mongo.py
@@ -371,10 +371,11 @@ def _get_result_count(self, base_filter):
             # pymongo<3.7 compat
             return self.library.count(base_filter)
 
-    def get_count_and_latest_time_per_report(self):
+    def get_count_and_latest_time_per_report(self, subfolder: Optional[str]):
+        base_filer = {} if not subfolder else {"report_name": {"$regex": subfolder + ".*"}}
         reports = list(
             self._get_raw_results(
-                base_filter={},
+                base_filter=base_filer,
                 projection={"report_name": 1, "job_start_time": 1, "scheduler_job_id": 1, "_id": 0},
                 limit=0,
             )
diff --git a/notebooker/utils/results.py b/notebooker/utils/results.py
@@ -139,8 +139,8 @@ def get_all_available_results_json(
     return json_output
 
 
-def get_count_and_latest_time_per_report(serializer: MongoResultSerializer):
-    reports = serializer.get_count_and_latest_time_per_report()
+def get_count_and_latest_time_per_report(serializer: MongoResultSerializer, subfolder: Optional[str] = None):
+    reports = serializer.get_count_and_latest_time_per_report(subfolder)
     output = {}
     for report_name, metadata in sorted(reports.items(), key=lambda x: x[1]["latest_run"], reverse=True):
         metadata["report_name"] = report_name
diff --git a/notebooker/web/routes/index.py b/notebooker/web/routes/index.py
@@ -1,3 +1,4 @@
+from typing import Optional
 import traceback
 
 import inflection
@@ -10,7 +11,8 @@
 
 
 @index_bp.route("/", methods=["GET"])
-def index():
+@index_bp.route("/folder/<path:subfolder>", methods=["GET"])
+def index(subfolder: Optional[str] = None):
     """
     The index page which shows cards of each report which has at least one result in the database.
     """
diff --git a/notebooker/web/routes/templates.py b/notebooker/web/routes/templates.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 from flask import jsonify, Blueprint
 
 from notebooker.utils.results import get_count_and_latest_time_per_report
@@ -8,14 +10,15 @@
 templates_bp = Blueprint("templates_bp", __name__)
 
 
-@templates_bp.route("/core/get_all_templates_with_results")
-def all_available_templates_with_results():
+@templates_bp.route("/core/get_all_templates_with_results/folder/")
+@templates_bp.route("/core/get_all_templates_with_results/folder/<path:subfolder>")
+def all_available_templates_with_results(subfolder: Optional[str] = None):
     """
     Core function for the index.html view which shows the templates which have results available.
 
     :returns: A JSON containing a list of template names with a count of how many results are in each.
     """
-    return jsonify(get_count_and_latest_time_per_report(get_serializer()))
+    return jsonify(get_count_and_latest_time_per_report(get_serializer(), subfolder))
 
 
 @templates_bp.route("/core/all_possible_templates")
diff --git a/notebooker/web/static/notebooker/index.js b/notebooker/web/static/notebooker/index.js
@@ -1,34 +1,124 @@
+
+function getCurrentFolder(){
+    if (window.location.pathname.startsWith("/folder/")){
+        currentFolder = window.location.pathname.substring("/folder/".length) + "/"
+    } else {
+        currentFolder = ""
+    }
+    return currentFolder
+}
+
+function updateNavigationPanel(currentFolder){
+    let $folderNavigationPanel = $('#folderNavigationPanel');
+    $folderNavigationPanel.empty()
+    
+    if (currentFolder != "") {
+        $folderNavigationPanel.append('<a href="/">Start</a>')
+        parts = currentFolder.split("/")
+        folderSoFar = ""
+        for(let idx in parts){
+            part = parts[idx]
+            if(part.length > 0){
+                folderSoFar = folderSoFar + "/" + part
+                if(idx == parts.length - 2){
+                    $folderNavigationPanel.append('&nbsp;&gt;&nbsp;<span>' + part + "</span>")
+                }else{
+                    $folderNavigationPanel.append('&nbsp;&gt;&nbsp;<a href="/folder' + folderSoFar + '">' + part + " </a>")
+                }
+                
+            }
+        }
+    }
+}
+
+
+function entryAfterLevel(path, level){
+    while(level > 0) {
+        path = path.substring(path.indexOf('/') + 1)
+        level = level - 1
+    }
+    return path
+}
+
+function updateContents(currentFolder, entries){
+    let $cardContainer = $('#cardContainer');
+    $cardContainer.empty();
+    subfoldersInfo = {}
+    reportParts = []
+    for (let report in entries) {
+        report_data = entries[report]
+        report_path = report_data.report_name
+        if(!report_path.startsWith(currentFolder)){
+            continue;
+        }
+        remainingPath = report_path.substring(currentFolder.length)
+        if(remainingPath.includes('/')) {
+            // it is a folder
+            subfolderPathName = remainingPath.substring(0, remainingPath.indexOf('/'))
+            subfolderPath = currentFolder + subfolderPathName
+            if (subfolderPath in subfoldersInfo) {
+                info.reportCount = info.reportCount + 1
+            } else{
+                info = {}
+                info.pathName = subfolderPathName
+                info.path = subfolderPath
+                info.reportCount = 1
+                subfoldersInfo[subfolderPath] = info
+            }
+        }else{
+            level = currentFolder.split('/').length - 1
+            displayName = entryAfterLevel(report, level)
+            let stats = entries[report];
+            reportParts.push('<a class="ui card" href="/result_listing/' + stats.report_name + '">' +
+                '  <div class="content">' +
+                '    <h1>' + displayName + '</h1>\n' +
+                '    <div class="meta">\n' +
+                '      <span class="date">Last ran ' + stats.time_diff + ' ago</span>\n' +
+                '    </div>' +
+                '    <span>\n' +
+                stats.count +
+                '        Runs\n' +
+                '    </span>' +
+                '<br/>' +
+                '    <span>\n' +
+                stats.scheduler_runs +
+                '        Scheduler Runs\n' +
+                '    </span>' +
+                '  </div>' +
+                '  <div class="extra content">' +
+                '      <span>Original report name: ' + stats.report_name + '</span>\n' +
+                '  </div>' +
+                '</a>');
+        }
+        
+        
+    }
+    // first add all folders
+    for (let subfolder in subfoldersInfo) {
+        info = subfoldersInfo[subfolder]
+        $cardContainer.append(
+            '<a class="ui card folder" href="/folder/' + info.path + '">' +
+            '  <div class="content">' +
+            '    <h1><i class="fa-solid fa-folder fa-xs"></i> ' + info.pathName + '</h1>\n' +
+            ' <span>Reports: ' + info.reportCount + '</span>' + 
+            '  </div>' +
+            '</a>');
+    }
+    // only then add individual items
+    for (let idx in reportParts) {
+        $cardContainer.append(reportParts[idx])
+    }
+}
+
+
 load_data = () => {
     $.ajax({
-        url: `/core/get_all_templates_with_results`,
+        url: `/core/get_all_templates_with_results/folder/`.concat(getCurrentFolder()),
         dataType: 'json',
         success: (result) => {
-            let $cardContainer = $('#cardContainer');
-            $cardContainer.empty();
-            for (let report in result) {
-                let stats = result[report];
-                $cardContainer.append(
-                    '<a class="ui card" href="/result_listing/' + stats.report_name + '">' +
-                    '  <div class="content">' +
-                    '    <h1>' + report + '</h1>\n' +
-                    '    <div class="meta">\n' +
-                    '      <span class="date">Last ran ' + stats.time_diff + ' ago</span>\n' +
-                    '    </div>' +
-                    '    <span>\n' +
-                    stats.count +
-                    '        Runs\n' +
-                    '    </span>' +
-                    '<br/>' +
-                    '    <span>\n' +
-                    stats.scheduler_runs +
-                    '        Scheduler Runs\n' +
-                    '    </span>' +
-                    '  </div>' +
-                    '  <div class="extra content">' +
-                    '      <span>Original report name: ' + stats.report_name + '</span>\n' +
-                    '  </div>' +
-                    '</a>');
-            }
+            currentFolder = getCurrentFolder()
+            updateNavigationPanel(currentFolder)
+            updateContents(currentFolder, result)
         },
         error: (jqXHR, textStatus, errorThrown) => {
             $('#failedLoad').fadeIn();
diff --git a/notebooker/web/static/notebooker/one_click_notebooks.css b/notebooker/web/static/notebooker/one_click_notebooks.css
@@ -46,6 +46,10 @@ code {
     color: #222222;
 }
 
+.ui.card.folder {
+    background-color: #EEEEEE;
+}
+
 .ui.table {
     width: 99.5%;  /* Because a border with overflow-y:auto causes a tiny scrollbar */
 }
@@ -92,6 +96,14 @@ td button i {
     max-width: 280px;  /* Enough to show full job id */
 }
 
+.folderImg {
+    margin-right: 10px;
+}
+
+#folderNavigationPanel {
+    margin-bottom: 1em;
+}
+
 .resultHeader {
     margin-left: 1em !important;
 }
diff --git a/notebooker/web/templates/header.html b/notebooker/web/templates/header.html
@@ -11,6 +11,7 @@
             <script src="{{ url_for('static', filename='node_modules/bootstrap-table/dist/themes/semantic/bootstrap-table-semantic.js') }}"></script>
             <script type="application/javascript" src="{{ url_for('static', filename='notebooker/iframe_loading.js') }}"></script>
             <script type="application/javascript" src="{{ url_for('static', filename='notebooker/header.js') }}"></script>
+            <script src="https://kit.fontawesome.com/58031af4b9.js" crossorigin="anonymous"></script>
             <link rel="stylesheet" href="{{ url_for('static', filename='notebooker/one_click_notebooks.css') }}">
         </head>
         <body>
diff --git a/notebooker/web/templates/index.html b/notebooker/web/templates/index.html
@@ -16,6 +16,7 @@ <h2>Couldn't load any reports!</h2>
     </span>
     <div class="ui centered grid" id="cardGrid">
         <div class="fifteen wide centered column">
+            <div id="folderNavigationPanel"></div>
             <div class="ui cards" id="cardContainer">
                 <div class="ui loading card">
                     <div class="content">
diff --git a/notebooker/web/utils.py b/notebooker/web/utils.py
@@ -54,7 +54,11 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U
         if not _valid_dirname(path):
             continue
         folders = path[start:].split(os.sep)
-        subdir = {os.sep.join(folders[1:] + [f.replace(".ipynb", "").replace(".py", "")]): None for f in files if _valid_filename(f)}
+        subdir = {
+            os.sep.join(folders[1:] + [f.replace(".ipynb", "").replace(".py", "")]): None
+            for f in files
+            if _valid_filename(f)
+        }
         parent = reduce(dict.get, folders[:-1], all_dirs)
         parent[folders[-1]] = subdir
     return all_dirs[rootdir[start:]]
diff --git a/tests/integration/test_templates.py b/tests/integration/test_templates.py
@@ -10,5 +10,6 @@ def test_get_all_possible_templates(flask_app):
                 "sample/test_plotly": None,
                 "sample/plot_random_raw": None,
                 "sample/slideshow_test": None,
-            }
+            },
+            "other_folder": {"other_folder/other_slideshow_test": None},
         }
diff --git a/tests/integration/web/routes/test_core.py b/tests/integration/web/routes/test_core.py

Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,10 @@ code {`
`46`	`46`	`color: #222222;`
`47`	`47`	`}`
`48`	`48`
	`49`	`+.ui.card.folder {`
	`50`	`+ background-color: #EEEEEE;`
	`51`	`+}`
	`52`	`+`
`49`	`53`	`.ui.table {`
`50`	`54`	`width: 99.5%; /* Because a border with overflow-y:auto causes a tiny scrollbar */`
`51`	`55`	`}`
`@@ -92,6 +96,14 @@ td button i {`
`92`	`96`	`max-width: 280px; /* Enough to show full job id */`
`93`	`97`	`}`
`94`	`98`
	`99`	`+.folderImg {`
	`100`	`+ margin-right: 10px;`
	`101`	`+}`
	`102`	`+`
	`103`	`+#folderNavigationPanel {`
	`104`	`+ margin-bottom: 1em;`
	`105`	`+}`
	`106`	`+`
`95`	`107`	`.resultHeader {`
`96`	`108`	`margin-left: 1em !important;`
`97`	`109`	`}`