Merge branch 'master' into sharding_fix

ceallen · web-flow · commit 83f7e6f7405e · 2023-09-01T09:20:50.000-04:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -130,9 +130,8 @@ defaults: &defaults
           pip install -e ".[docs]"
           python setup.py build_sphinx
           cp -r ./build "$CIRCLE_ARTIFACTS"
-    # Build egg
     - run:
-        name: Build egg
+        name: Build distribution
         command: |
           set -ex
           . ci/bin/activate
@@ -146,8 +145,7 @@ defaults: &defaults
             cat ../log
             exit 1
           fi # rst2html.py alwaysexits with 0, check log size
-          python setup.py bdist_wheel --universal
-          python setup.py bdist_egg
+          python setup.py bdist_wheel
           python setup.py sdist
           mkdir -p "$CIRCLE_ARTIFACTS/dist"
           cp -r ./dist/* "$CIRCLE_ARTIFACTS/dist"
@@ -192,7 +190,7 @@ jobs:
       PYTHON_VERSION: "3_6"
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_6
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_6
-      VERSION: 0.5.1
+      VERSION: 0.6.0
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -204,7 +202,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_7
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_7
-      VERSION: 0.5.1
+      VERSION: 0.6.0
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -216,7 +214,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_8
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_8
-      VERSION: 0.5.1
+      VERSION: 0.6.0
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
-0.5.2 (2023-??)
+0.6.0 (2023-09-01)
 ------------------
+* Feature: Reports are now grouped by their containing folder on the main UI.
 * Feature: Allow configuring error email addresses and email subject via UI. 
 * Bugfix: . and .. should now be allowed to be used when specifying the templates directory.
 * Bugfix: corrected cron schedule incorrectly shifting back one day upon save.
diff --git a/docs/conf.py b/docs/conf.py
@@ -23,7 +23,7 @@
 author = "Man Group Quant Tech"
 
 # The full version, including alpha/beta/rc tags
-release = "0.5.1"
+release = "0.6.0"
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/notebooker/notebook_templates_example/other_folder/other_slideshow_test.py b/notebooker/notebook_templates_example/other_folder/other_slideshow_test.py
@@ -0,0 +1,33 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
+#   kernelspec:
+#     display_name: py3-local-pegasus
+#     language: python
+#     name: env
+# ---
+
+# + tags=["parameters"]
+the_range = 10
+# -
+
+# + [markdown] slideshow={"slide_type": "slide"}
+# # This is slide number one
+
+# + [markdown] slideshow={"slide_type": "subslide"}
+# ## This is slide two
+#
+# - I have some
+# - things to talk
+# - about.
+
+# + [markdown] slideshow={"slide_type": "slide"}
+# # Okay onto the good stuff!
+# -
+
+list(range(the_range))
diff --git a/notebooker/serialization/mongo.py b/notebooker/serialization/mongo.py
@@ -389,10 +389,11 @@ def _get_result_count(self, base_filter):
             # pymongo<3.7 compat
             return self.library.count(base_filter)
 
-    def get_count_and_latest_time_per_report(self):
+    def get_count_and_latest_time_per_report(self, subfolder: Optional[str]):
+        base_filer = {} if not subfolder else {"report_name": {"$regex": subfolder + ".*"}}
         reports = list(
             self._get_raw_results(
-                base_filter={},
+                base_filter=base_filer,
                 projection={"report_name": 1, "job_start_time": 1, "scheduler_job_id": 1, "_id": 0},
                 limit=0,
             )
diff --git a/notebooker/utils/results.py b/notebooker/utils/results.py
@@ -139,8 +139,8 @@ def get_all_available_results_json(
     return json_output
 
 
-def get_count_and_latest_time_per_report(serializer: MongoResultSerializer):
-    reports = serializer.get_count_and_latest_time_per_report()
+def get_count_and_latest_time_per_report(serializer: MongoResultSerializer, subfolder: Optional[str] = None):
+    reports = serializer.get_count_and_latest_time_per_report(subfolder)
     output = {}
     for report_name, metadata in sorted(reports.items(), key=lambda x: x[1]["latest_run"], reverse=True):
         metadata["report_name"] = report_name
diff --git a/notebooker/version.py b/notebooker/version.py
@@ -1 +1 @@
-__version__ = "0.5.1"
+__version__ = "0.6.0"
diff --git a/notebooker/web/routes/index.py b/notebooker/web/routes/index.py
@@ -1,3 +1,4 @@
+from typing import Optional
 import traceback
 
 import inflection
@@ -10,7 +11,8 @@
 
 
 @index_bp.route("/", methods=["GET"])
-def index():
+@index_bp.route("/folder/<path:subfolder>", methods=["GET"])
+def index(subfolder: Optional[str] = None):
     """
     The index page which shows cards of each report which has at least one result in the database.
     """
diff --git a/notebooker/web/routes/templates.py b/notebooker/web/routes/templates.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 from flask import jsonify, Blueprint
 
 from notebooker.utils.results import get_count_and_latest_time_per_report
@@ -8,14 +10,15 @@
 templates_bp = Blueprint("templates_bp", __name__)
 
 
-@templates_bp.route("/core/get_all_templates_with_results")
-def all_available_templates_with_results():
+@templates_bp.route("/core/get_all_templates_with_results/folder/")
+@templates_bp.route("/core/get_all_templates_with_results/folder/<path:subfolder>")
+def all_available_templates_with_results(subfolder: Optional[str] = None):
     """
     Core function for the index.html view which shows the templates which have results available.
 
     :returns: A JSON containing a list of template names with a count of how many results are in each.
     """
-    return jsonify(get_count_and_latest_time_per_report(get_serializer()))
+    return jsonify(get_count_and_latest_time_per_report(get_serializer(), subfolder))
 
 
 @templates_bp.route("/core/all_possible_templates")
diff --git a/notebooker/web/static/notebooker/index.js b/notebooker/web/static/notebooker/index.js
@@ -1,34 +1,124 @@
+
+function getCurrentFolder(){
+    if (window.location.pathname.startsWith("/folder/")){
+        currentFolder = window.location.pathname.substring("/folder/".length) + "/"
+    } else {
+        currentFolder = ""
+    }
+    return currentFolder
+}
+
+function updateNavigationPanel(currentFolder){
+    let $folderNavigationPanel = $('#folderNavigationPanel');
+    $folderNavigationPanel.empty()
+    
+    if (currentFolder != "") {
+        $folderNavigationPanel.append('<a href="/">Start</a>')
+        parts = currentFolder.split("/")
+        folderSoFar = ""
+        for(let idx in parts){
+            part = parts[idx]
+            if(part.length > 0){
+                folderSoFar = folderSoFar + "/" + part
+                if(idx == parts.length - 2){
+                    $folderNavigationPanel.append('&nbsp;&gt;&nbsp;<span>' + part + "</span>")
+                }else{
+                    $folderNavigationPanel.append('&nbsp;&gt;&nbsp;<a href="/folder' + folderSoFar + '">' + part + " </a>")
+                }
+                
+            }
+        }
+    }
+}
+
+
+function entryAfterLevel(path, level){
+    while(level > 0) {
+        path = path.substring(path.indexOf('/') + 1)
+        level = level - 1
+    }
+    return path
+}
+
+function updateContents(currentFolder, entries){
+    let $cardContainer = $('#cardContainer');
+    $cardContainer.empty();
+    subfoldersInfo = {}
+    reportParts = []
+    for (let report in entries) {
+        report_data = entries[report]
+        report_path = report_data.report_name
+        if(!report_path.startsWith(currentFolder)){
+            continue;
+        }
+        remainingPath = report_path.substring(currentFolder.length)
+        if(remainingPath.includes('/')) {
+            // it is a folder
+            subfolderPathName = remainingPath.substring(0, remainingPath.indexOf('/'))
+            subfolderPath = currentFolder + subfolderPathName
+            if (subfolderPath in subfoldersInfo) {
+                info.reportCount = info.reportCount + 1
+            } else{
+                info = {}
+                info.pathName = subfolderPathName
+                info.path = subfolderPath
+                info.reportCount = 1
+                subfoldersInfo[subfolderPath] = info
+            }
+        }else{
+            level = currentFolder.split('/').length - 1
+            displayName = entryAfterLevel(report, level)
+            let stats = entries[report];
+            reportParts.push('<a class="ui card" href="/result_listing/' + stats.report_name + '">' +
+                '  <div class="content">' +
+                '    <h1>' + displayName + '</h1>\n' +
+                '    <div class="meta">\n' +
+                '      <span class="date">Last ran ' + stats.time_diff + ' ago</span>\n' +
+                '    </div>' +
+                '    <span>\n' +
+                stats.count +
+                '        Runs\n' +
+                '    </span>' +
+                '<br/>' +
+                '    <span>\n' +
+                stats.scheduler_runs +
+                '        Scheduler Runs\n' +
+                '    </span>' +
+                '  </div>' +
+                '  <div class="extra content">' +
+                '      <span>Original report name: ' + stats.report_name + '</span>\n' +
+                '  </div>' +
+                '</a>');
+        }
+        
+        
+    }
+    // first add all folders
+    for (let subfolder in subfoldersInfo) {
+        info = subfoldersInfo[subfolder]
+        $cardContainer.append(
+            '<a class="ui card folder" href="/folder/' + info.path + '">' +
+            '  <div class="content">' +
+            '    <h1><i class="fa-solid fa-folder fa-xs"></i> ' + info.pathName + '</h1>\n' +
+            ' <span>Reports: ' + info.reportCount + '</span>' + 
+            '  </div>' +
+            '</a>');
+    }
+    // only then add individual items
+    for (let idx in reportParts) {
+        $cardContainer.append(reportParts[idx])
+    }
+}
+
+
 load_data = () => {
     $.ajax({
-        url: `/core/get_all_templates_with_results`,
+        url: `/core/get_all_templates_with_results/folder/`.concat(getCurrentFolder()),
         dataType: 'json',
         success: (result) => {
-            let $cardContainer = $('#cardContainer');
-            $cardContainer.empty();
-            for (let report in result) {
-                let stats = result[report];
-                $cardContainer.append(
-                    '<a class="ui card" href="/result_listing/' + stats.report_name + '">' +
-                    '  <div class="content">' +
-                    '    <h1>' + report + '</h1>\n' +
-                    '    <div class="meta">\n' +
-                    '      <span class="date">Last ran ' + stats.time_diff + ' ago</span>\n' +
-                    '    </div>' +
-                    '    <span>\n' +
-                    stats.count +
-                    '        Runs\n' +
-                    '    </span>' +
-                    '<br/>' +
-                    '    <span>\n' +
-                    stats.scheduler_runs +
-                    '        Scheduler Runs\n' +
-                    '    </span>' +
-                    '  </div>' +
-                    '  <div class="extra content">' +
-                    '      <span>Original report name: ' + stats.report_name + '</span>\n' +
-                    '  </div>' +
-                    '</a>');
-            }
+            currentFolder = getCurrentFolder()
+            updateNavigationPanel(currentFolder)
+            updateContents(currentFolder, result)
         },
         error: (jqXHR, textStatus, errorThrown) => {
             $('#failedLoad').fadeIn();
diff --git a/notebooker/web/static/notebooker/one_click_notebooks.css b/notebooker/web/static/notebooker/one_click_notebooks.css
@@ -46,6 +46,10 @@ code {
     color: #222222;
 }
 
+.ui.card.folder {
+    background-color: #EEEEEE;
+}
+
 .ui.table {
     width: 99.5%;  /* Because a border with overflow-y:auto causes a tiny scrollbar */
 }
@@ -92,6 +96,14 @@ td button i {
     max-width: 280px;  /* Enough to show full job id */
 }
 
+.folderImg {
+    margin-right: 10px;
+}
+
+#folderNavigationPanel {
+    margin-bottom: 1em;
+}
+
 .resultHeader {
     margin-left: 1em !important;
 }
diff --git a/notebooker/web/static/package.json b/notebooker/web/static/package.json
@@ -1,6 +1,6 @@
 {
   "name": "notebooker",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Notebooker - Turn notebooks into reports",
   "dependencies": {
     "bootstrap-table": "1.20.2",
diff --git a/notebooker/web/templates/header.html b/notebooker/web/templates/header.html
@@ -11,6 +11,7 @@
             <script src="{{ url_for('static', filename='node_modules/bootstrap-table/dist/themes/semantic/bootstrap-table-semantic.js') }}"></script>
             <script type="application/javascript" src="{{ url_for('static', filename='notebooker/iframe_loading.js') }}"></script>
             <script type="application/javascript" src="{{ url_for('static', filename='notebooker/header.js') }}"></script>
+            <script src="https://kit.fontawesome.com/58031af4b9.js" crossorigin="anonymous"></script>
             <link rel="stylesheet" href="{{ url_for('static', filename='notebooker/one_click_notebooks.css') }}">
         </head>
         <body>
diff --git a/notebooker/web/templates/index.html b/notebooker/web/templates/index.html
@@ -16,6 +16,7 @@ <h2>Couldn't load any reports!</h2>
     </span>
     <div class="ui centered grid" id="cardGrid">
         <div class="fifteen wide centered column">
+            <div id="folderNavigationPanel"></div>
             <div class="ui cards" id="cardContainer">
                 <div class="ui loading card">
                     <div class="content">
diff --git a/notebooker/web/utils.py b/notebooker/web/utils.py
@@ -54,7 +54,11 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U
         if not _valid_dirname(path):
             continue
         folders = path[start:].split(os.sep)
-        subdir = {os.sep.join(folders[1:] + [f.replace(".ipynb", "").replace(".py", "")]): None for f in files if _valid_filename(f)}
+        subdir = {
+            os.sep.join(folders[1:] + [f.replace(".ipynb", "").replace(".py", "")]): None
+            for f in files
+            if _valid_filename(f)
+        }
         parent = reduce(dict.get, folders[:-1], all_dirs)
         parent[folders[-1]] = subdir
     return all_dirs[rootdir[start:]]
diff --git a/tests/integration/test_templates.py b/tests/integration/test_templates.py
@@ -10,5 +10,6 @@ def test_get_all_possible_templates(flask_app):
                 "sample/test_plotly": None,
                 "sample/plot_random_raw": None,
                 "sample/slideshow_test": None,
-            }
+            },
+            "other_folder": {"other_folder/other_slideshow_test": None},
         }
diff --git a/tests/integration/web/routes/test_core.py b/tests/integration/web/routes/test_core.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.5.1"`
	`1`	`+__version__ = "0.6.0"`