Merge pull request #72 from AhmedBasem20/optimize-bandwidth

Optimize bandwidth

Author: oliverchampion

.github/workflows/website.yml

@@ -46,9 +46,12 @@ jobs:
         with:
           name: 'Data'
 
+      - name: 'Filter and compress results file.'
+        run: python utilities/reduce_output_size.py test_output.csv test_output.csv.gz
+
       - name: move data to the dashboard folder
         run: |
-          mv test_output.csv website/dashboard
+          mv test_output.csv.gz website/dashboard
 
       - name: Build documentation
         run: |

utilities/reduce_output_size.py

@@ -0,0 +1,43 @@
+import os
+import gzip
+import csv
+import sys
+
+def reduce_output_file_size(input_file:str, output_file:str):
+    """
+    Simplify the data generated by the analysis pipeline by retaining only the essential information required for the frontend.
+    """
+    if os.path.exists(input_file):
+        # Open the input and output files
+        with open(input_file, 'r') as infile, gzip.open(output_file, 'wt', newline='') as outfile:
+            reader = csv.DictReader(infile)
+
+            # Drop b_values columns
+            fieldnames = [field for field in reader.fieldnames if not field.startswith('bval_')]
+            writer = csv.DictWriter(outfile, fieldnames=fieldnames)
+            writer.writeheader()
+
+            columns_to_round = ['f', 'Dp', 'D', 'f_fitted', 'Dp_fitted', 'D_fitted']
+
+            for row in reader:
+                #Delete columns starting with 'bval_'
+                for key in list(row.keys()):
+                    if key.startswith('bval_'):
+                        del row[key]
+
+                # Round values in the remaining relevant columns
+                for column in columns_to_round:
+                    if column in row:
+                        row[column] = round(float(row[column]), 4)
+                writer.writerow(row)
+    else:
+        print(f"File '{input_file}' not found.")
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: python reduce_output_size.py <input_file> <output_file>")
+        sys.exit(1)
+    
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    reduce_output_file_size(input_file, output_file)

website/dashboard/index.html

@@ -8,6 +8,7 @@
   <script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.7.1/jszip.min.js"></script>
   <script src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>
   <script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.3.0/papaparse.min.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/pako@1.0.11/dist/pako.min.js"></script>
   <script src="index.js"></script>
   <link rel="stylesheet" href="index.css">
 </head>

website/dashboard/index.js

@@ -205,17 +205,31 @@ document.addEventListener('DOMContentLoaded', function() {
 
     showLoading();
 
-    Papa.parse('test_output.csv', {
-        download: true,
-        header: true,
-        complete: results => {
-            data = results;
-            hideLoading();
-            populateOptions(data);
-            drawBoxPlot();
-            drawRegionBoxPlot();
-
+    fetch('test_output.csv.gz')
+    .then(response => {
+        if (!response.ok) {
+            throw new Error('Network response was not ok');
         }
+        return response.arrayBuffer();
+    })
+    .then(buffer => {
+        // Use pako to decompress the data
+        var decompressed = pako.inflate(new Uint8Array(buffer), { to: 'string' });
+        // Now use Papa Parse to parse the decompressed CSV data
+        Papa.parse(decompressed, {
+            header: true,
+            complete: results => {
+                console.log(results);
+                data = results;
+                hideLoading();
+                populateOptions(data);
+                drawBoxPlot();
+                drawRegionBoxPlot();
+            }
+        });
+    })
+    .catch(error => {
+        console.error('There has been a problem with your fetch operation:', error);
     });
 
     function populateOptions(data) {