Optimize data handling: Implement rounding and gzip compression for CSV output

- Implement rounding to four decimal places for specific numerical columns.
- gzip compression for CSV outputs to reduce file size and improve bandwidth usage during data transfers.

Author: AhmedBasem20

.github/workflows/website.yml

@@ -46,9 +46,12 @@ jobs:
         with:
           name: 'Data'
 
+      - name: 'Filter and compress results file.'
+        run: python tests/IVIMmodels/unit_tests/reduce_output_size.py
+
       - name: move data to the dashboard folder
         run: |
-          mv test_output.csv website/dashboard
+          mv test_output.csv.gz website/dashboard
 
       - name: Build documentation
         run: |

tests/IVIMmodels/unit_tests/reduce_output_size.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+df = pd.read_csv('test_output.csv')
+
+# Columns to be rounded to four decimal places
+columns_to_round = [
+    'f', 'Dp', 'D', 'f_fitted', 'Dp_fitted', 'D_fitted',
+    'bval_0.0', 'bval_1.0', 'bval_2.0', 'bval_5.0', 'bval_10.0', 'bval_20.0',
+    'bval_30.0', 'bval_50.0', 'bval_75.0', 'bval_100.0', 'bval_150.0', 'bval_250.0',
+    'bval_350.0', 'bval_400.0', 'bval_550.0', 'bval_700.0', 'bval_850.0', 'bval_1000.0'
+]
+for column in columns_to_round:
+    df[column] = df[column].round(4)
+
+#df = df.loc[:, ~df.columns.str.startswith('bval')] 
+
+#compress and save the file.
+df.to_csv('test_output.csv.gz', compression='gzip', index=False)

website/dashboard/index.html

@@ -8,6 +8,7 @@
   <script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.7.1/jszip.min.js"></script>
   <script src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>
   <script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.3.0/papaparse.min.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/pako@1.0.11/dist/pako.min.js"></script>
   <script src="index.js"></script>
   <link rel="stylesheet" href="index.css">
 </head>

website/dashboard/index.js

@@ -205,17 +205,31 @@ document.addEventListener('DOMContentLoaded', function() {
 
     showLoading();
 
-    Papa.parse('test_output.csv', {
-        download: true,
-        header: true,
-        complete: results => {
-            data = results;
-            hideLoading();
-            populateOptions(data);
-            drawBoxPlot();
-            drawRegionBoxPlot();
-
+    fetch('test_output.csv.gz')
+    .then(response => {
+        if (!response.ok) {
+            throw new Error('Network response was not ok');
         }
+        return response.arrayBuffer();
+    })
+    .then(buffer => {
+        // Use pako to decompress the data
+        var decompressed = pako.inflate(new Uint8Array(buffer), { to: 'string' });
+        // Now use Papa Parse to parse the decompressed CSV data
+        Papa.parse(decompressed, {
+            header: true,
+            complete: results => {
+                console.log(results);
+                data = results;
+                hideLoading();
+                populateOptions(data);
+                drawBoxPlot();
+                drawRegionBoxPlot();
+            }
+        });
+    })
+    .catch(error => {
+        console.error('There has been a problem with your fetch operation:', error);
     });
 
     function populateOptions(data) {