Merge pull request #235 from acrellin/download_features_models

Allow computed features and models to be downloaded

Author: acrellin

cesium_app/app_server.py

@@ -56,8 +56,10 @@ def make_app(cfg, baselayer_handlers, baselayer_settings):
     handlers = baselayer_handlers + [
         (r'/project(/.*)?', ProjectHandler),
         (r'/dataset(/.*)?', DatasetHandler),
-        (r'/features(/.*)?', FeatureHandler),
-        (r'/models(/.*)?', ModelHandler),
+        (r'/features(/[0-9]+)?', FeatureHandler),
+        (r'/features/([0-9]+)/(download)', FeatureHandler),
+        (r'/models(/[0-9]+)?', ModelHandler),
+        (r'/models/([0-9]+)/(download)', ModelHandler),
         (r'/predictions(/[0-9]+)?', PredictionHandler),
         (r'/predictions/([0-9]+)/(download)', PredictionHandler),
         (r'/predict_raw_data', PredictRawDataHandler),

cesium_app/handlers/feature.py

@@ -13,20 +13,35 @@
 from os.path import join as pjoin
 import uuid
 import datetime
+import pandas as pd
 
 
 class FeatureHandler(BaseHandler):
     @auth_or_token
-    def get(self, featureset_id=None):
-        if featureset_id is not None:
-            featureset_info = Featureset.get_if_owned_by(featureset_id,
-                                                         self.current_user)
+    def get(self, featureset_id=None, action=None):
+        if action == 'download':
+            featureset = Featureset.get_if_owned_by(featureset_id,
+                                                    self.current_user)
+            fset_path = featureset.file_uri
+            fset, data = featurize.load_featureset(fset_path)
+            if 'labels' in data:
+                fset['labels'] = data['labels']
+            self.set_header("Content-Type", 'text/csv; charset="utf-8"')
+            self.set_header(
+                "Content-Disposition", "attachment; "
+                f"filename=cesium_featureset_{featureset.project.name}"
+                f"_{featureset.name}_{featureset.finished}.csv")
+            self.write(fset.to_csv(index=True))
         else:
-            featureset_info = [f for p in self.current_user.projects
-                               for f in p.featuresets]
-            featureset_info.sort(key=lambda f: f.created_at, reverse=True)
-
-        self.success(featureset_info)
+            if featureset_id is not None:
+                featureset_info = Featureset.get_if_owned_by(featureset_id,
+                                                             self.current_user)
+            else:
+                featureset_info = [f for p in self.current_user.projects
+                                   for f in p.featuresets]
+                featureset_info.sort(key=lambda f: f.created_at, reverse=True)
+
+            self.success(featureset_info)
 
     @auth_or_token
     async def _await_featurization(self, future, fset):

cesium_app/handlers/model.py

@@ -13,6 +13,7 @@
 import uuid
 import datetime
 
+import sklearn
 from sklearn.model_selection import GridSearchCV
 import joblib
 
@@ -72,14 +73,28 @@ def _build_model_compute_statistics(fset_path, model_type, model_params,
 
 class ModelHandler(BaseHandler):
     @auth_or_token
-    def get(self, model_id=None):
-        if model_id is not None:
-            model_info = Model.get_if_owned_by(model_id, self.current_user)
+    def get(self, model_id=None, action=None):
+        if action == 'download':
+            model = Model.get_if_owned_by(model_id, self.current_user)
+            model_path = model.file_uri
+            with open(model_path, 'rb') as f:
+                model_data = f.read()
+            self.set_header("Content-Type", "application/octet-stream")
+            self.set_header(
+                "Content-Disposition", "attachment; "
+                f"filename=cesium_model_{model.project.name}"
+                f"_{model.name}_{str(model.finished).replace(' ', 'T')}"
+                f"_joblib_v{joblib.__version__}"
+                f"_sklearn_v{sklearn.__version__}.pkl")
+            self.write(model_data)
         else:
-            model_info = [model for p in self.current_user.projects
-                          for model in p.models]
+            if model_id is not None:
+                model_info = Model.get_if_owned_by(model_id, self.current_user)
+            else:
+                model_info = [model for p in self.current_user.projects
+                              for model in p.models]
 
-        return self.success(model_info)
+            return self.success(model_info)
 
     @auth_or_token
     async def _await_model_statistics(self, model_stats_future, model):

cesium_app/handlers/prediction.py

@@ -121,8 +121,8 @@ async def post(self):
     @auth_or_token
     def get(self, prediction_id=None, action=None):
         if action == 'download':
-            pred_path = Prediction.get_if_owned_by(prediction_id,
-                                                   self.current_user).file_uri
+            prediction = Prediction.get_if_owned_by(prediction_id, self.current_user)
+            pred_path = prediction.file_uri
             fset, data = featurize.load_featureset(pred_path)
             result = pd.DataFrame(({'label': data['labels']}
                                    if len(data['labels']) > 0 else None),
@@ -133,8 +133,11 @@ def get(self, prediction_id=None, action=None):
                 result['prediction'] = data['preds']
             result.index.name = 'ts_name'
             self.set_header("Content-Type", 'text/csv; charset="utf-8"')
-            self.set_header("Content-Disposition", "attachment; "
-                            "filename=cesium_prediction_results.csv")
+            self.set_header(
+                "Content-Disposition", "attachment; "
+                f"filename=cesium_prediction_results_{prediction.project.name}"
+                f"_{prediction.dataset.name}"
+                f"_{prediction.model.name}_{prediction.finished}.csv")
             self.write(result.to_csv(index=True))
         else:
             if prediction_id is None:

cesium_app/tests/frontend/test_predict.py

@@ -10,7 +10,12 @@
 import pandas as pd
 import json
 import subprocess
+import glob
 from cesium_app.model_util import create_token_user
+from baselayer.app.config import load_config
+
+
+cfg = load_config()
 
 
 def _add_prediction(proj_id, driver):
@@ -154,41 +159,47 @@ def test_download_prediction_csv_class(driver, project, dataset, featureset,
                                        model, prediction):
     driver.get('/')
     _click_download(project.id, driver)
-    assert os.path.exists('/tmp/cesium_prediction_results.csv')
+    matching_downloads_paths = glob.glob(f'{cfg["paths:downloads_folder"]}/'
+                                         'cesium_prediction_results*.csv')
+    assert len(matching_downloads_paths) == 1
     try:
         npt.assert_equal(
-            np.genfromtxt('/tmp/cesium_prediction_results.csv', dtype='str'),
+            np.genfromtxt(matching_downloads_paths[0], dtype='str'),
             ['ts_name,label,prediction',
              '0,Mira,Mira',
              '1,Classical_Cepheid,Classical_Cepheid',
              '2,Mira,Mira',
              '3,Classical_Cepheid,Classical_Cepheid',
              '4,Mira,Mira'])
     finally:
-        os.remove('/tmp/cesium_prediction_results.csv')
+        os.remove(matching_downloads_paths[0])
 
 
 @pytest.mark.parametrize('model__type', ['LinearSGDClassifier'])
 def test_download_prediction_csv_class_unlabeled(driver, project, unlabeled_prediction):
     driver.get('/')
     _click_download(project.id, driver)
-    assert os.path.exists('/tmp/cesium_prediction_results.csv')
+    matching_downloads_paths = glob.glob(f'{cfg["paths:downloads_folder"]}/'
+                                         'cesium_prediction_results*.csv')
+    assert len(matching_downloads_paths) == 1
     try:
-        result = np.genfromtxt('/tmp/cesium_prediction_results.csv', dtype='str')
+        result = np.genfromtxt(matching_downloads_paths[0], dtype='str')
         assert result[0] == 'ts_name,prediction'
         assert all([el[0].isdigit() and el[1] == ',' and el[2:] in
                     ['Mira', 'Classical_Cepheid'] for el in result[1:]])
     finally:
-        os.remove('/tmp/cesium_prediction_results.csv')
+        os.remove(matching_downloads_paths[0])
 
 
 def test_download_prediction_csv_class_prob(driver, project, dataset,
                                             featureset, model, prediction):
     driver.get('/')
     _click_download(project.id, driver)
-    assert os.path.exists('/tmp/cesium_prediction_results.csv')
+    matching_downloads_paths = glob.glob(f'{cfg["paths:downloads_folder"]}/'
+                                         'cesium_prediction_results*.csv')
+    assert len(matching_downloads_paths) == 1
     try:
-        result = pd.read_csv('/tmp/cesium_prediction_results.csv')
+        result = pd.read_csv(matching_downloads_paths[0])
         npt.assert_array_equal(result.ts_name, np.arange(5))
         npt.assert_array_equal(result.label, ['Mira', 'Classical_Cepheid',
                                               'Mira', 'Classical_Cepheid',
@@ -198,16 +209,19 @@ def test_download_prediction_csv_class_prob(driver, project, dataset,
                                [1, 0, 1, 0, 1])
         assert (pred_probs.values >= 0.0).all()
     finally:
-        os.remove('/tmp/cesium_prediction_results.csv')
+        os.remove(matching_downloads_paths[0])
 
 
 @pytest.mark.parametrize('featureset__name, model__type', [('regr', 'LinearRegressor')])
-def test_download_prediction_csv_regr(driver, project, dataset, featureset, model, prediction):
+def test_download_prediction_csv_regr(driver, project, dataset, featureset,
+                                      model, prediction):
     driver.get('/')
     _click_download(project.id, driver)
-    assert os.path.exists('/tmp/cesium_prediction_results.csv')
+    matching_downloads_paths = glob.glob(f'{cfg["paths:downloads_folder"]}/'
+                                         'cesium_prediction_results*.csv')
+    assert len(matching_downloads_paths) == 1
     try:
-        results = np.genfromtxt('/tmp/cesium_prediction_results.csv',
+        results = np.genfromtxt(matching_downloads_paths[0],
                                 dtype='str', delimiter=',')
         npt.assert_equal(results[0],
                          ['ts_name', 'label', 'prediction'])
@@ -219,7 +233,7 @@ def test_download_prediction_csv_regr(driver, project, dataset, featureset, mode
              [3, 2.2, 2.2],
              [4, 3.1, 3.1]])
     finally:
-        os.remove('/tmp/cesium_prediction_results.csv')
+        os.remove(matching_downloads_paths[0])
 
 
 def test_predict_specific_ts_name(driver, project, dataset, featureset, model):

static/css/base.css

@@ -33,4 +33,7 @@ body {
 .loginBox .logo {
     float: left;
     padding: 1em;
-}
+}
+a:hover {
+ cursor:pointer;
+}

static/js/components/Download.jsx

@@ -0,0 +1,27 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+
+
+const Download = (props) => {
+  const style = {
+    display: 'inline-block'
+  };
+  return (
+    <a
+      href={props.url}
+      style={style}
+      onClick={
+        (e) => {
+          e.stopPropagation();
+        }
+      }
+    >
+      Download
+    </a>
+  );
+};
+Download.propTypes = {
+  url: PropTypes.string.isRequired
+};
+
+export default Download;

static/js/components/Features.jsx

@@ -13,6 +13,7 @@ import Plot from './Plot';
 import FoldableRow from './FoldableRow';
 import { reformatDatetime, contains } from '../utils';
 import Delete from './Delete';
+import Download from './Download';
 
 const { Tab, Tabs, TabList, TabPanel } = { ...ReactTabs };
 
@@ -257,7 +258,14 @@ export let FeatureTable = props => (
                 <td>{featureset.name}</td>
                 <td>{reformatDatetime(featureset.created_at)}</td>
                 {status}
-                <td><DeleteFeatureset ID={featureset.id} /></td>
+                <td>
+                  {
+                    done &&
+                    <Download url={`/features/${featureset.id}/download`} />
+                  }
+                  &nbsp;&nbsp;
+                  <DeleteFeatureset ID={featureset.id} />
+                </td>
               </tr>
               {foldedContent}
             </FoldableRow>
@@ -290,7 +298,6 @@ FeatureTable = connect(ftMapStateToProps)(FeatureTable);
 const mapDispatchToProps = dispatch => (
   { delete: id => dispatch(Action.deleteFeatureset(id)) }
 );
-
 const DeleteFeatureset = connect(null, mapDispatchToProps)(Delete);
 
 export default FeaturesTab;

static/js/components/Models.jsx

@@ -10,6 +10,7 @@ import * as Validate from '../validate';
 import * as Action from '../actions';
 import Expand from './Expand';
 import Delete from './Delete';
+import Download from './Download';
 import { $try, reformatDatetime } from '../utils';
 import FoldableRow from './FoldableRow';
 
@@ -237,7 +238,14 @@ export let ModelTable = props => (
               <td>{model.name}</td>
               <td>{reformatDatetime(model.created_at)}</td>
               {status}
-              <td><DeleteModel ID={model.id} /></td>
+              <td>
+                {
+                  done &&
+                  <Download url={`/models/${model.id}/download`} />
+                }
+                &nbsp;&nbsp;
+                <DeleteModelButton ID={model.id} />
+              </td>
             </tr>
             {foldedContent}
           </FoldableRow>
@@ -265,11 +273,10 @@ const mtMapStateToProps = (state, ownProps) => (
 ModelTable = connect(mtMapStateToProps)(ModelTable);
 
 
-const dmMapDispatchToProps = dispatch => (
+const deleteMapDispatchToProps = dispatch => (
   { delete: id => dispatch(Action.deleteModel(id)) }
 );
-
-const DeleteModel = connect(null, dmMapDispatchToProps)(Delete);
+const DeleteModelButton = connect(null, deleteMapDispatchToProps)(Delete);
 
 
 export default ModelsTab;

static/js/components/Predictions.jsx

@@ -135,7 +135,10 @@ let PredictionsTable = props => (
               <td>{reformatDatetime(prediction.created_at)}</td>
               {status}
               <td>
-                <DownloadPredCSV ID={prediction.id} />
+                {
+                  done &&
+                  <DownloadPredCSV ID={prediction.id} />
+                }
                 &nbsp;&nbsp;
                 <DeletePrediction ID={prediction.id} />
               </td>