upload result to s3 for callback mode (POST: /runs)

Author: abhishek97

config.js

@@ -23,5 +23,14 @@ exports = module.exports = {
     PASS: process.env.AMQP_PASS || 'codingblocks',
     HOST: process.env.AMQP_HOST || 'localhost',
     PORT: process.env.AMQP_PORT || 5672
+  },
+
+  S3: {
+    endpoint: process.env.S3_ENDPOINT || 'localhost',
+    port: process.env.S3_PORT || 9000,
+    ssl: process.env.S3_SSL || false,
+    accessKey: process.env.S3_ACCESS_KEY || '',
+    secretKey: process.env.S3_SECRET_KEY || '',
+    bucket: process.env.S3_BUCKET || 'judge-submissions'
   }
 }

migrations/001-addOutputsToSubmission.sql

@@ -0,0 +1,2 @@
+alter table submissions 
+  add column outputs varchar[];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "judge-api",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Judge API",
   "main": "dist/server.js",
   "repository": "https://github.com/coding-blocks/judge-api",
@@ -15,14 +15,17 @@
     "base-64": "^0.1.0",
     "debug": "^4.0.0",
     "express": "^4.16.2",
+    "minio": "^7.0.3",
     "pg": "^7.4.3",
     "pg-hstore": "^2.3.2",
-    "sequelize": "^4.22.6"
+    "sequelize": "^4.22.6",
+    "uuid": "^3.3.2"
   },
   "devDependencies": {
     "@types/chai": "^4.0.4",
     "@types/debug": "^0.0.30",
     "@types/express": "^4.0.39",
+    "@types/minio": "^7.0.1",
     "@types/mocha": "^5.0.0",
     "@types/request": "^2.0.8",
     "@types/sequelize": "^4.0.79",

src/db/models.ts

@@ -42,7 +42,8 @@ const Submissions = db.define('submissions', {
   },
   start_time: Sequelize.DATE,
   end_time: Sequelize.DATE,
-  results: Sequelize.ARRAY(Sequelize.INTEGER)
+  results: Sequelize.ARRAY(Sequelize.INTEGER),
+  outputs: Sequelize.ARRAY(Sequelize.STRING)
 }, {
   paranoid: true, // We do not want to lose any submission data
   timestamps: false // Start and end times are already logged
@@ -53,6 +54,7 @@ export type SubmissionAttributes = {
   start_time: Date
   end_time?: Date
   results?: Array<number>
+  outputs?: Array<string>
 }
 
 const ApiKeys = db.define('apikeys', {

src/routes/api/run.ts

@@ -1,9 +1,10 @@
 import {Response, Router, Request} from 'express'
 import axios from 'axios'
 
-import {SubmissionAttributes, Submissions} from '../../db/models'
+import {SubmissionAttributes, Submissions, db} from '../../db/models'
 import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue'
 import {isInvalidRunRequest} from '../../validators/SubmissionValidators'
+import {upload} from '../../utils/s3'
 import config = require('../../../config')
 
 const route: Router = Router()
@@ -58,7 +59,22 @@ const handleSuccessForSubmission = function (result: RunResponse) {
       break;
     case 'callback':
       // send a post request to callback 
-      axios.post(job.callback, result)
+      (async () => {
+        // 1. upload the result to s3 and get the url
+        const {url} = await upload(result)
+
+        // 2. save the url in db
+        await Submissions.update({
+          outputs: [url]
+        }, {
+          where: {
+            id: result.id
+          }
+        })
+
+        // make the callback request
+        await axios.post(job.callback, {id: result.id, outputs: [url]})
+      })()
       break;
   }
 }
@@ -112,6 +128,13 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo
  *  }
  *  @apiSuccessExample {JSON} Success-Response(mode=callback):
  *  HTTP/1.1 200 OK
+ * 
+ *  @apiSuccessExample {JSON} Body for Callback(mode=callback):
+ *  HTTP/1.1 200 OK
+ *  {
+ *    "id": 10,
+ *    "outputs": ["http://localhost/judge-submissions/file.json"]
+ *  }
  */
 route.post('/', (req, res, next) => {
   const invalidRequest = isInvalidRunRequest(req)

src/utils/s3.ts

@@ -0,0 +1,27 @@
+import Minio = require('minio')
+import v4 =  require('uuid/v4')
+import config = require('../../config')
+
+const client = new Minio.Client({
+  endPoint: config.S3.endpoint,
+  port: config.S3.port,
+  useSSL: config.S3.ssl,
+  accessKey: config.S3.accessKey,
+  secretKey: config.S3.secretKey,
+})
+
+export type savedFile = {
+  etag: string,
+  url: string
+}
+
+export const urlForFilename = (bucket: string, filename: string) : string => `http${config.S3.ssl ? 's': ''}://${config.S3.endpoint}/${bucket}/${filename}`
+
+export const upload = function (object:object, filename:string = v4() + '.json' ,bucket:string = config.S3.bucket) : Promise<savedFile> {
+  return new Promise((resolve, reject) => {
+    client.putObject(bucket, filename, JSON.stringify(object), function(err, etag) {
+      if (err) return reject(err)
+      resolve({etag, url: urlForFilename(bucket, filename) })
+    })
+  })
+}