Merge pull request #431 from mayeaux/caching-update

Caching update

Author: mayeaux

caching/cachePopularUploads.js

@@ -2,6 +2,7 @@ const Promise = require('bluebird');
 const _ = require('lodash');
 const View = require('../models/index').View;
 const Upload = require('../models/index').Upload;
+const moment = require('moment');
 
 const c = {
   l : console.log
@@ -22,11 +23,11 @@ const logCaching = process.env.LOG_CACHING;
 async function getPopularUploads(){
 
   if(logCaching == 'true'){
-    c.l('Getting popular uploads');
+    c.l('Beginning caching popular uploads, getting popular uploads from db');
+    console.log(moment(new Date).format('hh:mm:ss A'))
   }
 
   // TODO: have to have a job to update upload's view amounts
-
   // TODO: have to build 4 arrays of ~1000
 
   const searchQuery = {
@@ -37,14 +38,14 @@ async function getPopularUploads(){
     category : { $exists: true }
   };
 
-  const selectString = 'formattedDuration rating title views checkedViews uploader fileType thumbnailUrl ' +
+  const selectString = 'formattedDuration rating title uploader fileType thumbnailUrl ' +
     'uploadUrl uniqueTag customThumbnailUrl fileExtension thumbnails reacts uncurated category subcategory description';
 
   let popularUploads = await Upload.find(searchQuery).select(selectString).populate('uploader reacts')
     .lean();
 
   if(logCaching == 'true'){
-    c.l('Uploads received from database');
+    c.l('Uploads received from database for popular calculating');
 
     c.l(popularUploads.length);
   }
@@ -60,20 +61,19 @@ async function setPopularUploads(){
     return upload.uploader;
   });
 
-  // calculate view periods for each upload
+  let formattedPopularUploads = [];
 
-  // TODO: why this promise all functionality? this is almost surely what's blowing everything up
-  popularUploads = await Promise.all(popularUploads.map(async function(upload){
+  console.log(moment(new Date).format('hh:mm:ss A'))
 
-    // get all valid views per upload
-    const uploadViews = await View.find({ upload, validity: 'real' }).select('createdAt');
+  for(const upload of popularUploads){
+    const uploadViews = await View.find({ upload, validity: 'real' }).select('createdAt')
 
-    // calculate their views per period (last24h, lastweek)
-    return calculateViewsByPeriod(upload, uploadViews);
-  }));
+    formattedPopularUploads.push(calculateViewsByPeriod(upload, uploadViews));
+  }
 
   if(logCaching == 'true'){
     c.l('Popular uploads have been calculated according to view periods');
+    console.log(moment(new Date).format('hh:mm:ss A'))
   }
 
   // build json objects representing uploads

caching/cacheRecentUploads.js

@@ -3,6 +3,7 @@ const _ = require('lodash');
 const View = require('../models/index').View;
 const Upload = require('../models/index').Upload;
 const categories = require('../config/categories');
+const moment = require('moment');
 
 const javascriptTimeAgo = require('javascript-time-ago');
 javascriptTimeAgo.locale(require('javascript-time-ago/locales/en'));
@@ -21,12 +22,17 @@ const logCaching = process.env.LOG_CACHING;
 /** Get all recent uploads from the database per category and concat them **/
 async function getRecentUploads(){
 
+  if(logCaching == 'true'){
+    console.log(`Beginning running recent uploads`);
+
+  }
+
   let recentUploadsAllCategories = [];
 
   for(const category of categories){
 
     if(logCaching == 'true'){
-      console.log(`Getting uploads for category: ${category.name}`);
+      console.log(`Getting recent uploads for category: ${category.name}`);
     }
 
     const searchQuery = {
@@ -61,6 +67,9 @@ async function getRecentUploads(){
 async function setRecentUploads(){
   let recentUploads  = await getRecentUploads();
 
+  console.log(`Calculating view amounts`);
+  console.log(moment(new Date).format('hh:mm:ss A'))
+
   // calculate view periods for each upload
   recentUploads = await Promise.all(recentUploads.map(async function(upload){
 

caching/calculateUploadViews.js

@@ -2,6 +2,7 @@ const Promise = require('bluebird');
 const _ = require('lodash');
 const View = require('../models/index').View;
 const Upload = require('../models/index').Upload;
+const moment = require('moment');
 
 const c = {
   l : console.log
@@ -22,7 +23,8 @@ const logCaching = process.env.LOG_CACHING;
 async function calculateViewAmounts(){
 
   if(logCaching == 'true'){
-    c.l('Getting popular uploads');
+    c.l('Calculating view amounts for particular videos');
+    console.log(moment(new Date).format('hh:mm:ss A'))
   }
 
   // TODO: have to have a job to update upload's view amounts
@@ -40,13 +42,18 @@ async function calculateViewAmounts(){
   for(const upload of uploads){
     const amountOfViews = await View.countDocuments({ upload: upload.id, validity: 'real' });
 
-    console.log(upload.views);
+    // console.log(upload.views);
 
     upload.views = amountOfViews;
 
     await upload.save();
   }
 
+  if(logCaching == 'true'){
+    c.l('View amounts calculated');
+    console.log(moment(new Date).format('hh:mm:ss A'))
+  }
+
   // if(logCaching == 'true'){
   //   c.l('Uploads received from database');
   //

caching/helpers.js

@@ -52,7 +52,7 @@ var dayAgo = moment().subtract(24, 'hours').toDate();
 var hourAgo = moment().subtract(1, 'hours').toDate();
 var minuteAgo = moment().subtract(1, 'minutes').toDate();
 
-async function calculateViewsByPeriod(upload, uploadViews){
+function calculateViewsByPeriod(upload, uploadViews){
   upload.viewsAllTime = uploadViews.length;
 
   uploadViews = _.filter(uploadViews, function(uploadView){ return uploadView.createdAt > monthAgo; });
@@ -76,4 +76,4 @@ async function calculateViewsByPeriod(upload, uploadViews){
 module.exports = {
   calculateViewsByPeriod,
   buildObjects
-};
+};

caching/runCaching.js

@@ -1,7 +1,16 @@
 const mongoose = require('mongoose');
 const dotenv = require('dotenv');
-
 const logCaching = process.env.LOG_CACHING;
+const jsHelpers = require('../lib/helpers/js-helpers');
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/** FOR FINDING ERRANT LOGS **/
+if(process.env.SHOW_LOG_LOCATION == 'true' || 1 == 2){
+  jsHelpers.showLogLocation();
+}
 
 process.on('uncaughtException', (err) => {
   console.log('Uncaught Exception: ', err);
@@ -51,13 +60,11 @@ console.log(`CACHING IS RUNNING AGAINST: ${database} \n`);
 
 const setCache = require('./setCache'); // index and daily stats
 
+// functions to run caching jobs
 const cacheRecentUploads = require('./cacheRecentUploads'); // index and daily stats
 const cachePopularUploads = require('./cachePopularUploads'); // index and daily stats
-
 const calculateUploadViews = require('./calculateUploadViews'); // index and daily stats
 
-// const cacheRecentUploads = require('./cacheRecentAndPopularUploads');
-
 async function cacheOnlyRecentUploads(){
 
   try {
@@ -73,6 +80,7 @@ async function cacheOnlyRecentUploads(){
 async function cachePopularDailyStatsAndIndex(){
   try {
     await cachePopularUploads();
+    await calculateUploadViews();
     await setCache.setDailyStats();
     await setCache.setIndexValues();
 
@@ -95,24 +103,52 @@ const cacheIntervalInMs = cacheIntervalInMinutes * ( 1000 * 60 );
 
 console.log(`CACHE POPULAR, DAILY STATS AND INDEXES INTERVAL IN MINUTES: ${cacheIntervalInMinutes} \n`);
 
-async function main(){
+let cacheTotalViewsInterval = parseInt(process.env.CACHE_TOTAL_VIEW_INTERVAL) || 6;
 
-  setInterval(cacheOnlyRecentUploads, cacheRecentIntervalInMs);
+const cacheTotalViewsIntervalInMs = cacheTotalViewsInterval * ( 1000 * 60 );
 
-  setInterval(cachePopularDailyStatsAndIndex, cacheIntervalInMs);
+console.log(`CACHE TOTAL VIEW INTERVAL IN MINUTES: ${cacheIntervalInMinutes} \n`);
 
-  setInterval(calculateUploadViews, cacheIntervalInMs);
+// TODO: there is a bug here, where when times line up, they will run multiple jobs at once,
+// TODO: ideally, they should push into an array, run the most recent job in array, then delete it from array,
+// TODO: and run them sequentially
 
+async function runRecentInterval(){
   // calculate and cache recent uploads every minute
   await cacheOnlyRecentUploads();
+  await sleep(cacheRecentIntervalInMs)
+  runRecentInterval();
+}
 
-  await cachePopularDailyStatsAndIndex();
-
-  await calculateUploadViews();
+runRecentInterval();
 
+async function runOtherCaching(){
+  await cachePopularDailyStatsAndIndex();
+  await sleep(cacheIntervalInMs)
+  runOtherCaching()
 }
 
-main();
+runOtherCaching()
+
+// async function main(){
+//
+//   // setInterval(cacheOnlyRecentUploads, cacheRecentIntervalInMs);
+//   //
+//   // setInterval(cachePopularDailyStatsAndIndex, cacheIntervalInMs);
+//   //
+//   // // calculate total amount of views for channel display
+//   // // TODO: couldn't this just be done during popular? maybe not because it doesn't do for sensitive?
+//   // // TODO: then just do a separate job for
+//   // setInterval(calculateUploadViews, cacheTotalViewsIntervalInMs);
+//
+//   // calculate and cache recent uploads every minute
+//   // await cacheOnlyRecentUploads();
+//
+//   // also does total view amounts
+//   await cachePopularDailyStatsAndIndex();
+// }
+
+// main();
 
 // calculateUploadViews()