added kissanime support; added runnerOptions

Author: OpenByteDev

lib/LinkScrapper.js

@@ -1,8 +1,8 @@
 const Scrapper = require('./Scrapper.js');
 
 class LinkScrapper extends Scrapper {
-    constructor({name, domain, runner, exec}={}) {
-        super({name: name, type: "link", domain: domain, runner: runner, exec: exec});
+    constructor({name, domain, runner, runnerOptions={}, exec}={}) {
+        super({name: name, type: "link", domain: domain, runner: runner, runnerOptions: runnerOptions, exec: exec});
     }
 }
 

lib/Scrapper.js

@@ -9,7 +9,7 @@ const runners = require('./runners.js');
 
 
 class Scrapper {
-    constructor({name, type, domain, runner, exec}={}) {
+    constructor({name='Scrapper', type, domain, runner, runnerOptions={}, exec}={}) {
         Object.defineProperties(this, {
             type: {
                 get: () => type,
@@ -32,6 +32,7 @@ class Scrapper {
         this.type = type;
         this.domain = domain;
         this.runner = runner;
+        this.runnerOptions = runnerOptions;
         this.exec = exec;
     }
     isApplicable(url) {
@@ -48,7 +49,11 @@ class Scrapper {
         if (!(this.runner in runners))
             throw new TypeError('runner is not supported');
         try {
-            info = await runners[this.runner].run(url, this.exec);
+            info = await runners[this.runner].run({
+                url: url,
+                scrapper: this.exec,
+                options: this.runnerOptions
+            });
         } catch(err) {
             console.log(err);
             info = null; //new Error(1);

lib/StreamScrapper.js

@@ -1,8 +1,8 @@
 const Scrapper = require('./Scrapper.js');
 
 class StreamScrapper extends Scrapper {
-    constructor({name, domain, runner, exec}={}) {
-        super({name: name, type: 'stream', domain: domain, runner: runner, exec: exec});
+    constructor({name, domain, runner, runnerOptions={}, exec}={}) {
+        super({name: name, type: 'stream', domain: domain, runner: runner, runnerOptions: runnerOptions, exec: exec});
     }
 }
 

lib/runners.js

@@ -1,5 +1,6 @@
 require("./extensions.js");
 const typeOf = require("typeof");
+const objectMerge = require('object-merge');
 
 const Runner = require('./Runner.js');
 
@@ -8,24 +9,52 @@ const config = require('./config.js');
 
 let runners = {};
 
-const puppeteerRunner = new Runner('puppeteer', async (url, scrapper) => {
+const puppeteerRunner = new Runner('puppeteer', async ({url, scrapper, options={}}={}) => {
     const puppeteer = require('puppeteer');
 
-    const browser = await puppeteer.launch(config.puppeteer);
+    let _options = {
+        config: {
+            headless: false
+        },
+        requestInterception: {
+            active: true,
+            block: ({request}) => request.resourceType === 'font'
+        }
+    };
+    _options = objectMerge(_options, options.puppeteer);
+    _options.config = objectMerge(_options.config, config.puppeteer);
+
+    const browser = await puppeteer.launch(_options.config);
     const page = await browser.newPage();
 
     await page.emulateMedia('screen');
 
-    await page.setRequestInterception(true);
+    await page.setRequestInterception(_options.requestInterception.active);
     page.on('request', request => {
-        if (//request.resourceType === 'image' ||
-            //request.resourceType === 'stylesheet' ||
-            request.resourceType === 'font')
+        const block = _options.requestInterception.block && _options.requestInterception.block({
+            request: request,
+            resourceType: request.resourceType(),
+            url: request.url(),
+            page: page,
+            browser: browser,
+            puppeteer: puppeteer
+        });
+
+        if (typeOf(block) === 'undefined')
+            return;
+        if (block)
             request.abort();
         else
             request.continue();
     });
 
+    if (_options.init)
+        _options.init({
+            page: page,
+            browser: browser,
+            puppeteer: puppeteer
+        });
+
     /*await page.evaluateOnNewDocument(async () => {
         HTMLVideoElement.prototype.canPlayType = function () { return "probably"; };
     });*/
@@ -44,37 +73,45 @@ const puppeteerRunner = new Runner('puppeteer', async (url, scrapper) => {
 
     return ret;
 });
-const htmlRunner = new Runner('html', async (url, scrapper) => {
+const htmlRunner = new Runner('html', async ({url, scrapper, options={}}={}) => {
     const axios = require('axios');
 
-    return await axios.get(url, {
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
-    }).then(async response => await
-        scrapper({
+    let config = {
+        url: url,
+        method: 'get',
+        headers: {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
+        }
+    };
+    if (options.axios && options.axios.config)
+        config = objectMerge(config, options.axios.config);
+
+    const response = await axios.request(config);
+    return await scrapper({
         url: url,
         html: response.data,
+        response: response,
         scrapper: scrapper,
         runners: runners
-    }));
+    });
 });
-const domRunner = new Runner('dom', async (url, scrapper) => {
-    const {default: HTMLStringParser} = require("htmlstringparser");
-
-    return await htmlRunner.run(url, async ({html}) => {
-        const dom = new HTMLStringParser(html);
-
-        return await scrapper({
-            url: url,
-            html: html,
-            parser: HTMLStringParser,
-            dom: dom,
-            scrapper: scrapper,
-            runners: runners
-        });
+const domRunner = new Runner('dom', async ({url, scrapper, options={}}={}) => {
+    const { 'default': HTMLStringParser } = require("htmlstringparser");
+
+    return await htmlRunner.run({
+        url: url,
+        scrapper: async (args) => {
+            const dom = new HTMLStringParser(args.html);
+
+            args.parser = HTMLStringParser;
+            args.dom = dom;
+            return await scrapper(args);
+        },
+        options: options
     });
 });
-const urlRunner = new Runner('url', async (url, scrapper) => {
+const urlRunner = new Runner('url', async ({url, scrapper, options={}}={}) => {
     return await scrapper({
         url: url,
         scrapper: scrapper,