Adding html scraper block.

Author: matthewhudson

blocks/scrape-url-html/README.md

@@ -0,0 +1,6 @@
+# Scrape HTML
+
+Requires a free account w/
+[Scraper API](https://www.scraperapi.com/?fp_ref=hudson) (includes 1000
+request/mo, sessions, and an unbelievable affiliate program), and setting the
+following two `.env` variables: `SCRAPER_API_KEY` and `SCRAPER_API_ORIGIN`.

blocks/scrape-url-html/handler.js

@@ -0,0 +1,32 @@
+const axios = require('axios')
+
+const { SCRAPER_API_ORIGIN, SCRAPER_API_KEY } = process.env
+
+const handler = async inputs => {
+  const { url, session } = inputs
+  const output = await axios
+    .get(SCRAPER_API_ORIGIN, {
+      params: {
+        api_key: SCRAPER_API_KEY,
+        url,
+        session_number: session
+      }
+    })
+    .then(res => {
+      if (res.status !== 200) {
+        throw new Error(res.status)
+      }
+
+      return { html: res.data }
+    })
+    .catch(err => {
+      const { message, stack } = err
+      return {
+        error: { message, stack }
+      }
+    })
+
+  return output
+}
+
+module.exports = handler

blocks/scrape-url-html/index.js

@@ -0,0 +1,20 @@
+const { Block } = require('node-webpipe')
+const handler = require('./handler')
+
+new Block()
+  .name('Scrape URL HTML')
+  .description('Retrieve the fully rendered HTML content via Scraper API.')
+  .input('url', 'string', 'A url to scrape.')
+  .input(
+    'session',
+    'number',
+    'Any random integer. Allows you to continue using the same proxy for each request with that session number. Send a new integer to create a new session.'
+  )
+  .output('html', 'string', 'The raw HTML from the requested URL.')
+  .handle(async (inputs, cb) => {
+    // @TODO: This handler should be the main responsibility in this file,
+    // handle definition in export like lambda-api, or elsewhere
+    const html = await handler(inputs)
+    cb(null, { html: html.html })
+  })
+  .listen()

blocks/scrape-url-html/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "scrape-url-html",
+  "version": "1.0.0",
+  "dependencies": {
+    "axios": "^0.19.0"
+  }
+}

now.json

@@ -3,9 +3,11 @@
   "version": 2,
   "builds": [
     { "src": "blocks/calculate-square-root/*.js", "use": "@now/node" },
+    { "src": "blocks/parse-url/*.js", "use": "@now/node" },
     { "src": "blocks/parse-rss/*.js", "use": "@now/node" },
     { "src": "blocks/convert-markdown-to-html/*.js", "use": "@now/node" },
     { "src": "blocks/pkg-news-api/*.js", "use": "@now/node" },
+    { "src": "blocks/scrape-url-html/*.js", "use": "@now/node" },
     { "src": "blocks/upload-url-to-s3/*.js", "use": "@now/node" },
     { "src": "blocks/inspect-hostname-dns/*.js", "use": "@now/node" }
   ],
@@ -20,6 +22,8 @@
     "AWS_S3_ACCESS_KEY": "@aws_s3_access_key",
     "AWS_S3_SECRET_KEY": "@aws_s3_secret_key",
     "AWS_S3_BUCKET": "@aws_s3_bucket",
-    "API_KEY_NEWSAPI": "api_key_newsapi"
+    "API_KEY_NEWSAPI": "@api_key_newsapi",
+    "SCRAPER_API_ORIGIN": "@scraper_api_origin",
+    "SCRAPER_API_KEY": "@scraper_api_key"
   }
 }