Clone from gulp-etl-tap-csv

Author: donpedro

.gitignore

@@ -0,0 +1,3 @@
+/node_modules
+.serverless
+jspm_packages

.vscode/launch.json

@@ -0,0 +1,74 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "GulpTester",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
+            "stopOnEntry": false,
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "default"],
+            "cwd": "${workspaceRoot}",
+            "runtimeArgs": [
+                "--nolazy", "-r", "ts-node/register"
+            ],
+            "console": "internalConsole",
+            "env": {"DEBUG_LEVEL":"info"}
+        },
+        {        
+            "name": "GulpTester Debug",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
+            "stopOnEntry": false,
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "default"],
+            "cwd": "${workspaceRoot}",
+            "runtimeArgs": [
+                "--nolazy", "-r", "ts-node/register"
+            ],
+            "console": "internalConsole",
+            "env": {"DEBUG_LEVEL":"debug"}
+        },
+        {        
+            "name": "GulpTester Debug - Buffer Mode",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
+            "stopOnEntry": false,
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "runTapCsvBuffer"],
+            "cwd": "${workspaceRoot}",
+            "runtimeArgs": [
+                "--nolazy", "-r", "ts-node/register"
+            ],
+            "console": "internalConsole",
+            "env": {"DEBUG_LEVEL":"debug"}
+        },
+        {
+            "name": "csvParseWithoutGulp",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
+            "stopOnEntry": false,
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "csvParseWithoutGulp"],
+            "cwd": "${workspaceRoot}",
+            "runtimeArgs": [
+                "--nolazy", "-r", "ts-node/register"
+            ],
+            "console": "internalConsole",
+            "env": {"DEBUG_LEVEL":"info"}
+        },
+        {
+            "name": "aws-doParse: debug TS w/o transpile using ts-node",
+            "type": "node",
+            "request": "launch",
+            "args": ["src/aws/dev-handler.ts"],
+            "runtimeArgs": ["-r", "ts-node/register"],
+            "cwd": "${workspaceRoot}",
+            "protocol": "inspector",
+            "internalConsoleOptions": "openOnSessionStart",
+        },
+    ]
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 gulp ETL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,57 @@
+# gulp-etl-tap-mysql #
+
+*(this plugin is being developed from **gulp-etl-tap-mysql**. The original readme from [gulp-etl-tap-csv](https://github.com/gulpetl/gulp-etl-tap-csv) is below)*
+
+This plugin  converts CSV files to **gulp-etl** **Message Stream** files; originally adapted from the [gulp-etl-handlelines](https://github.com/gulpetl/gulp-etl-handlelines) model plugin. It is a **gulp-etl** wrapper for [csv-parse](https://csv.js.org/parse/).
+
+This is a **[gulp-etl](https://gulpetl.com/)** plugin, and as such it is a [gulp](https://gulpjs.com/) plugin. **gulp-etl** plugins work with [ndjson](http://ndjson.org/) data streams/files which we call **Message Streams** and which are compliant with the [Singer specification](https://github.com/singer-io/getting-started/blob/master/docs/SPEC.md#output). In the **gulp-etl** ecosystem, **taps** tap into an outside format or system (in this case, a CSV file) and convert their contents/output to a Message Stream, and **targets** convert/output Message Streams to an outside format or system. In this way, these modules can be stacked to convert from one format or system to another, either directly or with tranformations or other parsing in between. Message Streams look like this:
+
+```
+{"type": "SCHEMA", "stream": "users", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}}}}
+{"type": "RECORD", "stream": "users", "record": {"id": 1, "name": "Chris"}}
+{"type": "RECORD", "stream": "users", "record": {"id": 2, "name": "Mike"}}
+{"type": "SCHEMA", "stream": "locations", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}}}}
+{"type": "RECORD", "stream": "locations", "record": {"id": 1, "name": "Philadelphia"}}
+{"type": "STATE", "value": {"users": 2, "locations": 1}}
+```
+
+### Usage
+**gulp-etl** plugins accept a configObj as the first parameter; the configObj
+will contain any info the plugin needs. For this plugin the configObj is the "Options" object for [csv-parse](https://csv.js.org/parse/), described [here](https://csv.js.org/parse/options/); the only difference is that the "columns" property cannot be falsey, since it would result in arrays being returned
+for each row instead of objects. A falsey value for columns will be overridden to true.
+
+##### Sample gulpfile.js
+```
+/* parse all .CSV files in a folder into Message Stream files in a different folder */
+
+let gulp = require('gulp')
+var rename = require('gulp-rename')
+var tapCsv = require('gulp-etl-tap-csv').tapCsv
+
+exports.default = function() {
+    return gulp.src('data/*.csv')
+    .pipe(tapCsv({ columns:true }))
+    .pipe(rename({ extname: ".ndjson" })) // rename to *.ndjson
+    .pipe(gulp.dest('output/'));
+}
+```
+### Quick Start for Coding on This Plugin
+* Dependencies: 
+    * [git](https://git-scm.com/downloads)
+    * [nodejs](https://nodejs.org/en/download/releases/) - At least v6.3 (6.9 for Windows) required for TypeScript debugging
+    * npm (installs with Node)
+    * typescript - installed as a development dependency
+* Clone this repo and run `npm install` to install npm packages
+* Debug: with [VScode](https://code.visualstudio.com/download) use `Open Folder` to open the project folder, then hit F5 to debug. This runs without compiling to javascript using [ts-node](https://www.npmjs.com/package/ts-node)
+* Test: `npm test` or `npm t`
+* Compile to javascript: `npm run build`
+
+### Testing
+
+We are using [Jest](https://facebook.github.io/jest/docs/en/getting-started.html) for our testing. Each of our tests are in the `test` folder.
+
+- Run `npm test` to run the test suites
+
+
+
+Note: This document is written in [Markdown](https://daringfireball.net/projects/markdown/). We like to use [Typora](https://typora.io/) and [Markdown Preview Plus](https://chrome.google.com/webstore/detail/markdown-preview-plus/febilkbfcbhebfnokafefeacimjdckgl?hl=en-US) for our Markdown work..

debug/gulpfile.ts

@@ -0,0 +1,67 @@
+let gulp = require('gulp')
+import { tapCsv } from '../src/plugin'
+
+import * as loglevel from 'loglevel'
+const log = loglevel.getLogger('gulpfile')
+log.setLevel((process.env.DEBUG_LEVEL || 'warn') as log.LogLevelDesc)
+// if needed, you can control the plugin's logging level separately from 'gulpfile' logging above
+// const pluginLog = loglevel.getLogger(PLUGIN_NAME)
+// pluginLog.setLevel('debug')
+
+import * as rename from 'gulp-rename'
+const errorHandler = require('gulp-error-handle'); // handle all errors in one handler, but still stop the stream if there are errors
+
+const pkginfo = require('pkginfo')(module); // project package.json info into module.exports
+const PLUGIN_NAME = module.exports.name;
+
+import Vinyl = require('vinyl') 
+
+let gulpBufferMode = false;
+
+function switchToBuffer(callback: any) {
+  gulpBufferMode = true;
+
+  callback();
+}
+
+function runTapCsv(callback: any) {
+  log.info('gulp task starting for ' + PLUGIN_NAME)
+
+  return gulp.src('../testdata/*.csv',{buffer:gulpBufferMode})
+    .pipe(errorHandler(function(err:any) {
+      log.error('Error: ' + err)
+      callback(err)
+    }))
+    .on('data', function (file:Vinyl) {
+      log.info('Starting processing on ' + file.basename)
+    })    
+    .pipe(tapCsv({raw:true/*, info:true */}))
+    .pipe(rename({
+      extname: ".ndjson",
+    }))      
+    .pipe(gulp.dest('../testdata/processed'))
+    .on('data', function (file:Vinyl) {
+      log.info('Finished processing on ' + file.basename)
+    })    
+    .on('end', function () {
+      log.info('gulp task complete')
+      callback()
+    })
+
+}
+
+export function csvParseWithoutGulp(callback: any) {
+
+  const parse = require('csv-parse')
+
+  var parser = parse({delimiter: ',', columns:true});
+  
+  require('fs').createReadStream('../testdata/cars.csv').pipe(parser)
+  .on("data",(data:any)=>{
+    console.log(data)
+  });
+  
+}
+
+exports.default = gulp.series(runTapCsv)
+exports.runTapCsvBuffer = gulp.series(switchToBuffer, runTapCsv)