diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..1937215 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +# EditorConfig is awesome: https://editorconfig.org/ + +# top-most EditorConfig file +root = true + +[*.md] +trim_trailing_whitespace = false + +[*.js] +trim_trailing_whitespace = true + +# Unix-style newlines with a newline ending every file +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +insert_final_newline = true +max_line_length = 100 diff --git a/README.md b/README.md index 75b27af..bd39ef7 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Creates a fresh `glslify-deps` instance. Accepts the following options: * `readFile`: pass in a custom function reading files. * `resolve`: pass in a custom function for resolving require calls. It has the same signature as [glsl-resolve](http://github.com/hughsk/glsl-resolve). +* `transformResolve`: pass in a custom function for resolving non function transforms. * `files`: a filename/source object mapping of files to prepopulate the file cache with. Useful for overriding particular file paths manually, most notably the "entry" file. diff --git a/depper.js b/depper.js new file mode 100644 index 0000000..76e3518 --- /dev/null +++ b/depper.js @@ -0,0 +1,454 @@ +// @ts-check +/** @typedef {import('glsl-resolve')} GlslResolve */ +var Emitter = require('events/') +var inherits = require('inherits') +var map = require('map-limit') + +var { + genInlineName, + cacheWrap, + parseFiles, + getImportName, + extractPreprocessors, + asyncify, +} = require('./utils.js') + + +/** + * @callback GlslTransformSync + * @param {String} filename The absolute path of the file you're transforming. + * @param {String} src The shader source you'd like to transform. + * @param {Object} opts The transform options. + * @returns {String} transformed shader + */ + +/** + * @callback GlslTransformAsync + * @param {String} filename The absolute path of the file you're transforming. + * @param {String} src The shader source you'd like to transform. + * @param {Object} opts The transform options. + * @param {(err: Error, result: String) => any} [cb] callback with the transformed shader + */ + +/** + * @typedef {GlslTransformSync|GlslTransformAsync} GlslTransform + */ + +/** + * @callback TransformRequireSync + * @param {String|GlslTransform} transform + * @param {Object} opts + * @returns {GlslTransform} + */ + +/** + * @callback TransformRequireAsync + * @param {String|GlslTransform} transform + * @param {Object} opts + * @param {(err: Error, transform: GlslTransform) => any} [cb] + */ + +/** + * @typedef {TransformRequireSync|TransformRequireAsync} TransformRequire + */ + +/** + * @typedef {Object} TransformDefinition + * @prop {string|GlslTransform} tr + * @prop {string} name + * @prop {any} opts + */ + +/** + * @typedef {Object} TransformResolved + * @prop {GlslTransform} tr + * @prop {string} name + * @prop {any} opts + */ + +/** + * @typedef {Object} DepperOptions + * @prop {Boolean} [async] Defines the mechanism flow resolution. + * @prop {Function} [readFile] pass in a custom function reading files. + * @prop {GlslResolve} [resolve] pass in a custom function for resolving require calls. It has the same signature as glsl-resolve. + * @prop {Object} [files] a filename/source object mapping of files to prepopulate the file cache with. Useful for overriding. + * @prop {TransformRequireAsync|TransformRequireSync} [transformRequire] pass in a custom function for resolving non function transforms. + */ + +/** + * Creates a new instance of glslify-deps. Generally, you'll + * want to use one instance per bundle. + * + * note: this is an interface to be extended with a top class + * + * @class + * @param {DepperOptions} [opts] options + */ +function Depper(opts) { + if (!(this instanceof Depper)) return new Depper(opts) + // @ts-ignore + Emitter.call(this) + + opts = opts || {} + + this._inlineSource = '' + this._inlineName = genInlineName() + this._async = opts.async || false + this._i = 0 + this._deps = [] + + this._cache = {} + this._fileCache = parseFiles(Object.assign({}, opts.files) || {}) + + /** @type {TransformDefinition[]} */ + this._transforms = [] + /** @type {TransformDefinition[]} */ + this._globalTransforms = [] + + if (!opts.readFile) { + throw new Error('glslify-deps: readFile must be defined') + } + + this._readFile = cacheWrap(opts.readFile, this._fileCache) + + if (!opts.resolve) { + throw new Error('glslify-deps: resolve must be defined') + } + + this.resolve = opts.resolve; + + if (!opts.transformRequire) { + throw new Error('glslify-deps: transformRequire must be defined') + } + + this.transformRequire = opts.transformRequire + + // @ts-ignore + this._transformRequireAsync = !!opts.transformRequire.sync + + if (!this._async && this._transformRequireAsync) { + throw new Error('glslify-deps: transformRequire async detected \ + \nwhen sync context, please ensure your resolver is even with the context') + } +} + +Depper.prototype.inline = function(source, filename, done) { + this._inlineSource = source + return this.add(filename || this._inlineName, done) +} + +/** + * Adds a shader file to the graph, including its dependencies + * which are resolved in this step. Transforms are also applied + * in the process too, as they may potentially add or remove dependent + * modules. + * + * @param {String} filename The absolute path of this file. + * @param {Object} [opts] The options will be pased to _resolveImports function. + * @param {(err: Error, deps?: object[]) => any} [done] + * + * If async is defined then `done` callback will be called when the entire graph has been + * resolved, and will include an array of dependencies discovered + * so far as its second argument. + * + * If sync returns an array of dependencies discovered so far as its second argument. + */ +Depper.prototype.add = function(filename, opts, done) { + if (typeof opts === 'function') { + done = opts + opts = {} + } + + var self = this + var exports = [] + var imports = [] + var dep = this._addDep(filename) + var resolveOpts = Object.assign({ + deps: dep.deps, + }, opts) + + var process = asyncify( + function(_, next) {return self.readFile(filename, next) }, + function(_, next) {return self.getTransformsForFile(filename, next) }, + function(result, next) { + // @ts-ignore + self.emit('file', filename) + return self.applyTransforms(filename, result[0], result[1], next) + }, + function(result, next) { + extractPreprocessors(dep.source = result[2], imports, exports) + return self._resolveImports(imports, resolveOpts, next) + }, function(_, next) { + if(next) { + next(null, self._deps) + } + }) + + + if (this._async) { + process(done || function() { + console.warn('glslify-deps: depper.add() has not a callback defined using async flow') + }) + return dep + } else { + process() + return this._deps + } +} + +/** + * Dummy internal function for resolve transforms for a file + * @param {String} filename The absolute path of the file in question. + * @param {(err: Error, transforms?: GlslTransform[]) => any} [done] Applies when async true + * @returns {GlslTransform[]} List of transform for a file + */ +Depper.prototype.getTransformsForFile = function(filename, done) { + if(done) { + done(null, []) + } + console.warn('glslify-deps: depper.getTransformsForFile() not yet implemented') + return [] +} + +/** + * Adds a transform to use on your local dependencies. + * Note that this should be used before calling `add`. + * + * Transforms are handled using a different API to browserify, e.g.: + * + * ``` js + * module.exports = function transform(filename, src, opts, done) { + * done(null, src.toUpperCase()) + * } + * ``` + * + * Where `filename` is the absolute file path, `src` is the shader source + * as a string, `opts` is an options object for configuration, and `done` + * is a callback which takes the transformed shader source. + * + * @param {String|GlslTransform} transform + * @param {Object} [opts] + * @param {Boolean} [opts.global] adds transform to global scope + * @param {Boolean} [opts.post] + */ +Depper.prototype.transform = function(transform, opts) { + var name = typeof transform === 'string' ? transform : null + var list = opts && opts.global + ? this._globalTransforms + : this._transforms + + // post transforms are ignored by glslify-deps, to be handled + // by glslify after the file has been bundled. + if (opts && opts.post) return this + + + list.push({ tr: transform, opts: opts, name: name }) + + return this +} + +/** + * Resolves a transform. + * Works for both contexts async and sync + * Functions are retained as-is. + * Strings are resolved using the transformRequire option + * + * + * @param {String|GlslTransform} transform + * @param {Object} [opts] The options will be pased to transformRequire function. + * @param {(err: Error, transform?: GlslTransform) => any} [done] Applies if is defined + * @return {Function} + */ +Depper.prototype.resolveTransform = function(transform, opts, done) { + if (typeof opts === 'function') { + done = opts + opts = {} + } + + var self = this + + if (typeof transform === 'function') { + if (done) done(null, transform) + return transform + } + + function selectTransform(tr) { + if (self._async) return tr; + if (!tr || typeof tr.sync !== 'function') { + var err = new Error('transform ' + transform + ' does not provide a' + + ' synchronous interface') + if (done) { + done(err) + return null + } else { + throw err + } + } + return tr.sync + } + + if (this._transformRequireAsync) { + this.transformRequire(transform, opts, function(err, resolved) { + if (err) return done(err) + return done(null, selectTransform(resolved)) + }); + } else { + var tr = selectTransform(this.transformRequire(transform, opts)) + if (tr && done) done(null, tr) + return tr + } +} + +/** + * Applies a transform to a string. + * + * Note that transforms here are passed in differently to other methods: + * - `tr.tr` should point to the transform function. + * - `tr.opts` should contain the options for the transform, if applicable. + * + * @param {String} filename The absolute path of the file you're transforming. + * @param {String} src The shader source you'd like to transform. + * @param {TransformResolved[]} transforms The transforms you'd like to apply. + * @param {(err: Error, result?: string) => any} [done] Applies when async true + */ +Depper.prototype.applyTransforms = function(filename, src, transforms, done) { + if (this._async) { + var i = 0 + + next(null, src) + function next(err, updated) { + if (err) return done(err) + if (i >= transforms.length) return done(null, updated) + + var tr = transforms[i++] + var opts = tr.opts + + if (!opts || typeof opts !== 'object') opts = {} + tr.tr(filename, updated+'', tr.opts, next) + } + } else { + transforms.forEach(function (tr) { + var opts = tr.opts + if (!opts || typeof opts !== 'object') opts = {} + src = tr.tr(filename, src+'', tr.opts) + }) + return src + } +} + +/** + * Internal method to add dependencies + * @param {object} [extra] + */ +Depper.prototype._addDep = function(file, extra) { + var dep = Object.assign({ + id: this._i++ + , file: file + , deps: {} + , source: null + , entry: this._i === 1 + }, extra) + + this._deps.push(dep) + + return dep; +} + +/** + * Internal method to register transforms + * @param {TransformDefinition[]} transforms + * @param {(err: Error, resolved?: TransformResolved[]) => any} cb + * @returns {TransformResolved[]} + */ +Depper.prototype._register = function(transforms, cb) { + var self = this; + /** @type {TransformResolved[]} */ + // @ts-ignore + var result = transforms + .concat(this._globalTransforms) + // map acts as synchronous if the iterator is always in + // the main thread so is compatible with resolveTransform + map(result, 1, function(tr, next) { + self.resolveTransform(tr.tr, next) + }, (err, resolved) => { + if (err) { + if(cb) return cb(err) + throw err + } + result.forEach((tr, idx) => { + tr.tr = resolved[idx] + }) + if(cb) cb(null, result) + }) + + return result +} + +/** + * Internal async method to retrieve dependencies + * resolving imports using the internal cache + * + * @param {string[]} imports + * @param {object} [opts] The options will be pased to resolve function. + * @param {object} [opts.deps] Existing dependencies + * @param {number} [opts.parallel=10] Parallel threads when async + * @param {(err: Error) => any} [done] + * @return {object} Resolved dependencies + */ +Depper.prototype._resolveImports = function(imports, opts, done) { + if (typeof opts === 'function') { + done = opts + opts = {} + } + var self = this + var deps = opts && opts.deps || {} + var parallel = opts && opts.parallel || 10 + + var process = asyncify( + function(result, next) { return self.resolve(result[0], opts, next) }, + function(result, next) { + var importName = result[0] + var resolved = result[1] + if (self._cache[resolved]) { + deps[importName] = self._cache[resolved].id + return next && next() + } + if (next) { + self._cache[resolved] = self.add(resolved, function(err) { + if (err) return next(err) + deps[importName] = self._cache[resolved].id + next() + }) + } else { + var idx = self._i + self._cache[resolved] = self.add(resolved)[idx] + deps[importName] = self._cache[resolved].id + } + } + ) + + if (this._async) { + map(imports, parallel, function(imp, next) { + process([getImportName(imp)], next) + }, done) + } else { + imports.forEach(function (imp) { + process([getImportName(imp)]) + }) + } + + return deps +} + +Depper.prototype.readFile = function(filename, done) { + if (filename !== this._inlineName) + return this._readFile(filename, done) + + if(this._async) { + return done(null, this._inlineSource) + } + return this._inlineSource +} + +inherits(Depper, Emitter) +module.exports = Depper diff --git a/index.js b/index.js index a0ecbb4..986d21d 100644 --- a/index.js +++ b/index.js @@ -1,362 +1,9 @@ -var tokenize = require('glsl-tokenizer/string') -var findup = require('@choojs/findup') -var fs = require('graceful-fs') -var map = require('map-limit') -var inherits = require('inherits') -var Emitter = require('events/') -var path = require('path') +var NodeDepper = require('./node') -var glslResolve = require('glsl-resolve') -var nodeResolve = require('resolve') - -var inlineName = '__INLINE__' + Math.random() -var inlineSource = '' - -module.exports = Depper - -/** - * Creates a new instance of glslify-deps. Generally, you'll - * want to use one instance per bundle. - * - * @class - * @param {String} cwd The root directory of your shader. Defaults to process.cwd() - */ -inherits(Depper, Emitter) -function Depper(opts) { - if (!(this instanceof Depper)) return new Depper(opts) - Emitter.call(this) - - opts = typeof opts === 'string' ? { cwd: opts } : opts - opts = opts || {} - - this._deps = [] - this._cwd = opts.cwd || process.cwd() - this._cache = {} - this._i = 0 - this._transforms = [] - this._trCache = {} - this._fileCache = opts.files || {} - - this._globalTransforms = [] - - this._readFile = cacheWrap(opts.readFile || defaultRead, this._fileCache) - this.resolve = opts.resolve || glslResolve - - if (typeof this._cwd !== 'string') { - throw new Error('glslify-deps: cwd must be a string path') - } -} - -Depper.prototype.inline = function(source, basedir, done) { - var inlineFile = path.resolve(basedir || process.cwd(), inlineName) - - inlineSource = source - - this.add(inlineFile, function(err, tree) { - done && done(err, !err && tree) - }) -} - -/** - * Adds a transform to use on your local dependencies. - * Note that this should be used before calling `add`. - * - * Transforms are handled using a different API to browserify, e.g.: - * - * ``` js - * module.exports = function transform(filename, src, opts, done) { - * done(null, src.toUpperCase()) - * } - * ``` - * - * Where `filename` is the absolute file path, `src` is the shader source - * as a string, `opts` is an options object for configuration, and `done` - * is a callback which takes the transformed shader source. - * - * @param {String|Function} transform - * @param {Object} opts - */ -Depper.prototype.transform = function(transform, opts) { - var name = typeof transform === 'string' ? transform : null - var list = opts && opts.global - ? this._globalTransforms - : this._transforms - - // post transforms are ignored by glslify-deps, to be handled - // by glslify after the file has been bundled. - if (opts && opts.post) return this - - transform = this.resolveTransform(transform) - list.push({ tr: transform, opts: opts, name: name }) - - return this -} - -/** - * Adds a shader file to the graph, including its dependencies - * which are resolved in this step. Transforms are also applied - * in the process too, as they may potentially add or remove dependent - * modules. - * - * @param {String} filename The absolute path of this file. - * @param {String} src The shader source for this file. - * @param {Function} done(err, deps) - * - * The `done` callback will be called when the entire graph has been - * resolved, and will include an array of dependencies discovered - * so far as its second argument. - */ -Depper.prototype.add = function(filename, done) { - var basedir = path.dirname(filename = path.resolve(filename)) - var cache = this._cache - var self = this - var exports = [] - var imports = [] - - var dep = { - id: this._i++ - , deps: {} - , file: filename - , source: null - , entry: this._i === 1 - } - - this._deps.push(dep) - this.readFile(filename, function(err, src) { - if (err) return done(err) - - self.getTransformsForFile(filename, function(err, trs) { - if (err) return done(err) - - self.emit('file', filename) - self.applyTransforms(filename, src, trs, function(err, src) { - if (err) return done(err) - - dep.source = src - extractPreprocessors() - resolveImports(function(err) { - setTimeout(function() { - done && done(err, !err && self._deps) - }) - }) - }) - }) - }) - - return dep - - function extractPreprocessors() { - var tokens = tokenize(dep.source) - - for (var i = 0; i < tokens.length; i++) { - var token = tokens[i] - if (token.type !== 'preprocessor') continue - - var data = token.data - if (!glslifyPreprocessor(data)) continue - - var exp = glslifyExport(data) - var imp = glslifyImport(data) - if (exp) exports.push(exp[1]) - if (imp) imports.push(imp[2]) - } - } - - function resolveImports(resolved) { - map(imports, 10, function(imp, next) { - var importName = imp.split(/\s*,\s*/).shift() - - importName = importName.trim() - importName = importName.replace(/^'|'$/g, '') - importName = importName.replace(/^"|"$/g, '') - - self.resolve(importName, { basedir: basedir }, function(err, resolved) { - if (err) return next(err) - - if (cache[resolved]) { - dep.deps[importName] = cache[resolved].id - return next() - } - - cache[resolved] = self.add(resolved, function(err) { - if (err) return next(err) - dep.deps[importName] = cache[resolved].id - next() - }) - }) - }, resolved) - } -} - -Depper.prototype.readFile = function(filename, done) { - if (path.basename(filename) !== inlineName) - return this._readFile(filename, done) - - return done(null, inlineSource) +module.exports = function(opts) { + opts = (typeof opts === 'string' ? { cwd: opts } : opts) || {} + opts.async = true + return NodeDepper(opts) } -/** - * Determines which transforms to use for a particular file. - * The rules here are the same you see in browserify: - * - * - your shader files will have your specified transforms applied to them - * - shader files in node_modules do not get local transforms - * - all files will apply transforms specified in `glslify.transform` in your - * `package.json` file, albeit after any transforms you specified using - * `depper.transform`. - * - * @param {String} filename The absolute path of the file in question. - */ -Depper.prototype.getTransformsForFile = function(filename, done) { - var self = this - var entry = this._deps[0] - - if (!entry) return done(new Error( - 'getTransformsForFile may only be called after adding your entry file' - )) - - var entryDir = path.dirname(path.resolve(entry.file)) - var fileDir = path.dirname(path.resolve(filename)) - var relative = path.relative(entryDir, fileDir).split(path.sep) - var node_modules = relative.indexOf('node_modules') !== -1 - var trLocal = node_modules ? [] : this._transforms - var trCache = this._trCache - - if (trCache[fileDir]) { - return done(null, trCache[fileDir]) - } - - findup(fileDir, 'package.json', function(err, found) { - var notFound = err && err.message === 'not found' - if (notFound) return register([]) - if (err) return done(err) - - var pkg = path.join(found, 'package.json') - - self.readFile(pkg, function(err, pkgjson) { - if (err) return done(err) - - try { - pkgjson = JSON.parse(pkgjson) - } catch(e) { return done(e) } - - var transforms = ( - pkgjson['glslify'] - && pkgjson['glslify']['transform'] - || [] - ) - - transforms = transforms.map(function(key) { - var transform = Array.isArray(key) - ? key - : [key, {}] - - var key = transform[0] - var opt = transform[1] - - if (opt) { - delete opt.global - delete opt.post - } - - return { tr: key, opts: opt, name: key } - }).map(function(tr) { - tr.tr = self.resolveTransform(tr.tr) - return tr - }) - - register(transforms) - }) - }) - - function register(transforms) { - done(null, trCache[fileDir] = trLocal - .concat(transforms) - .concat(self._globalTransforms)) - } -} - -/** - * Resolves a transform. - * - * Functions are retained as-is. - * Strings are resolved using node's `require` resolution algorithm, - * and then required directly. - * - * @param {String|Function} transform - */ -Depper.prototype.resolveTransform = function(transform) { - if (typeof transform === 'string') { - transform = nodeResolve.sync(transform, { - basedir: this._cwd - }) - - transform = require(transform) - } - - return transform -} - -/** - * Applies a transform to a string. - * - * Note that transforms here are passed in differently to other methods: - * - `tr.tr` should point to the transform function. - * - `tr.opts` should contain the options for the transform, if applicable. - * - * @param {String} filename The absolute path of the file you're transforming. - * @param {String} src The shader source you'd like to transform. - * @param {Array} transforms The transforms you'd like to apply. - * @param {Function} done(err, transformed) - */ -Depper.prototype.applyTransforms = function(filename, src, transforms, done) { - var i = 0 - - next(null, src) - function next(err, updated) { - if (err) return done(err) - if (i >= transforms.length) return done(null, updated) - - var tr = transforms[i++] - var opts = tr.opts - - if (!opts || typeof opts !== 'object') opts = {} - tr.tr(filename, updated+'', tr.opts, next) - } -} - -function glslifyPreprocessor(data) { - return /#pragma glslify:/.test(data) -} - -function glslifyExport(data) { - return /#pragma glslify:\s*export\(([^\)]+)\)/.exec(data) -} - -function glslifyImport(data) { - return /#pragma glslify:\s*([^=\s]+)\s*=\s*require\(([^\)]+)\)/.exec(data) -} - -function defaultRead(src, done) { - fs.readFile(src, 'utf8', done) -} - -function cacheWrap(read, cache) { - // resolve all cached files such that they match - // all of the paths glslify handles, which are otherwise - // absolute - cache = Object.keys(cache).reduce(function(newCache, file) { - newCache[path.resolve(file)] = cache[file] - return newCache - }, {}) - - return function readFromCache(filename, done) { - if (!cache[filename]) { - return read(filename, done) - } - - process.nextTick(function() { - done(null, cache[filename]) - }) - } -} +module.exports.sync = NodeDepper diff --git a/node.js b/node.js new file mode 100644 index 0000000..74d116a --- /dev/null +++ b/node.js @@ -0,0 +1,176 @@ +/** @typedef {import('./depper').DepperOptions} DepperOptions */ +var Depper = require('./depper') +var path = require('path') +var inherits = require('inherits') +var fs = require('graceful-fs') +var findup = require('@choojs/findup') +var glslResolve = require('glsl-resolve') +var transformRequire = require('./transform-require') + +var { + getTransformsFromPkg, + mix, +} = require('./utils'); + +function createDefaultRead() { + function defaultReadAsync(src, done) { + fs.readFile(src, 'utf8', done) + } + + function defaultRead(src) { + return fs.readFileSync(src, 'utf8') + } + + return mix(defaultRead, defaultReadAsync) +} + +/** + * + * @constructor + * @param {string} cwd + *//** + * @constructor + * @param {DepperOptions} opts + * @param {String} [opts.cwd] The root directory of your shader. Defaults to process.cwd(). + */ +function NodeDepper(opts) { + if (!(this instanceof NodeDepper)) return new NodeDepper(opts) + opts = (typeof opts === 'string' ? { cwd: opts } : opts) || {} + opts.resolve = opts.resolve || mix(glslResolve.sync, glslResolve) + // keeps the original behaviour of transform resolution but overridable + opts.transformRequire = opts.transformRequire || transformRequire.sync + opts.readFile = opts.readFile || createDefaultRead() + Depper.call(this, opts) + + this._cwd = opts.cwd || process.cwd() + this._trCache = {} + + if (typeof this._cwd !== 'string') { + throw new Error('glslify-deps: cwd must be a string path') + } +} + +/** + * @override + * @param {*} source + * @param {*} basedir + * @param {*} done + */ +NodeDepper.prototype.inline = function(source, basedir, done) { + var inlineFile = path.resolve(basedir || this._cwd, this._inlineName) + return Depper.prototype.inline.call(this, source, inlineFile, done); +} + +/** + * @override + * @param {String} filename The absolute path of this file. + * @param {(err: Error, deps?: object[]) => any} [done] + */ +NodeDepper.prototype.add = function(filename, done) { + var resolved = path.resolve(filename); + return Depper.prototype.add.call(this, resolved, { + basedir: path.dirname(resolved) + }, done) +} + +/** + * @override + * @param {String|GlslTransform} transform + * @param {(err: Error, transform?: GlslTransform) => any} [done] Applies if is defined + * @return {Function} + */ +NodeDepper.prototype.resolveTransform = function(transform, done) { + return Depper.prototype.resolveTransform.call(this, transform, { + cwd: this._cwd + }, done) +} + +/** + * @override + * @param {*} filename + * @param {*} done + */ +NodeDepper.prototype.readFile = function(filename, done) { + if (path.basename(filename) !== this._inlineName) + return this._readFile(filename, done) + + if(this._async) { + return done(null, this._inlineSource) + } + return this._inlineSource +} + + +/** + * Determines which transforms to use for a particular file. + * The rules here are the same you see in browserify: + * + * - your shader files will have your specified transforms applied to them + * - shader files in node_modules do not get local transforms + * - all files will apply transforms specified in `glslify.transform` in your + * `package.json` file, albeit after any transforms you specified using + * `depper.transform`. + * + * @param {String} filename The absolute path of the file in question. + * @param {(err: Error, transforms?: GlslTransform[]) => any} [done] Applies when async true + * @returns {GlslTransform[]} List of transform for a file + */ +NodeDepper.prototype.getTransformsForFile = function(filename, done) { + var self = this + var entry = this._deps[0] + + if (!entry) return done(new Error( + 'getTransformsForFile may only be called after adding your entry file' + )) + + var entryDir = path.dirname(path.resolve(entry.file)) + var fileDir = path.dirname(path.resolve(filename)) + var relative = path.relative(entryDir, fileDir).split(path.sep) + var node_modules = relative.indexOf('node_modules') !== -1 + var trLocal = node_modules ? [] : this._transforms + var trCache = this._trCache + var pkgName = 'package.json' + + if (trCache[fileDir]) { + if (this._async) { + return done(null, trCache[fileDir]) + } else { + return trCache[fileDir] + } + } + + if (this._async) { + findup(fileDir, pkgName, function(err, found) { + var notFound = err && err.message === 'not found' + if (notFound) return register([], done) + if (err) return done(err) + + var pkg = path.join(found, pkgName) + + self.readFile(pkg, function(err, pkgJson) { + if (err) return done(err) + var transforms; + try { + transforms = getTransformsFromPkg(pkgJson) + } catch(e) { return done(e) } + + trCache[fileDir] = self._register(trLocal.concat(transforms), done) + }) + }) + } else { + try { var found = findup.sync(fileDir, pkgName) } + catch (err) { + var notFound = err.message === 'not found' + if (notFound) return register([]) + else throw err + } + + var pkg = path.join(found, pkgName) + var transforms = getTransformsFromPkg(self.readFile(pkg)) + + return trCache[fileDir] = self._register(trLocal.concat(transforms)) + } +} + +inherits(NodeDepper, Depper) +module.exports = NodeDepper diff --git a/sync.js b/sync.js index 54fe512..45699d5 100644 --- a/sync.js +++ b/sync.js @@ -1,327 +1,3 @@ -var tokenize = require('glsl-tokenizer/string') -var findup = require('@choojs/findup').sync -var fs = require('graceful-fs') -var map = require('map-limit') -var inherits = require('inherits') -var Emitter = require('events/') -var path = require('path') +var NodeDepper = require('./node') -var glslResolve = require('glsl-resolve').sync -var nodeResolve = require('resolve').sync - -var inlineName = '__INLINE__' + Math.random() -var inlineSource = '' - -module.exports = Depper - -/** - * Creates a new instance of glslify-deps. Generally, you'll - * want to use one instance per bundle. - * - * @class - * @param {String} cwd The root directory of your shader. Defaults to process.cwd() - */ -inherits(Depper, Emitter) -function Depper(opts) { - if (!(this instanceof Depper)) return new Depper(opts) - Emitter.call(this) - - opts = typeof opts === 'string' ? { cwd: opts } : opts - opts = opts || {} - - this._deps = [] - this._cwd = opts.cwd || process.cwd() - this._cache = {} - this._i = 0 - this._transforms = [] - this._trCache = {} - this._fileCache = opts.files || {} - - this._globalTransforms = [] - - this._readFile = cacheWrap(opts.readFileSync || defaultRead, this._fileCache) - this.resolve = opts.resolve || glslResolve - - if (typeof this._cwd !== 'string') { - throw new Error('glslify-deps: cwd must be a string path') - } -} - -Depper.prototype.inline = function(source, basedir) { - var inlineFile = path.resolve(basedir || process.cwd(), inlineName) - - inlineSource = source - - return this.add(inlineFile) -} - -/** - * Adds a transform to use on your local dependencies. - * Note that this should be used before calling `add`. - * - * Transforms are handled using a different API to browserify, e.g.: - * - * ``` js - * exports.sync = function transform(filename, src, opts) { - * return src.toUpperCase() - * } - * ``` - * - * This is also different from the async transform API. - * - * Where `filename` is the absolute file path, `src` is the shader source - * as a string, `opts` is an options object for configuration. - * - * @param {String|Function} transform - * @param {Object} opts - */ -Depper.prototype.transform = function(transform, opts) { - var name = typeof transform === 'string' ? transform : null - var list = opts && opts.global - ? this._globalTransforms - : this._transforms - - // post transforms are ignored by glslify-deps, to be handled - // by glslify after the file has been bundled. - if (opts && opts.post) return this - - transform = this.resolveTransform(transform) - list.push({ tr: transform, opts: opts, name: name }) - - return this -} - -/** - * Adds a shader file to the graph, including its dependencies - * which are resolved in this step. Transforms are also applied - * in the process too, as they may potentially add or remove dependent - * modules. - * - * @param {String} filename The absolute path of this file. - * @param {String} src The shader source for this file. - * - * Returns an array of dependencies discovered so far as its second argument. - */ -Depper.prototype.add = function(filename) { - var basedir = path.dirname(filename = path.resolve(filename)) - var cache = this._cache - var self = this - var exports = [] - var imports = [] - - var dep = { - id: this._i++ - , deps: {} - , file: filename - , source: null - , entry: this._i === 1 - } - - this._deps.push(dep) - var src = this.readFile(filename) - var trs = self.getTransformsForFile(filename) - self.emit('file', filename) - src = self.applyTransforms(filename, src, trs) - dep.source = src - extractPreprocessors() - - resolveImports() - return self._deps - - function extractPreprocessors() { - var tokens = tokenize(dep.source) - - for (var i = 0; i < tokens.length; i++) { - var token = tokens[i] - if (token.type !== 'preprocessor') continue - - var data = token.data - if (!glslifyPreprocessor(data)) continue - - var exp = glslifyExport(data) - var imp = glslifyImport(data) - if (exp) exports.push(exp[1]) - if (imp) imports.push(imp[2]) - } - } - - function resolveImports(resolved) { - imports.forEach(function (imp) { - var importName = imp.split(/\s*,\s*/).shift() - - importName = importName.trim() - importName = importName.replace(/^'|'$/g, '') - importName = importName.replace(/^"|"$/g, '') - - var resolved = self.resolve(importName, { basedir: basedir }) - if (cache[resolved]) { - dep.deps[importName] = cache[resolved].id - } - var i = self._i - cache[resolved] = self.add(resolved)[i] - dep.deps[importName] = cache[resolved].id - }) - } -} - -Depper.prototype.readFile = function(filename) { - if (path.basename(filename) !== inlineName) - return this._readFile(filename) - - return inlineSource -} - -/** - * Determines which transforms to use for a particular file. - * The rules here are the same you see in browserify: - * - * - your shader files will have your specified transforms applied to them - * - shader files in node_modules do not get local transforms - * - all files will apply transforms specified in `glslify.transform` in your - * `package.json` file, albeit after any transforms you specified using - * `depper.transform`. - * - * @param {String} filename The absolute path of the file in question. - */ -Depper.prototype.getTransformsForFile = function(filename) { - var self = this - var entry = this._deps[0] - - if (!entry) throw new Error( - 'getTransformsForFile may only be called after adding your entry file' - ) - - var entryDir = path.dirname(path.resolve(entry.file)) - var fileDir = path.dirname(path.resolve(filename)) - var relative = path.relative(entryDir, fileDir).split(path.sep) - var node_modules = relative.indexOf('node_modules') !== -1 - var trLocal = node_modules ? [] : this._transforms - var trCache = this._trCache - - if (trCache[fileDir]) { - return trCache[fileDir] - } - - try { var found = findup(fileDir, 'package.json') } - catch (err) { - var notFound = err.message === 'not found' - if (notFound) return register([]) - else throw err - } - - var pkg = path.join(found, 'package.json') - var pkgjson = JSON.parse(self.readFile(pkg)) - - var transforms = ( - pkgjson['glslify'] - && pkgjson['glslify']['transform'] - || [] - ) - - transforms = transforms.map(function(key) { - var transform = Array.isArray(key) - ? key - : [key, {}] - - var key = transform[0] - var opt = transform[1] - - if (opt) { - delete opt.global - delete opt.post - } - - return { tr: key, opts: opt, name: key } - }).map(function(tr) { - tr.tr = self.resolveTransform(tr.tr) - return tr - }) - - return register(transforms) - - function register(transforms) { - return trCache[fileDir] = trLocal - .concat(transforms) - .concat(self._globalTransforms) - } -} - -/** - * Resolves a transform. - * - * Functions are retained as-is. - * Strings are resolved using node's `require` resolution algorithm, - * and then required directly. - * - * @param {String|Function} transform - */ -Depper.prototype.resolveTransform = function(transform) { - if (typeof transform === 'string') { - transform = nodeResolve(transform, { - basedir: this._cwd - }) - - var m = require(transform) - if (!m || typeof m.sync !== 'function') { - throw new Error('transform ' + transform + ' does not provide a' - + ' synchronous interface') - } - transform = m.sync - } - return transform -} - -/** - * Applies a transform to a string. - * - * Note that transforms here are passed in differently to other methods: - * - `tr.tr` should point to the transform function. - * - `tr.opts` should contain the options for the transform, if applicable. - * - * @param {String} filename The absolute path of the file you're transforming. - * @param {String} src The shader source you'd like to transform. - * @param {Array} transforms The transforms you'd like to apply. - * - * Returns the transformed source string. - */ -Depper.prototype.applyTransforms = function(filename, src, transforms) { - transforms.forEach(function (tr) { - var opts = tr.opts - if (!opts || typeof opts !== 'object') opts = {} - src = tr.tr(filename, src+'', tr.opts) - }) - return src -} - -function glslifyPreprocessor(data) { - return /#pragma glslify:/.test(data) -} - -function glslifyExport(data) { - return /#pragma glslify:\s*export\(([^\)]+)\)/.exec(data) -} - -function glslifyImport(data) { - return /#pragma glslify:\s*([^=\s]+)\s*=\s*require\(([^\)]+)\)/.exec(data) -} - -function defaultRead(src) { - return fs.readFileSync(src, 'utf8') -} - -function cacheWrap(read, cache) { - // resolve all cached files such that they match - // all of the paths glslify handles, which are otherwise - // absolute - cache = Object.keys(cache).reduce(function(newCache, file) { - newCache[path.resolve(file)] = cache[file] - return newCache - }, {}) - - return function readFromCache(filename) { - if (!cache[filename]) { - cache[filename] = read(filename) - } - return cache[filename] - } -} +module.exports = NodeDepper diff --git a/test/resolve-transform-sync.js b/test/resolve-transform-sync.js index 741f7ad..57d42c0 100644 --- a/test/resolve-transform-sync.js +++ b/test/resolve-transform-sync.js @@ -1,5 +1,6 @@ var test = require('tape') var deps = require('../sync') +var transformRequire = require('../transform-require') test('sync resolveTransform', function(t) { var dep = deps(__dirname) @@ -7,3 +8,14 @@ test('sync resolveTransform', function(t) { t.equal(require('glslify-hex'), dep.resolveTransform('glslify-hex'), 'resolves transforms like node') t.end() }) + +test('sync resolveTransform throws error when resolveTransform is async', function(t) { + + t.throws(function() { + deps({ + cwd: __dirname, + transformRequire: transformRequire + }) + }, Error, 'should throw error') + t.end() +}) diff --git a/test/resolve-transform.js b/test/resolve-transform.js index 99fb7b8..92d0ac9 100644 --- a/test/resolve-transform.js +++ b/test/resolve-transform.js @@ -1,5 +1,6 @@ var test = require('tape') var deps = require('../') +var transformRequire = require('../transform-require') test('resolveTransform', function(t) { var dep = deps(__dirname) @@ -7,3 +8,16 @@ test('resolveTransform', function(t) { t.equal(require('glslify-hex'), dep.resolveTransform('glslify-hex'), 'resolves transforms like node') t.end() }) + +test('resolveTransform with async resolver', function(t) { + var dep = deps({ + cwd: __dirname, + transformRequire: transformRequire + }) + + dep.resolveTransform('glslify-hex', function(err, transform) { + t.true(!err) + t.equal(require('glslify-hex'), transform, 'resolves transforms like node asynchronously') + t.end() + }) +}) diff --git a/test/transform.js b/test/transform.js index 722c698..2116975 100644 --- a/test/transform.js +++ b/test/transform.js @@ -2,14 +2,28 @@ var test = require('tape') var path = require('path') var deps = require('../') var fs = require('fs') +var transformRequire = require('../transform-require') var fixture = path.resolve(__dirname, 'fixtures/transform/index.glsl') var another = path.resolve(__dirname, 'fixtures/transform/another.glsl') var fake = path.resolve(__dirname, 'fixtures/node_modules/glsl-fake/index.glsl') -test('.transform(string)', function(t) { +var suite = [[ + 'transformResolve sync' +], [ + 'transformResolve sync', { + transformRequire: transformRequire + }] +] + +suite.forEach(function(s) { + + var context = s[0] + var opts = s[1] + +test(context + ' .transform(string)', function(t) { var src = fs.readFileSync(fixture, 'utf8') - var depper = deps() + var depper = deps(opts) depper.transform('glslify-hex') depper.add(fixture, function(err, deps) { @@ -19,9 +33,9 @@ test('.transform(string)', function(t) { }) }) -test('.transform(fn)', function(t) { +test(context + ' .transform(fn)', function(t) { var src = fs.readFileSync(fake, 'utf8') - var depper = deps() + var depper = deps(opts) depper.transform(function(file, src, opts, done) { return done(null, src.toUpperCase()) @@ -34,9 +48,9 @@ test('.transform(fn)', function(t) { }) }) -test('.transform(fn, opts)', function(t) { +test(context + ' .transform(fn, opts)', function(t) { var src = fs.readFileSync(fake, 'utf8') - var depper = deps() + var depper = deps(opts) var opts = { hello: 'world' } @@ -51,3 +65,5 @@ test('.transform(fn, opts)', function(t) { t.end() }) }) + +}) diff --git a/transform-require.js b/transform-require.js new file mode 100644 index 0000000..5b6f8eb --- /dev/null +++ b/transform-require.js @@ -0,0 +1,51 @@ +// @ts-check +/** @typedef {import('./depper').GlslTransform} GlslTransform */ +var nodeResolve = require('resolve') + +/** + * Async transform resolution in node. + * + * Functions are retained as-is. + * Strings are resolved using node's `require` resolution algorithm, + * and then required directly. + * + * @param {String|GlslTransform} transform + * @param {object} opts + * @param {string} opts.cwd current work directory + * @param {(err: Error, transform?: GlslTransform) => any} cb + */ +var transformRequire = function (transform, opts, cb) { + var cwd = opts && opts.cwd + if (typeof transform === 'string') { + return nodeResolve(transform, { + basedir: cwd + }, (err) => { + if (err) return cb(err) + cb(null, require(transform)) + }) + } + process.nextTick(() => { + cb(null, transform) + }); +} + +/** + * Sync transform resolution in node. + * + * Functions are retained as-is. + * Strings are resolved using node's `require` resolution algorithm, + * and then required directly. + * + * @param {String|GlslTransform} transform + * @param {object} opts + * @param {string} opts.cwd current work directory + * @returns {GlslTransform} + */ +transformRequire.sync = function (transform, opts) { + var cwd = opts && opts.cwd + return typeof transform === 'string' ? require(nodeResolve.sync(transform, { + basedir: cwd + })) : transform +} + +module.exports = transformRequire; diff --git a/utils.js b/utils.js new file mode 100644 index 0000000..952431b --- /dev/null +++ b/utils.js @@ -0,0 +1,259 @@ +// @ts-check + +var tokenize = require('glsl-tokenizer/string') +var path = require('path') + +function glslifyPreprocessor(data) { + return /#pragma glslify:/.test(data) +} + +function glslifyExport(data) { + return /#pragma glslify:\s*export\(([^\)]+)\)/.exec(data) +} + +function glslifyImport(data) { + return /#pragma glslify:\s*([^=\s]+)\s*=\s*require\(([^\)]+)\)/.exec(data) +} + +function genInlineName() { + return '__INLINE__' + Math.random() +} + + +/** + * Gets glslify transform from given package.json + * + * @param {object|string} pkgJson package.json string data or json + * @returns {({tr: string, name: string, opts: object})[]} + */ +function getTransformsFromPkg(pkgJson) { + if (typeof pkgJson === 'string') { + pkgJson = JSON.parse(pkgJson); + } + + var transforms = ( + pkgJson['glslify'] + && pkgJson['glslify']['transform'] + || [] + ) + + return transforms.map(function(key) { + var transform = Array.isArray(key) + ? key + : [key, {}] + + var key = transform[0] + var opt = transform[1] + + if (opt) { + delete opt.global + delete opt.post + } + + return { tr: key, opts: opt, name: key } + }); +} + +/** + * Extracts preprocessors copying the imports and exports + * into respective parameters + * @param {string} source + * @param {string[]} imports + * @param {string[]} exports + */ +function extractPreprocessors(source, imports, exports) { + var tokens = tokenize(source) + + for (var i = 0; i < tokens.length; i++) { + var token = tokens[i] + if (token.type !== 'preprocessor') continue + + var data = token.data + if (!glslifyPreprocessor(data)) continue + + var exp = glslifyExport(data) + var imp = glslifyImport(data) + if (exp) exports.push(exp[1]) + if (imp) imports.push(imp[2]) + } +} + +function getImportName(imp) { + return imp + .split(/\s*,\s*/) + .shift() + .trim() + .replace(/^'|'$/g, '') + .replace(/^"|"$/g, '') +} + +/** Fast apply */ +function apply(fn, args) { + switch(args.length) { + case 1: + return fn(args[0]) + case 2: + return fn(args[0], args[1]) + case 3: + return fn(args[0], args[1], args[2]) + case 4: + return fn(args[0], args[1], args[2], args[3]) + default: + return fn.apply(null, args) + } +} +/** + * Takes an sync and async functions and return a function which detects if the last argument + * is a callback in order to select which flow to use + * @param {function} sync + * @param {function} async + * @returns {(...args, done) => any} + */ +function mix(sync, async) { + function mixed() { + if(typeof arguments[arguments.length - 1] === 'function') { + if (!async) { + throw Error('There\'s no async function available') + } + apply(async, arguments) + } + if (!sync) { + throw Error('There\'s no sync function available') + } + return apply(sync, arguments) + } + + mixed.sync = sync; + mixed.async = async; + + return mixed +} + + +/** + * Allows reuse sync/async logics detecting if done is defined to select which strategy to use. + * Arguments must be functions, if sync is detected then takes the returned value, + * otherwise when async next will be defined and will take the result from there + * + * @param {...(prevState: any[], next?: (err?: Error, result?: any) => null) => any} args + * @returns {((initialState?: any[], done?: (err: Error, state?: any[]) => any) => any[])&((done?: (err: Error, state?: any[]) => any) => any[])} + * @example + * + * const process = asyncify( + * ([foo], next) => next ? next(null, 'bar') : 'bar', + * ([foo, bar], next) => next ? next(null, foo + bar) : foo + bar + * ) + * + * // sync + * const state = process(['foo']) + * console.log(state) // ['foo', 'bar', 'foobar'] + * + * // async + * process(['bar'], (err, result) => console.log(result)) // ['foo', 'bar', 'foobar'] + * + */ +function asyncify() { + var fns = arguments; + return function(initialState, done) { + if (typeof initialState === 'function') { + done = initialState + initialState = [] + } + + var state = initialState || [] + + if (!Array.isArray(state)) { + throw new Error('asyncify: initialState must be an array') + } + + var cursor = state.length + + var i = 0 + if (!fns.length) { + throw new Error('asyncify: no functions detected') + } + + if(typeof state[state.length - 1] === 'function') { + done = state.pop(); + cursor = state.length + } + + function error() { + return new Error('asyncify: arguments must be functions') + } + + if (!done) { + for(; i < fns.length; i++) { + if (typeof fns[i] !== 'function') { + throw error() + } + state[cursor + i] = fns[i](state); + } + } else { + function next(err, result) { + if(err) { + done(err) + } else { + state[cursor + i++] = result + if (i < fns.length) { + if (typeof fns[i] !== 'function') { + done(error()) + } else { + fns[i](state, next) + } + } else { + done(null, state[state.length - 1]) + } + } + } + + fns[i](state, next) + } + + return state; + } +} + +function cacheWrap(read, cache) { + function readFromCache(filename) { + if (!cache[filename]) { + cache[filename] = read(filename) + } + return cache[filename] + } + + function readFromCacheAsync(filename, done) { + if (!cache[filename]) { + return read(filename, (err, content) => { + if (err) done(err); + done(err, cache[filename] = content) + }) + } + return process.nextTick(function() { + done(null, cache[filename]) + }) + } + + return mix(readFromCache, readFromCacheAsync) +} + +function parseFiles(files) { + // resolve all files such that they match + // all of the paths glslify handles, which are otherwise + // absolute + return Object.keys(files).reduce(function(newCache, file) { + newCache[path.resolve(file)] = files[file] + return newCache + }, {}) +} + +module.exports = { + getTransformsFromPkg, + getImportName, + extractPreprocessors, + genInlineName, + cacheWrap, + mix, + parseFiles, + asyncify +}