From 174afedec23d859bb67523bff99cd04ada22d03e Mon Sep 17 00:00:00 2001 From: Antonio Garrote Date: Wed, 10 May 2017 11:26:28 +0100 Subject: [PATCH] Fixing infrastructure for web version of the lib with SHACL support --- js/browserify.sh | 1 + js/externs.js | 1 + js/js-support-bundle.js | 37306 +++++++++++++++- js/yaml.js | 3 +- src/api_modeling_framework/platform.clj | 8 +- src/api_modeling_framework/platform.cljs | 12 +- .../api_modeling_framework/js_support.cljs | 2 +- .../parser/domain/raml_test.cljc | 1 - 8 files changed, 37007 insertions(+), 327 deletions(-) create mode 100644 js/browserify.sh diff --git a/js/browserify.sh b/js/browserify.sh new file mode 100644 index 0000000..947262c --- /dev/null +++ b/js/browserify.sh @@ -0,0 +1 @@ +browserify -s yaml yaml.js > js-support-bundle.js diff --git a/js/externs.js b/js/externs.js index 77e1355..b5beb8c 100644 --- a/js/externs.js +++ b/js/externs.js @@ -11,3 +11,4 @@ JS_REST.location = function () { }; var JS_AST = function () { }; var AMF_LOADING_EVENT = function () { }; +var SHACL = function () { _ }; diff --git a/js/js-support-bundle.js b/js/js-support-bundle.js index 76746c4..df0a61d 100644 --- a/js/js-support-bundle.js +++ b/js/js-support-bundle.js @@ -4075,6 +4075,7 @@ var json_ast = require("./json_ast"); global.FRAGMENTS_CACHE = {}; global.PENDING_LIBRARIES = []; +global.SHACL = require("shacl").validate; var ensureFileUri = function (uri) { if (uri.indexOf("://") === -1) { @@ -4220,7 +4221,7 @@ var collectLibraries = function (fragment, location) { for (var p in libraries) { if (p !== "__location__") { var value = libraries[p]; - if (typeof(value) === "object" && value["amf-lexical-token"]) { + if (typeof (value) === "object" && value["amf-lexical-token"]) { value = value["amf-lexical-token"]; } var resolvedLocation = resolvePath(location, value); @@ -4326,7 +4327,7 @@ global.JS_REST = function (location) { }; }).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) -},{"./js-yaml/index.js":1,"./json_ast":31,"path":40,"rest":34}],33:[function(require,module,exports){ +},{"./js-yaml/index.js":1,"./json_ast":31,"path":108,"rest":34,"shacl":93}],33:[function(require,module,exports){ (function (global, factory) { if (typeof define === "function" && define.amd) { define(['module'], factory); @@ -5659,7 +5660,36483 @@ responsePromise.promise = function (func) { module.exports = responsePromise; },{"./normalizeHeaderName":38}],40:[function(require,module,exports){ +// Functions implementing the validators of SHACL-JS +// Also include validators for the constraint components of the DASH namespace + +// There is no validator for sh:property as this is expected to be +// natively implemented by the surrounding engine. + +var rdfquery = require("./rdfquery"); +var validator = require("./shacl-validator"); + +var XSDIntegerTypes = new rdfquery.NodeSet(); +XSDIntegerTypes.add(T("xsd:integer")); + +var XSDDecimalTypes = new rdfquery.NodeSet(); +XSDDecimalTypes.addAll(XSDIntegerTypes.toArray()); +XSDDecimalTypes.add(T("xsd:decimal")); +XSDDecimalTypes.add(T("xsd:float")); + +var validateAnd = function ($value, $and) { + var shapes = new rdfquery.RDFQueryUtil($shapes).rdfListToArray($and); + for (var i = 0; i < shapes.length; i++) { + if (!validator.SHACL.nodeConformsToShape($value, shapes[i])) { + return false; + } + } + return true; +}; + +var validateClass = function ($value, $class) { + return new rdfquery.RDFQueryUtil($data).isInstanceOf($value, $class); +}; + +var validateClosed = function ($value, $closed, $ignoredProperties, $currentShape) { + if (!T("true").equals($closed)) { + return; + } + var allowed = $shapes.query(). + match($currentShape, "sh:property", "?propertyShape"). + match("?propertyShape", "sh:path", "?path"). + filter(function (solution) { return solution.path.isURI() }). + getNodeSet("?path"); + if ($ignoredProperties) { + allowed.addAll(new rdfquery.RDFQueryUtil($shapes).rdfListToArray($ignoredProperties)); + } + var results = []; + $data.query(). + match($value, "?predicate", "?object"). + filter(function (sol) { return !allowed.contains(sol.predicate) }). + forEach(function (sol) { + results.push({ + path: sol.predicate, + value: sol.object + }); + }); + return results; +}; + +var validateClosedByTypesNode = function ($this, $closedByTypes) { + if (!T("true").equals($closedByTypes)) { + return; + } + var results = []; + var allowedProperties = new rdfquery.NodeSet(); + $data.query(). + match($this, "rdf:type", "?directType"). + path("?directType", { zeroOrMore: T("rdfs:subClassOf") }, "?type"). + forEachNode("?type", function (type) { + $shapes.query(). + match(type, "sh:property", "?pshape"). + match("?pshape", "sh:path", "?path"). + filter(function (sol) { return sol.path.isURI() }). + addAllNodes("?path", allowedProperties); + }); + $data.query(). + match($this, "?predicate", "?object"). + filter(function (sol) { return !T("rdf:type").equals(sol.predicate) }). + filter(function (sol) { return !allowedProperties.contains(sol.predicate) }). + forEach(function (sol) { + results.push({ + path: sol.predicate, + value: sol.object + }); + }) + return results; +}; + +var validateDatatype = function ($value, $datatype) { + if ($value.isLiteral()) { + return $datatype.equals($value.datatype) && isValidForDatatype($value.lex, $datatype); + } + else { + return false; + } +}; + +var validateDisjoint = function ($this, $value, $disjoint) { + return !$data.query().match($this, $disjoint, $value).hasSolution(); +}; + +var validateEqualsProperty = function ($this, $path, $equals) { + var results = []; + var path = rdfquery.toRDFQueryPath($path); + $data.query().path($this, path, "?value").forEach( + function (solution) { + if (!$data.query().match($this, $equals, solution.value).hasSolution()) { + results.push({ + value: solution.value + }); + } + }); + $data.query().match($this, $equals, "?value").forEach( + function (solution) { + if (!$data.query().path($this, path, solution.value).hasSolution()) { + results.push({ + value: solution.value + }); + } + }); + return results; +}; + +var validateHasValueNode = function ($this, $hasValue) { + return $this.equals($hasValue); +}; + +var validateHasValueProperty = function ($this, $path, $hasValue) { + var count = $data.query().path($this, rdfquery.toRDFQueryPath($path), $hasValue).getCount(); + return count > 0; +}; + +var validateIn = function ($value, $in) { + var set = new rdfquery.NodeSet(); + set.addAll(new rdfquery.RDFQueryUtil($shapes).rdfListToArray($in)); + return set.contains($value); +} + +var validateLanguageIn = function ($value, $languageIn) { + if (!$value.isLiteral()) { + return false; + } + var lang = $value.language; + if (!lang || lang === "") { + return false; + } + var ls = new rdfquery.RDFQueryUtil($shapes).rdfListToArray($languageIn); + for (var i = 0; i < ls.length; i++) { + if (lang.startsWith(ls[i].lex)) { + return true; + } + } + return false; +} + +var validateLessThanProperty = function ($this, $path, $lessThan) { + var results = []; + $data.query(). + path($this, rdfquery.toRDFQueryPath($path), "?value"). + match($this, $lessThan, "?otherValue"). + forEach(function (sol) { + var c = validator.SHACL.compareNodes(sol.value, sol.otherValue); + if (c == null || c >= 0) { + results.push({ + value: sol.value + }); + } + }); + return results; +} + +var validateLessThanOrEqualsProperty = function ($this, $path, $lessThanOrEquals) { + var results = []; + $data.query(). + path($this, rdfquery.toRDFQueryPath($path), "?value"). + match($this, $lessThanOrEquals, "?otherValue"). + forEach(function (sol) { + var c = validator.SHACL.compareNodes(sol.value, sol.otherValue); + if (c == null || c > 0) { + results.push({ + value: sol.value + }); + } + }); + return results; +} + +var validateMaxCountProperty = function ($this, $path, $maxCount) { + var count = $data.query().path($this, rdfquery.toRDFQueryPath($path), "?any").getCount(); + return count <= Number($maxCount.value); +} + +var validateMaxExclusive = function ($value, $maxExclusive) { + return $value.isLiteral() && Number($value.lex) < Number($maxExclusive.lex); +} + +var validateMaxInclusive = function ($value, $maxInclusive) { + return $value.isLiteral() && Number($value.lex) <= Number($maxInclusive.lex); +} + +var validateMaxLength = function ($value, $maxLength) { + if ($value.isBlankNode()) { + return false; + } + return $value.value.length <= Number($maxLength.lex); +} + +var validateMinCountProperty = function ($this, $path, $minCount) { + var count = $data.query().path($this, rdfquery.toRDFQueryPath($path), "?any").getCount(); + return count >= Number($minCount.value); +} + +var validateMinExclusive = function ($value, $minExclusive) { + return $value.isLiteral() && Number($value.lex) > Number($minExclusive.lex); +} + +var validateMinInclusive = function ($value, $minInclusive) { + return $value.isLiteral() && Number($value.lex) >= Number($minInclusive.lex); +} + +var validateMinLength = function ($value, $minLength) { + if ($value.isBlankNode()) { + return false; + } + return $value.value.length >= Number($minLength.lex); +} + +var validateNodeKind = function ($value, $nodeKind) { + if ($value.isBlankNode()) { + return T("sh:BlankNode").equals($nodeKind) || + T("sh:BlankNodeOrIRI").equals($nodeKind) || + T("sh:BlankNodeOrLiteral").equals($nodeKind); + } + else if ($value.isURI()) { + return T("sh:IRI").equals($nodeKind) || + T("sh:BlankNodeOrIRI").equals($nodeKind) || + T("sh:IRIOrLiteral").equals($nodeKind); + } + else if ($value.isLiteral()) { + return T("sh:Literal").equals($nodeKind) || + T("sh:BlankNodeOrLiteral").equals($nodeKind) || + T("sh:IRIOrLiteral").equals($nodeKind); + } +} + +var validateNode = function ($value, $node) { + console.log("validateNode..."); + return validator.SHACL.nodeConformsToShape($value, $node); +} + +var validateNonRecursiveProperty = function ($this, $path, $nonRecursive) { + if (T("true").equals($nonRecursive)) { + if ($data.query().path($this, rdfquery.toRDFQueryPath($path), $this).hasSolution()) { + return { + path: $path, + value: $this + } + } + } +} + +var validateNot = function ($value, $not) { + return !validator.SHACL.nodeConformsToShape($value, $not); +} + +var validateOr = function ($value, $or) { + var shapes = new rdfquery.RDFQueryUtil($shapes).rdfListToArray($or); + for (var i = 0; i < shapes.length; i++) { + if (validator.SHACL.nodeConformsToShape($value, shapes[i])) { + return true; + } + } + return false; +} + +var validatePattern = function ($value, $pattern, $flags) { + if ($value.isBlankNode()) { + return false; + } + var re = $flags ? new RegExp($pattern.lex, $flags.lex) : new RegExp($pattern.lex); + return re.test($value.value); +} + +var validatePrimaryKeyProperty = function ($this, $path, $uriStart) { + if (!$this.isURI()) { + return "Must be an IRI"; + } + if ($data.query().path($this, rdfquery.toRDFQueryPath($path), null).getCount() != 1) { + return "Must have exactly one value"; + } + var value = $data.query().path($this, rdfquery.toRDFQueryPath($path), "?value").getNode("?value"); + var uri = $uriStart.lex + encodeURIComponent(value.value); + if (!$this.uri.equals(uri)) { + return "Does not have URI " + uri; + } +} + +var validateQualifiedMaxCountProperty = function ($this, $path, $qualifiedValueShape, $qualifiedValueShapesDisjoint, $qualifiedMaxCount, $currentShape) { + var c = validateQualifiedHelper($this, $path, $qualifiedValueShape, $qualifiedValueShapesDisjoint, $currentShape); + return c <= Number($qualifiedMaxCount.lex); +} + +var validateQualifiedMinCountProperty = function ($this, $path, $qualifiedValueShape, $qualifiedValueShapesDisjoint, $qualifiedMinCount, $currentShape) { + var c = validateQualifiedHelper($this, $path, $qualifiedValueShape, $qualifiedValueShapesDisjoint, $currentShape); + return c >= Number($qualifiedMinCount.lex); +} + +var validateQualifiedHelper = function ($this, $path, $qualifiedValueShape, $qualifiedValueShapesDisjoint, $currentShape) { + var siblingShapes = new rdfquery.NodeSet(); + if (T("true").equals($qualifiedValueShapesDisjoint)) { + $shapes.query(). + match("?parentShape", "sh:property", $currentShape). + match("?parentShape", "sh:property", "?sibling"). + match("?sibling", "sh:qualifiedValueShape", "?siblingShape"). + filter(exprNotEquals("?siblingShape", $qualifiedValueShape)). + addAllNodes("?siblingShape", siblingShapes); + } + return $data.query(). + path($this, rdfquery.toRDFQueryPath($path), "?value"). + filter(function (sol) { + return validator.SHACL.nodeConformsToShape(sol.value, $qualifiedValueShape) && + !validateQualifiedConformsToASibling(sol.value, siblingShapes.toArray()); + }). + getCount(); +} + +var validateQualifiedConformsToASibling = function (value, siblingShapes) { + for (var i = 0; i < siblingShapes.length; i++) { + if (validator.SHACL.nodeConformsToShape(value, siblingShapes[i])) { + return true; + } + } + return false; +} + +var validateRootClass = function ($value, $rootClass) { + return $data.query().path($value, { zeroOrMore: T("rdfs:subClassOf") }, $rootClass).hasSolution(); +} + +var validateStem = function ($value, $stem) { + return $value.isURI() && $value.uri.startsWith($stem.lex); +} + +var validateSubSetOf = function ($this, $subSetOf, $value) { + return $data.query().match($this, $subSetOf, $value).hasSolution(); +} + +var validateUniqueLangProperty = function ($this, $uniqueLang, $path) { + if (!T("true").equals($uniqueLang)) { + return; + } + var map = {}; + $data.query().path($this, rdfquery.toRDFQueryPath($path), "?value").forEach(function (sol) { + var lang = sol.value.language; + if (lang && lang != "") { + var old = map[lang]; + if (!old) { + map[lang] = 1; + } + else { + map[lang] = old + 1; + } + } + }); + var results = []; + for (var lang in map) { + if (map.hasOwnProperty(lang)) { + var count = map[lang]; + if (count > 1) { + results.push("Language \"" + lang + "\" has been used by " + count + " values"); + } + } + } + return results; +} + +var validateXone = function ($value, $xone) { + var shapes = new rdfquery.RDFQueryUtil($shapes).rdfListToArray($xone); + var count = 0; + for (var i = 0; i < shapes.length; i++) { + if (validator.SHACL.nodeConformsToShape($value, shapes[i])) { + count++; + } + } + return count == 1; +} + + + + +// Private helper functions + +//TODO: Support more datatypes +var isValidForDatatype = function (lex, datatype) { + if (XSDIntegerTypes.contains(datatype)) { + var r = parseInt(lex); + return !isNaN(r); + } + else if (XSDDecimalTypes.contains(datatype)) { + var r = parseFloat(lex); + return !isNan(r); + } + else { + return true; + } +} + + +validator.ValidationFunction.functionRegistry.validateAnd = validateAnd; +validator.ValidationFunction.functionRegistry.validateClass = validateClass; +validator.ValidationFunction.functionRegistry.validateClosed = validateClosed; +validator.ValidationFunction.functionRegistry.validateClosedByTypesNode = validateClosedByTypesNode; +validator.ValidationFunction.functionRegistry.validateDatatype = validateDatatype; +validator.ValidationFunction.functionRegistry.validateDisjoint = validateDisjoint; +validator.ValidationFunction.functionRegistry.validateEqualsProperty = validateEqualsProperty; +validator.ValidationFunction.functionRegistry.validateHasValueNode = validateHasValueNode; +validator.ValidationFunction.functionRegistry.validateHasValueProperty = validateHasValueProperty; +validator.ValidationFunction.functionRegistry.validateIn = validateIn; +validator.ValidationFunction.functionRegistry.validateLanguageIn = validateLanguageIn; +validator.ValidationFunction.functionRegistry.validateLessThanProperty = validateLessThanProperty; +validator.ValidationFunction.functionRegistry.validateLessThanOrEqualsProperty = validateLessThanOrEqualsProperty; +validator.ValidationFunction.functionRegistry.validateMaxCountProperty = validateMaxCountProperty; +validator.ValidationFunction.functionRegistry.validateMaxExclusive = validateMaxExclusive; +validator.ValidationFunction.functionRegistry.validateMaxInclusive = validateMaxInclusive; +validator.ValidationFunction.functionRegistry.validateMaxLength = validateMaxLength; +validator.ValidationFunction.functionRegistry.validateMinCountProperty = validateMinCountProperty; +validator.ValidationFunction.functionRegistry.validateMinExclusive = validateMinExclusive; +validator.ValidationFunction.functionRegistry.validateMinInclusive = validateMinInclusive; +validator.ValidationFunction.functionRegistry.validateMinLength = validateMinLength; +validator.ValidationFunction.functionRegistry.validateNodeKind = validateNodeKind; +validator.ValidationFunction.functionRegistry.validateNode = validateNode; +validator.ValidationFunction.functionRegistry.validateNonRecursiveProperty = validateNonRecursiveProperty; +validator.ValidationFunction.functionRegistry.validateNot = validateNot; +validator.ValidationFunction.functionRegistry.validateOr = validateOr; +validator.ValidationFunction.functionRegistry.validatePattern = validatePattern; +validator.ValidationFunction.functionRegistry.validatePrimaryKeyProperty = validatePrimaryKeyProperty; +validator.ValidationFunction.functionRegistry.validateQualifiedMaxCountProperty = validateQualifiedMaxCountProperty; +validator.ValidationFunction.functionRegistry.validateQualifiedMinCountProperty = validateQualifiedMinCountProperty; +validator.ValidationFunction.functionRegistry.validateQualifiedHelper = validateQualifiedHelper; +validator.ValidationFunction.functionRegistry.validateQualifiedConformsToASibling = validateQualifiedConformsToASibling; +validator.ValidationFunction.functionRegistry.validateRootClass = validateRootClass; +validator.ValidationFunction.functionRegistry.validateStem = validateStem; +validator.ValidationFunction.functionRegistry.validateSubSetOf = validateSubSetOf; +validator.ValidationFunction.functionRegistry.validateUniqueLangProperty = validateUniqueLangProperty; +validator.ValidationFunction.functionRegistry.validateXone = validateXone; + +},{"./rdfquery":91,"./shacl-validator":92}],41:[function(require,module,exports){ (function (process){ +/*! + * async + * https://github.com/caolan/async + * + * Copyright 2010-2014 Caolan McMahon + * Released under the MIT license + */ +/*jshint onevar: false, indent:4 */ +/*global setImmediate: false, setTimeout: false, console: false */ +(function () { + + var async = {}; + + // global on the server, window in the browser + var root, previous_async; + + root = this; + if (root != null) { + previous_async = root.async; + } + + async.noConflict = function () { + root.async = previous_async; + return async; + }; + + function only_once(fn) { + var called = false; + return function() { + if (called) throw new Error("Callback was already called."); + called = true; + fn.apply(root, arguments); + } + } + + //// cross-browser compatiblity functions //// + + var _toString = Object.prototype.toString; + + var _isArray = Array.isArray || function (obj) { + return _toString.call(obj) === '[object Array]'; + }; + + var _each = function (arr, iterator) { + for (var i = 0; i < arr.length; i += 1) { + iterator(arr[i], i, arr); + } + }; + + var _map = function (arr, iterator) { + if (arr.map) { + return arr.map(iterator); + } + var results = []; + _each(arr, function (x, i, a) { + results.push(iterator(x, i, a)); + }); + return results; + }; + + var _reduce = function (arr, iterator, memo) { + if (arr.reduce) { + return arr.reduce(iterator, memo); + } + _each(arr, function (x, i, a) { + memo = iterator(memo, x, i, a); + }); + return memo; + }; + + var _keys = function (obj) { + if (Object.keys) { + return Object.keys(obj); + } + var keys = []; + for (var k in obj) { + if (obj.hasOwnProperty(k)) { + keys.push(k); + } + } + return keys; + }; + + //// exported async module functions //// + + //// nextTick implementation with browser-compatible fallback //// + if (typeof process === 'undefined' || !(process.nextTick)) { + if (typeof setImmediate === 'function') { + async.nextTick = function (fn) { + // not a direct alias for IE10 compatibility + setImmediate(fn); + }; + async.setImmediate = async.nextTick; + } + else { + async.nextTick = function (fn) { + setTimeout(fn, 0); + }; + async.setImmediate = async.nextTick; + } + } + else { + async.nextTick = process.nextTick; + if (typeof setImmediate !== 'undefined') { + async.setImmediate = function (fn) { + // not a direct alias for IE10 compatibility + setImmediate(fn); + }; + } + else { + async.setImmediate = async.nextTick; + } + } + + async.each = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + _each(arr, function (x) { + iterator(x, only_once(done) ); + }); + function done(err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(); + } + } + } + }; + async.forEach = async.each; + + async.eachSeries = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + var iterate = function () { + iterator(arr[completed], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(); + } + else { + iterate(); + } + } + }); + }; + iterate(); + }; + async.forEachSeries = async.eachSeries; + + async.eachLimit = function (arr, limit, iterator, callback) { + var fn = _eachLimit(limit); + fn.apply(null, [arr, iterator, callback]); + }; + async.forEachLimit = async.eachLimit; + + var _eachLimit = function (limit) { + + return function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length || limit <= 0) { + return callback(); + } + var completed = 0; + var started = 0; + var running = 0; + + (function replenish () { + if (completed >= arr.length) { + return callback(); + } + + while (running < limit && started < arr.length) { + started += 1; + running += 1; + iterator(arr[started - 1], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + running -= 1; + if (completed >= arr.length) { + callback(); + } + else { + replenish(); + } + } + }); + } + })(); + }; + }; + + + var doParallel = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.each].concat(args)); + }; + }; + var doParallelLimit = function(limit, fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [_eachLimit(limit)].concat(args)); + }; + }; + var doSeries = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.eachSeries].concat(args)); + }; + }; + + + var _asyncMap = function (eachfn, arr, iterator, callback) { + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + if (!callback) { + eachfn(arr, function (x, callback) { + iterator(x.value, function (err) { + callback(err); + }); + }); + } else { + var results = []; + eachfn(arr, function (x, callback) { + iterator(x.value, function (err, v) { + results[x.index] = v; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + async.map = doParallel(_asyncMap); + async.mapSeries = doSeries(_asyncMap); + async.mapLimit = function (arr, limit, iterator, callback) { + return _mapLimit(limit)(arr, iterator, callback); + }; + + var _mapLimit = function(limit) { + return doParallelLimit(limit, _asyncMap); + }; + + // reduce only has a series version, as doing reduce in parallel won't + // work in many situations. + async.reduce = function (arr, memo, iterator, callback) { + async.eachSeries(arr, function (x, callback) { + iterator(memo, x, function (err, v) { + memo = v; + callback(err); + }); + }, function (err) { + callback(err, memo); + }); + }; + // inject alias + async.inject = async.reduce; + // foldl alias + async.foldl = async.reduce; + + async.reduceRight = function (arr, memo, iterator, callback) { + var reversed = _map(arr, function (x) { + return x; + }).reverse(); + async.reduce(reversed, memo, iterator, callback); + }; + // foldr alias + async.foldr = async.reduceRight; + + var _filter = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.filter = doParallel(_filter); + async.filterSeries = doSeries(_filter); + // select alias + async.select = async.filter; + async.selectSeries = async.filterSeries; + + var _reject = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (!v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.reject = doParallel(_reject); + async.rejectSeries = doSeries(_reject); + + var _detect = function (eachfn, arr, iterator, main_callback) { + eachfn(arr, function (x, callback) { + iterator(x, function (result) { + if (result) { + main_callback(x); + main_callback = function () {}; + } + else { + callback(); + } + }); + }, function (err) { + main_callback(); + }); + }; + async.detect = doParallel(_detect); + async.detectSeries = doSeries(_detect); + + async.some = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (v) { + main_callback(true); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(false); + }); + }; + // any alias + async.any = async.some; + + async.every = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (!v) { + main_callback(false); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(true); + }); + }; + // all alias + async.all = async.every; + + async.sortBy = function (arr, iterator, callback) { + async.map(arr, function (x, callback) { + iterator(x, function (err, criteria) { + if (err) { + callback(err); + } + else { + callback(null, {value: x, criteria: criteria}); + } + }); + }, function (err, results) { + if (err) { + return callback(err); + } + else { + var fn = function (left, right) { + var a = left.criteria, b = right.criteria; + return a < b ? -1 : a > b ? 1 : 0; + }; + callback(null, _map(results.sort(fn), function (x) { + return x.value; + })); + } + }); + }; + + async.auto = function (tasks, callback) { + callback = callback || function () {}; + var keys = _keys(tasks); + var remainingTasks = keys.length + if (!remainingTasks) { + return callback(); + } + + var results = {}; + + var listeners = []; + var addListener = function (fn) { + listeners.unshift(fn); + }; + var removeListener = function (fn) { + for (var i = 0; i < listeners.length; i += 1) { + if (listeners[i] === fn) { + listeners.splice(i, 1); + return; + } + } + }; + var taskComplete = function () { + remainingTasks-- + _each(listeners.slice(0), function (fn) { + fn(); + }); + }; + + addListener(function () { + if (!remainingTasks) { + var theCallback = callback; + // prevent final callback from calling itself if it errors + callback = function () {}; + + theCallback(null, results); + } + }); + + _each(keys, function (k) { + var task = _isArray(tasks[k]) ? tasks[k]: [tasks[k]]; + var taskCallback = function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + if (err) { + var safeResults = {}; + _each(_keys(results), function(rkey) { + safeResults[rkey] = results[rkey]; + }); + safeResults[k] = args; + callback(err, safeResults); + // stop subsequent errors hitting callback multiple times + callback = function () {}; + } + else { + results[k] = args; + async.setImmediate(taskComplete); + } + }; + var requires = task.slice(0, Math.abs(task.length - 1)) || []; + var ready = function () { + return _reduce(requires, function (a, x) { + return (a && results.hasOwnProperty(x)); + }, true) && !results.hasOwnProperty(k); + }; + if (ready()) { + task[task.length - 1](taskCallback, results); + } + else { + var listener = function () { + if (ready()) { + removeListener(listener); + task[task.length - 1](taskCallback, results); + } + }; + addListener(listener); + } + }); + }; + + async.retry = function(times, task, callback) { + var DEFAULT_TIMES = 5; + var attempts = []; + // Use defaults if times not passed + if (typeof times === 'function') { + callback = task; + task = times; + times = DEFAULT_TIMES; + } + // Make sure times is a number + times = parseInt(times, 10) || DEFAULT_TIMES; + var wrappedTask = function(wrappedCallback, wrappedResults) { + var retryAttempt = function(task, finalAttempt) { + return function(seriesCallback) { + task(function(err, result){ + seriesCallback(!err || finalAttempt, {err: err, result: result}); + }, wrappedResults); + }; + }; + while (times) { + attempts.push(retryAttempt(task, !(times-=1))); + } + async.series(attempts, function(done, data){ + data = data[data.length - 1]; + (wrappedCallback || callback)(data.err, data.result); + }); + } + // If a callback is passed, run this as a controll flow + return callback ? wrappedTask() : wrappedTask + }; + + async.waterfall = function (tasks, callback) { + callback = callback || function () {}; + if (!_isArray(tasks)) { + var err = new Error('First argument to waterfall must be an array of functions'); + return callback(err); + } + if (!tasks.length) { + return callback(); + } + var wrapIterator = function (iterator) { + return function (err) { + if (err) { + callback.apply(null, arguments); + callback = function () {}; + } + else { + var args = Array.prototype.slice.call(arguments, 1); + var next = iterator.next(); + if (next) { + args.push(wrapIterator(next)); + } + else { + args.push(callback); + } + async.setImmediate(function () { + iterator.apply(null, args); + }); + } + }; + }; + wrapIterator(async.iterator(tasks))(); + }; + + var _parallel = function(eachfn, tasks, callback) { + callback = callback || function () {}; + if (_isArray(tasks)) { + eachfn.map(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + eachfn.each(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.parallel = function (tasks, callback) { + _parallel({ map: async.map, each: async.each }, tasks, callback); + }; + + async.parallelLimit = function(tasks, limit, callback) { + _parallel({ map: _mapLimit(limit), each: _eachLimit(limit) }, tasks, callback); + }; + + async.series = function (tasks, callback) { + callback = callback || function () {}; + if (_isArray(tasks)) { + async.mapSeries(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + async.eachSeries(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.iterator = function (tasks) { + var makeCallback = function (index) { + var fn = function () { + if (tasks.length) { + tasks[index].apply(null, arguments); + } + return fn.next(); + }; + fn.next = function () { + return (index < tasks.length - 1) ? makeCallback(index + 1): null; + }; + return fn; + }; + return makeCallback(0); + }; + + async.apply = function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + return function () { + return fn.apply( + null, args.concat(Array.prototype.slice.call(arguments)) + ); + }; + }; + + var _concat = function (eachfn, arr, fn, callback) { + var r = []; + eachfn(arr, function (x, cb) { + fn(x, function (err, y) { + r = r.concat(y || []); + cb(err); + }); + }, function (err) { + callback(err, r); + }); + }; + async.concat = doParallel(_concat); + async.concatSeries = doSeries(_concat); + + async.whilst = function (test, iterator, callback) { + if (test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.whilst(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doWhilst = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + var args = Array.prototype.slice.call(arguments, 1); + if (test.apply(null, args)) { + async.doWhilst(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.until = function (test, iterator, callback) { + if (!test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.until(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doUntil = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + var args = Array.prototype.slice.call(arguments, 1); + if (!test.apply(null, args)) { + async.doUntil(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.queue = function (worker, concurrency) { + if (concurrency === undefined) { + concurrency = 1; + } + function _insert(q, data, pos, callback) { + if (!q.started){ + q.started = true; + } + if (!_isArray(data)) { + data = [data]; + } + if(data.length == 0) { + // call drain immediately if there are no tasks + return async.setImmediate(function() { + if (q.drain) { + q.drain(); + } + }); + } + _each(data, function(task) { + var item = { + data: task, + callback: typeof callback === 'function' ? callback : null + }; + + if (pos) { + q.tasks.unshift(item); + } else { + q.tasks.push(item); + } + + if (q.saturated && q.tasks.length === q.concurrency) { + q.saturated(); + } + async.setImmediate(q.process); + }); + } + + var workers = 0; + var q = { + tasks: [], + concurrency: concurrency, + saturated: null, + empty: null, + drain: null, + started: false, + paused: false, + push: function (data, callback) { + _insert(q, data, false, callback); + }, + kill: function () { + q.drain = null; + q.tasks = []; + }, + unshift: function (data, callback) { + _insert(q, data, true, callback); + }, + process: function () { + if (!q.paused && workers < q.concurrency && q.tasks.length) { + var task = q.tasks.shift(); + if (q.empty && q.tasks.length === 0) { + q.empty(); + } + workers += 1; + var next = function () { + workers -= 1; + if (task.callback) { + task.callback.apply(task, arguments); + } + if (q.drain && q.tasks.length + workers === 0) { + q.drain(); + } + q.process(); + }; + var cb = only_once(next); + worker(task.data, cb); + } + }, + length: function () { + return q.tasks.length; + }, + running: function () { + return workers; + }, + idle: function() { + return q.tasks.length + workers === 0; + }, + pause: function () { + if (q.paused === true) { return; } + q.paused = true; + }, + resume: function () { + if (q.paused === false) { return; } + q.paused = false; + // Need to call q.process once per concurrent + // worker to preserve full concurrency after pause + for (var w = 1; w <= q.concurrency; w++) { + async.setImmediate(q.process); + } + } + }; + return q; + }; + + async.priorityQueue = function (worker, concurrency) { + + function _compareTasks(a, b){ + return a.priority - b.priority; + }; + + function _binarySearch(sequence, item, compare) { + var beg = -1, + end = sequence.length - 1; + while (beg < end) { + var mid = beg + ((end - beg + 1) >>> 1); + if (compare(item, sequence[mid]) >= 0) { + beg = mid; + } else { + end = mid - 1; + } + } + return beg; + } + + function _insert(q, data, priority, callback) { + if (!q.started){ + q.started = true; + } + if (!_isArray(data)) { + data = [data]; + } + if(data.length == 0) { + // call drain immediately if there are no tasks + return async.setImmediate(function() { + if (q.drain) { + q.drain(); + } + }); + } + _each(data, function(task) { + var item = { + data: task, + priority: priority, + callback: typeof callback === 'function' ? callback : null + }; + + q.tasks.splice(_binarySearch(q.tasks, item, _compareTasks) + 1, 0, item); + + if (q.saturated && q.tasks.length === q.concurrency) { + q.saturated(); + } + async.setImmediate(q.process); + }); + } + + // Start with a normal queue + var q = async.queue(worker, concurrency); + + // Override push to accept second parameter representing priority + q.push = function (data, priority, callback) { + _insert(q, data, priority, callback); + }; + + // Remove unshift function + delete q.unshift; + + return q; + }; + + async.cargo = function (worker, payload) { + var working = false, + tasks = []; + + var cargo = { + tasks: tasks, + payload: payload, + saturated: null, + empty: null, + drain: null, + drained: true, + push: function (data, callback) { + if (!_isArray(data)) { + data = [data]; + } + _each(data, function(task) { + tasks.push({ + data: task, + callback: typeof callback === 'function' ? callback : null + }); + cargo.drained = false; + if (cargo.saturated && tasks.length === payload) { + cargo.saturated(); + } + }); + async.setImmediate(cargo.process); + }, + process: function process() { + if (working) return; + if (tasks.length === 0) { + if(cargo.drain && !cargo.drained) cargo.drain(); + cargo.drained = true; + return; + } + + var ts = typeof payload === 'number' + ? tasks.splice(0, payload) + : tasks.splice(0, tasks.length); + + var ds = _map(ts, function (task) { + return task.data; + }); + + if(cargo.empty) cargo.empty(); + working = true; + worker(ds, function () { + working = false; + + var args = arguments; + _each(ts, function (data) { + if (data.callback) { + data.callback.apply(null, args); + } + }); + + process(); + }); + }, + length: function () { + return tasks.length; + }, + running: function () { + return working; + } + }; + return cargo; + }; + + var _console_fn = function (name) { + return function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + fn.apply(null, args.concat([function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (typeof console !== 'undefined') { + if (err) { + if (console.error) { + console.error(err); + } + } + else if (console[name]) { + _each(args, function (x) { + console[name](x); + }); + } + } + }])); + }; + }; + async.log = _console_fn('log'); + async.dir = _console_fn('dir'); + /*async.info = _console_fn('info'); + async.warn = _console_fn('warn'); + async.error = _console_fn('error');*/ + + async.memoize = function (fn, hasher) { + var memo = {}; + var queues = {}; + hasher = hasher || function (x) { + return x; + }; + var memoized = function () { + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + var key = hasher.apply(null, args); + if (key in memo) { + async.nextTick(function () { + callback.apply(null, memo[key]); + }); + } + else if (key in queues) { + queues[key].push(callback); + } + else { + queues[key] = [callback]; + fn.apply(null, args.concat([function () { + memo[key] = arguments; + var q = queues[key]; + delete queues[key]; + for (var i = 0, l = q.length; i < l; i++) { + q[i].apply(null, arguments); + } + }])); + } + }; + memoized.memo = memo; + memoized.unmemoized = fn; + return memoized; + }; + + async.unmemoize = function (fn) { + return function () { + return (fn.unmemoized || fn).apply(null, arguments); + }; + }; + + async.times = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.map(counter, iterator, callback); + }; + + async.timesSeries = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.mapSeries(counter, iterator, callback); + }; + + async.seq = function (/* functions... */) { + var fns = arguments; + return function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + async.reduce(fns, args, function (newargs, fn, cb) { + fn.apply(that, newargs.concat([function () { + var err = arguments[0]; + var nextargs = Array.prototype.slice.call(arguments, 1); + cb(err, nextargs); + }])) + }, + function (err, results) { + callback.apply(that, [err].concat(results)); + }); + }; + }; + + async.compose = function (/* functions... */) { + return async.seq.apply(null, Array.prototype.reverse.call(arguments)); + }; + + var _applyEach = function (eachfn, fns /*args...*/) { + var go = function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + return eachfn(fns, function (fn, cb) { + fn.apply(that, args.concat([cb])); + }, + callback); + }; + if (arguments.length > 2) { + var args = Array.prototype.slice.call(arguments, 2); + return go.apply(this, args); + } + else { + return go; + } + }; + async.applyEach = doParallel(_applyEach); + async.applyEachSeries = doSeries(_applyEach); + + async.forever = function (fn, callback) { + function next(err) { + if (err) { + if (callback) { + return callback(err); + } + throw err; + } + fn(next); + } + next(); + }; + + // Node.js + if (typeof module !== 'undefined' && module.exports) { + module.exports = async; + } + // AMD / RequireJS + else if (typeof define !== 'undefined' && define.amd) { + define([], function () { + return async; + }); + } + // included directly via