WIP: use WASM for hashing #48
Open
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This is faster in every browser but there’s still a fair amount of variation depending on the browser and we presumably need to spend some time troubleshooting performance: * Safari goes from 74MB/s to 98MB/s on large files * Firefox goes from 6MB/s to ~11MB/s on the same files * Chrome goes from 59MB/s to 93MB/s on the same files (sha256deep is ~131MB/s on the same files) This is complicated by Firefox’s profiler not supporting either workers or WASM, making it hard to tell why it’s so far behind. Source for the new worker implementation: acdha/wasm-hashing@673e09b
houndci-bot
reviewed
Aug 7, 2018
| @@ -0,0 +1,2 @@ | |||
| !function(e){self.webpackChunk=function(t,r){for(var o in r)e[o]=r[o];for(;t.length;)n[t.pop()]=1};var t={},n={0:1},r={};var o={2:function(){return{"./wasm_hashing":{__wbindgen_throw:function(e,n){return t[1].exports.__wbindgen_throw(e,n)}}}}};function i(n){if(t[n])return t[n].exports;var r=t[n]={i:n,l:!1,exports:{}};return e[n].call(r.exports,r,r.exports,i),r.l=!0,r.exports}i.e=function(e){var t=[];return t.push(Promise.resolve().then(function(){n[e]||importScripts(e+".webworker.js")})),({1:[2]}[e]||[]).forEach(function(e){var n=r[e];if(n)t.push(n);else{var s,a=o[e](),f=fetch(i.p+""+{2:"28e7423f674361fb315d"}[e]+".module.wasm");if(a instanceof Promise&&"function"==typeof WebAssembly.compileStreaming)s=Promise.all([WebAssembly.compileStreaming(f),a]).then(function(e){return WebAssembly.instantiate(e[0],e[1])});else if("function"==typeof WebAssembly.instantiateStreaming)s=WebAssembly.instantiateStreaming(f,a);else{s=f.then(function(e){return e.arrayBuffer()}).then(function(e){return WebAssembly.instantiate(e,a)})}t.push(r[e]=s.then(function(t){return i.w[e]=(t.instance||t).exports}))}}),Promise.all(t)},i.m=e,i.c=t,i.d=function(e,t,n){i.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,t){if(1&t&&(e=i(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(i.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var r in e)i.d(n,r,function(t){return e[t]}.bind(null,r));return n},i.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(t,"a",t),t},i.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},i.p="",i.w={},i(i.s=0)}([function(e,t,n){n.e(1).then(n.bind(null,1)).then(e=>{let t=e.Sha2_256Hasher;self.addEventListener("message",({data:{file:e,fullPath:n}})=>{let r=new t;const o=e.size,i=performance.now();for(let t,s=new FileReaderSync,a=0;a<o;a=t){t=a+Math.min(1048576,o-a);let f=e.slice(a,t),l=s.readAsArrayBuffer(f);r.update(new Uint8Array(l)),postMessage({type:"PROGRESS_UPDATE",fullPath:n,bytesHashed:t,elapsedMilliseconds:performance.now()-i})}let s=r.hex_digest();if(e.size>0&&"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"==s)throw`Hashed ${e.size} bytes as an empty result!`;postMessage({type:"RESULT",fullPath:n,sha256:s,bytesHashed:o,elapsedMilliseconds:performance.now()-i})})})}]); | |||
There was a problem hiding this comment.
'importScripts' is not defined no-undef
'FileReaderSync' is not defined no-undef
| @@ -0,0 +1,2 @@ | |||
| self.webpackChunk([1],[,function(t,e,r){"use strict";r.r(e),r.d(e,"hash_sha1",function(){return h}),r.d(e,"hash_sha2_256",function(){return y}),r.d(e,"hash_sha2_512",function(){return d}),r.d(e,"hash_sha3_256",function(){return g}),r.d(e,"hash_sha3_512",function(){return w}),r.d(e,"Sha2_512Hasher",function(){return v}),r.d(e,"Sha3_256Hasher",function(){return b}),r.d(e,"Sha2_256Hasher",function(){return _}),r.d(e,"Sha3_512Hasher",function(){return x}),r.d(e,"Sha1Hasher",function(){return E}),r.d(e,"__wbindgen_throw",function(){return j});var n=r(2);let o=null;function i(){return null!==o&&o.buffer===n.n.buffer||(o=new Uint8Array(n.n.buffer)),o}function u(t){const e=n.h(1*t.length);return i().set(t,e/1),[e,t.length]}let c=new("object"==typeof self&&self.TextDecoder?self.TextDecoder:r(3).TextDecoder)("utf-8");function s(t,e){return c.decode(i().subarray(t,t+e))}let f=null;function l(){return null===f&&(f=n.g()),f}let a=null;function p(){return null!==a&&a.buffer===n.n.buffer||(a=new Uint32Array(n.n.buffer)),a}function h(t){const[e,r]=u(t),o=l();try{n.i(o,e,r);const t=p(),i=t[o/4],u=t[o/4+1],c=s(i,u).slice();return n.f(i,1*u),c}finally{n.f(e,1*r)}}function y(t){const[e,r]=u(t),o=l();try{n.j(o,e,r);const t=p(),i=t[o/4],u=t[o/4+1],c=s(i,u).slice();return n.f(i,1*u),c}finally{n.f(e,1*r)}}function d(t){const[e,r]=u(t),o=l();try{n.k(o,e,r);const t=p(),i=t[o/4],u=t[o/4+1],c=s(i,u).slice();return n.f(i,1*u),c}finally{n.f(e,1*r)}}function g(t){const[e,r]=u(t),o=l();try{n.l(o,e,r);const t=p(),i=t[o/4],u=t[o/4+1],c=s(i,u).slice();return n.f(i,1*u),c}finally{n.f(e,1*r)}}function w(t){const[e,r]=u(t),o=l();try{n.m(o,e,r);const t=p(),i=t[o/4],u=t[o/4+1],c=s(i,u).slice();return n.f(i,1*u),c}finally{n.f(e,1*r)}}class m{constructor(t){this.ptr=t}}class v{static __construct(t){return new v(new m(t))}constructor(...t){if(1===t.length&&t[0]instanceof m)return void(this.ptr=t[0].ptr);let e=v.new(...t);this.ptr=e.ptr}free(){const t=this.ptr;this.ptr=0,n.c(t)}static new(){return v.__construct(n.v())}update(t){if(0===this.ptr)throw new Error("Attempt to use a moved value");const[e,r]=u(t);try{return n.w(this.ptr,e,r)}finally{n.f(e,1*r)}}hex_digest(){if(0===this.ptr)throw new Error("Attempt to use a moved value");const t=l();n.u(t,this.ptr);const e=p(),r=e[t/4],o=e[t/4+1],i=s(r,o).slice();return n.f(r,1*o),i}}class b{static __construct(t){return new b(new m(t))}constructor(...t){if(1===t.length&&t[0]instanceof m)return void(this.ptr=t[0].ptr);let e=b.new(...t);this.ptr=e.ptr}free(){const t=this.ptr;this.ptr=0,n.d(t)}static new(){return b.__construct(n.y())}update(t){if(0===this.ptr)throw new Error("Attempt to use a moved value");const[e,r]=u(t);try{return n.z(this.ptr,e,r)}finally{n.f(e,1*r)}}hex_digest(){if(0===this.ptr)throw new Error("Attempt to use a moved value");const t=l();n.x(t,this.ptr);const e=p(),r=e[t/4],o=e[t/4+1],i=s(r,o).slice();return n.f(r,1*o),i}}class _{static __construct(t){return new _(new m(t))}constructor(...t){if(1===t.length&&t[0]instanceof m)return void(this.ptr=t[0].ptr);let e=_.new(...t);this.ptr=e.ptr}free(){const t=this.ptr;this.ptr=0,n.b(t)}static new(){return _.__construct(n.s())}update(t){if(0===this.ptr)throw new Error("Attempt to use a moved value");const[e,r]=u(t);try{return n.t(this.ptr,e,r)}finally{n.f(e,1*r)}}hex_digest(){if(0===this.ptr)throw new Error("Attempt to use a moved value");const t=l();n.r(t,this.ptr);const e=p(),r=e[t/4],o=e[t/4+1],i=s(r,o).slice();return n.f(r,1*o),i}}class x{static __construct(t){return new x(new m(t))}constructor(...t){if(1===t.length&&t[0]instanceof m)return void(this.ptr=t[0].ptr);let e=x.new(...t);this.ptr=e.ptr}free(){const t=this.ptr;this.ptr=0,n.e(t)}static new(){return x.__construct(n.B())}update(t){if(0===this.ptr)throw new Error("Attempt to use a moved value");const[e,r]=u(t);try{return n.C(this.ptr,e,r)}finally{n.f(e,1*r)}}hex_digest(){if(0===this.ptr)throw new Error("Attempt to use a moved value");const t=l();n.A(t,this.ptr);const e=p(),r=e[t/4],o=e[t/4+1],i=s(r,o).slice();return n.f(r,1*o),i}}class E{static __construct(t){return new E(new m(t))}constructor(...t){if(1===t.length&&t[0]instanceof m)return void(this.ptr=t[0].ptr);let e=E.new(...t);this.ptr=e.ptr}free(){const t=this.ptr;this.ptr=0,n.a(t)}static new(){return E.__construct(n.p())}update(t){if(0===this.ptr)throw new Error("Attempt to use a moved value");const[e,r]=u(t);try{return n.q(this.ptr,e,r)}finally{n.f(e,1*r)}}hex_digest(){if(0===this.ptr)throw new Error("Attempt to use a moved value");const t=l();n.o(t,this.ptr);const e=p(),r=e[t/4],o=e[t/4+1],i=s(r,o).slice();return n.f(r,1*o),i}}function j(t,e){throw new Error(s(t,e))}},function(t,e,r){"use strict";var n=r.w[t.i];t.exports=n;r(1);n.D()},function(t,e,r){(function(t,n){var o=/%[sdj%]/g;e.format=function(t){if(!w(t)){for(var e=[],r=0;r<arguments.length;r++)e.push(c(arguments[r]));return e.join(" ")}r=1;for(var n=arguments,i=n.length,u=String(t).replace(o,function(t){if("%%"===t)return"%";if(r>=i)return t;switch(t){case"%s":return String(n[r++]);case"%d":return Number(n[r++]);case"%j":try{return JSON.stringify(n[r++])}catch(t){return"[Circular]"}default:return t}}),s=n[r];r<i;s=n[++r])d(s)||!b(s)?u+=" "+s:u+=" "+c(s);return u},e.deprecate=function(r,o){if(m(t.process))return function(){return e.deprecate(r,o).apply(this,arguments)};if(!0===n.noDeprecation)return r;var i=!1;return function(){if(!i){if(n.throwDeprecation)throw new Error(o);n.traceDeprecation?console.trace(o):console.error(o),i=!0}return r.apply(this,arguments)}};var i,u={};function c(t,r){var n={seen:[],stylize:f};return arguments.length>=3&&(n.depth=arguments[2]),arguments.length>=4&&(n.colors=arguments[3]),y(r)?n.showHidden=r:r&&e._extend(n,r),m(n.showHidden)&&(n.showHidden=!1),m(n.depth)&&(n.depth=2),m(n.colors)&&(n.colors=!1),m(n.customInspect)&&(n.customInspect=!0),n.colors&&(n.stylize=s),l(n,t,n.depth)}function s(t,e){var r=c.styles[e];return r?"["+c.colors[r][0]+"m"+t+"["+c.colors[r][1]+"m":t}function f(t,e){return t}function l(t,r,n){if(t.customInspect&&r&&E(r.inspect)&&r.inspect!==e.inspect&&(!r.constructor||r.constructor.prototype!==r)){var o=r.inspect(n,t);return w(o)||(o=l(t,o,n)),o}var i=function(t,e){if(m(e))return t.stylize("undefined","undefined");if(w(e)){var r="'"+JSON.stringify(e).replace(/^"|"$/g,"").replace(/'/g,"\\'").replace(/\\"/g,'"')+"'";return t.stylize(r,"string")}if(g(e))return t.stylize(""+e,"number");if(y(e))return t.stylize(""+e,"boolean");if(d(e))return t.stylize("null","null")}(t,r);if(i)return i;var u=Object.keys(r),c=function(t){var e={};return t.forEach(function(t,r){e[t]=!0}),e}(u);if(t.showHidden&&(u=Object.getOwnPropertyNames(r)),x(r)&&(u.indexOf("message")>=0||u.indexOf("description")>=0))return a(r);if(0===u.length){if(E(r)){var s=r.name?": "+r.name:"";return t.stylize("[Function"+s+"]","special")}if(v(r))return t.stylize(RegExp.prototype.toString.call(r),"regexp");if(_(r))return t.stylize(Date.prototype.toString.call(r),"date");if(x(r))return a(r)}var f,b="",j=!1,S=["{","}"];(h(r)&&(j=!0,S=["[","]"]),E(r))&&(b=" [Function"+(r.name?": "+r.name:"")+"]");return v(r)&&(b=" "+RegExp.prototype.toString.call(r)),_(r)&&(b=" "+Date.prototype.toUTCString.call(r)),x(r)&&(b=" "+a(r)),0!==u.length||j&&0!=r.length?n<0?v(r)?t.stylize(RegExp.prototype.toString.call(r),"regexp"):t.stylize("[Object]","special"):(t.seen.push(r),f=j?function(t,e,r,n,o){for(var i=[],u=0,c=e.length;u<c;++u)A(e,String(u))?i.push(p(t,e,r,n,String(u),!0)):i.push("");return o.forEach(function(o){o.match(/^\d+$/)||i.push(p(t,e,r,n,o,!0))}),i}(t,r,n,c,u):u.map(function(e){return p(t,r,n,c,e,j)}),t.seen.pop(),function(t,e,r){if(t.reduce(function(t,e){return 0,e.indexOf("\n")>=0&&0,t+e.replace(/\u001b\[\d\d?m/g,"").length+1},0)>60)return r[0]+(""===e?"":e+"\n ")+" "+t.join(",\n ")+" "+r[1];return r[0]+e+" "+t.join(", ")+" "+r[1]}(f,b,S)):S[0]+b+S[1]}function a(t){return"["+Error.prototype.toString.call(t)+"]"}function p(t,e,r,n,o,i){var u,c,s;if((s=Object.getOwnPropertyDescriptor(e,o)||{value:e[o]}).get?c=s.set?t.stylize("[Getter/Setter]","special"):t.stylize("[Getter]","special"):s.set&&(c=t.stylize("[Setter]","special")),A(n,o)||(u="["+o+"]"),c||(t.seen.indexOf(s.value)<0?(c=d(r)?l(t,s.value,null):l(t,s.value,r-1)).indexOf("\n")>-1&&(c=i?c.split("\n").map(function(t){return" "+t}).join("\n").substr(2):"\n"+c.split("\n").map(function(t){return" "+t}).join("\n")):c=t.stylize("[Circular]","special")),m(u)){if(i&&o.match(/^\d+$/))return c;(u=JSON.stringify(""+o)).match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)?(u=u.substr(1,u.length-2),u=t.stylize(u,"name")):(u=u.replace(/'/g,"\\'").replace(/\\"/g,'"').replace(/(^"|"$)/g,"'"),u=t.stylize(u,"string"))}return u+": "+c}function h(t){return Array.isArray(t)}function y(t){return"boolean"==typeof t}function d(t){return null===t}function g(t){return"number"==typeof t}function w(t){return"string"==typeof t}function m(t){return void 0===t}function v(t){return b(t)&&"[object RegExp]"===j(t)}function b(t){return"object"==typeof t&&null!==t}function _(t){return b(t)&&"[object Date]"===j(t)}function x(t){return b(t)&&("[object Error]"===j(t)||t instanceof Error)}function E(t){return"function"==typeof t}function j(t){return Object.prototype.toString.call(t)}function S(t){return t<10?"0"+t.toString(10):t.toString(10)}e.debuglog=function(t){if(m(i)&&(i=n.env.NODE_DEBUG||""),t=t.toUpperCase(),!u[t])if(new RegExp("\\b"+t+"\\b","i").test(i)){var r=n.pid;u[t]=function(){var n=e.format.apply(e,arguments);console.error("%s %d: %s",t,r,n)}}else u[t]=function(){};return u[t]},e.inspect=c,c.colors={bold:[1,22],italic:[3,23],underline:[4,24],inverse:[7,27],white:[37,39],grey:[90,39],black:[30,39],blue:[34,39],cyan:[36,39],green:[32,39],magenta:[35,39],red:[31,39],yellow:[33,39]},c.styles={special:"cyan",number:"yellow",boolean:"yellow",undefined:"grey",null:"bold",string:"green",date:"magenta",regexp:"red"},e.isArray=h,e.isBoolean=y,e.isNull=d,e.isNullOrUndefined=function(t){return null==t},e.isNumber=g,e.isString=w,e.isSymbol=function(t){return"symbol"==typeof t},e.isUndefined=m,e.isRegExp=v,e.isObject=b,e.isDate=_,e.isError=x,e.isFunction=E,e.isPrimitive=function(t){return null===t||"boolean"==typeof t||"number"==typeof t||"string"==typeof t||"symbol"==typeof t||void 0===t},e.isBuffer=r(6);var O=["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];function A(t,e){return Object.prototype.hasOwnProperty.call(t,e)}e.log=function(){console.log("%s - %s",function(){var t=new Date,e=[S(t.getHours()),S(t.getMinutes()),S(t.getSeconds())].join(":");return[t.getDate(),O[t.getMonth()],e].join(" ")}(),e.format.apply(e,arguments))},e.inherits=r(7),e._extend=function(t,e){if(!e||!b(e))return t;for(var r=Object.keys(e),n=r.length;n--;)t[r[n]]=e[r[n]];return t}}).call(this,r(4),r(5))},function(t,e){var r;r=function(){return this}();try{r=r||Function("return this")()||(0,eval)("this")}catch(t){"object"==typeof window&&(r=window)}t.exports=r},function(t,e){var r,n,o=t.exports={};function i(){throw new Error("setTimeout has not been defined")}function u(){throw new Error("clearTimeout has not been defined")}function c(t){if(r===setTimeout)return setTimeout(t,0);if((r===i||!r)&&setTimeout)return r=setTimeout,setTimeout(t,0);try{return r(t,0)}catch(e){try{return r.call(null,t,0)}catch(e){return r.call(this,t,0)}}}!function(){try{r="function"==typeof setTimeout?setTimeout:i}catch(t){r=i}try{n="function"==typeof clearTimeout?clearTimeout:u}catch(t){n=u}}();var s,f=[],l=!1,a=-1;function p(){l&&s&&(l=!1,s.length?f=s.concat(f):a=-1,f.length&&h())}function h(){if(!l){var t=c(p);l=!0;for(var e=f.length;e;){for(s=f,f=[];++a<e;)s&&s[a].run();a=-1,e=f.length}s=null,l=!1,function(t){if(n===clearTimeout)return clearTimeout(t);if((n===u||!n)&&clearTimeout)return n=clearTimeout,clearTimeout(t);try{n(t)}catch(e){try{return n.call(null,t)}catch(e){return n.call(this,t)}}}(t)}}function y(t,e){this.fun=t,this.array=e}function d(){}o.nextTick=function(t){var e=new Array(arguments.length-1);if(arguments.length>1)for(var r=1;r<arguments.length;r++)e[r-1]=arguments[r];f.push(new y(t,e)),1!==f.length||l||c(h)},y.prototype.run=function(){this.fun.apply(null,this.array)},o.title="browser",o.browser=!0,o.env={},o.argv=[],o.version="",o.versions={},o.on=d,o.addListener=d,o.once=d,o.off=d,o.removeListener=d,o.removeAllListeners=d,o.emit=d,o.prependListener=d,o.prependOnceListener=d,o.listeners=function(t){return[]},o.binding=function(t){throw new Error("process.binding is not supported")},o.cwd=function(){return"/"},o.chdir=function(t){throw new Error("process.chdir is not supported")},o.umask=function(){return 0}},function(t,e){t.exports=function(t){return t&&"object"==typeof t&&"function"==typeof t.copy&&"function"==typeof t.fill&&"function"==typeof t.readUInt8}},function(t,e){"function"==typeof Object.create?t.exports=function(t,e){t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}})}:t.exports=function(t,e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}]); | |||
There was a problem hiding this comment.
Unexpected comma in middle of array no-sparse-arrays
Unexpected console statement no-console
'e' is defined but never used no-unused-vars
'r' is defined but never used no-unused-vars
't' is defined but never used no-unused-vars
This avoids any possibility of timing skew due to the messaging code
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This is faster in every browser but there’s still a fair
amount of variation depending on the browser and we
presumably need to spend some time troubleshooting
performance:
(sha256deep is ~131MB/s on the same files)
Major questions before we can consider merging this: