Add fallback encoding for non-UTF-8 lines

Default to cp1252
kiwiirc · Apr 1, 2018 · bdb08e6 · bdb08e6
1 parent 495314b
commit bdb08e6
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 5 deletions.
diff --git a/docs/clientapi.md b/docs/clientapi.md
@@ -9,6 +9,7 @@ new Irc.Client({
     username: 'ircbot',
     gecos: 'ircbot',
     encoding: 'utf8',
+    encoding_fallback: 'cp1252',
     version: 'node.js irc-framework',
     enable_chghost: false,
     enable_echomessage: false,

diff --git a/package.json b/package.json
@@ -8,7 +8,8 @@
     "iconv-lite": "^0.4.11",
     "lodash": "^4.17.4",
     "middleware-handler": "^0.2.0",
-    "socksjs": "^0.5.0"
+    "socksjs": "^0.5.0",
+    "utf-8-validate": "^4.0.0"
   },
   "devDependencies": {
     "babel-cli": "^6.26.0",

diff --git a/src/client.js b/src/client.js
@@ -36,6 +36,7 @@ module.exports = class IrcClient extends EventEmitter {
             username: 'ircbot',
             gecos: 'ircbot',
             encoding: 'utf8',
+            encoding_fallback: 'cp1252',
             version: 'node.js irc-framework',
             enable_chghost: false,
             enable_echomessage: false,

diff --git a/src/connection.js b/src/connection.js
@@ -58,6 +58,10 @@ module.exports = class Connection extends EventEmitter {
             this.setEncoding('utf8');
         }
 
+        if (!options.encoding_fallback || !this.setEncodingFallback(options.encoding_fallback)) {
+            this.setEncodingFallback('cp1252');
+        }
+
         // Some transports may emit extra events
         transport.on('extra', function(/*event_name, argN*/) {
             that.emit.apply(that, arguments);
@@ -241,6 +245,14 @@ module.exports = class Connection extends EventEmitter {
         }
     }
 
+    setEncodingFallback(encoding) {
+        this.debugOut('Connection.setEncodingFallback() encoding=' + encoding);
+
+        if (this.transport) {
+            return this.transport.setEncodingFallback(encoding);
+        }
+    }
+
 
     /**
      * Process the buffered messages recieved from the IRCd

diff --git a/src/transports/net.js b/src/transports/net.js
@@ -10,6 +10,7 @@ var util            = require('util');
 var EventEmitter    = require('events').EventEmitter;
 var Socks           = require('socksjs');
 var iconv           = require('iconv-lite');
+var isValidUTF8     = require('utf-8-validate');
 
 var SOCK_DISCONNECTED = 0;
 var SOCK_CONNECTING = 1;
@@ -27,6 +28,7 @@ module.exports = class Connection extends EventEmitter {
         this.socket_events = [];
 
         this.encoding = 'utf8';
+        this.encoding_fallback = 'cp1252';
         this.incoming_buffer = '';
     }
 
@@ -154,7 +156,7 @@ module.exports = class Connection extends EventEmitter {
     }
 
     onSocketData(data) {
-    	this.incoming_buffer += iconv.decode(data, this.encoding);
+        this.incoming_buffer += this.decodeBuffer(data);
 
     	var lines = this.incoming_buffer.split('\n');
     	if (lines[lines.length - 1] !== '') {
@@ -197,19 +199,49 @@ module.exports = class Connection extends EventEmitter {
         }
     }
 
+    decodeBuffer(data) {
+        if (this.encoding === 'utf8' && this.encoding_fallback) {
+            if (isValidUTF8(data)) {
+                return iconv.decode(data, 'utf8');
+            } else {
+                return iconv.decode(data, this.encoding_fallback);
+            }
+        } else {
+            return iconv.decode(data, this.encoding);
+        }
+    }
 
     setEncoding(encoding) {
-        var encoded_test;
-
         this.debugOut('Connection.setEncoding() encoding=' + encoding);
 
+        if (this.testEncoding(encoding)) {
+            this.encoding = encoding;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    setEncodingFallback(encoding) {
+        this.debugOut('Connection.setEncodingFallback() encoding=' + encoding);
+
+        if (this.testEncoding(encoding)) {
+            this.encoding_fallback = encoding;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    testEncoding(encoding) {
+        var encoded_test;
+
         try {
             encoded_test = iconv.encode('TEST', encoding);
             // This test is done to check if this encoding also supports
             // the ASCII charset required by the IRC protocols
             // (Avoid the use of base64 or incompatible encodings)
             if (encoded_test == 'TEST') { // jshint ignore:line
-                this.encoding = encoding;
                 return true;
             }
             return false;

diff --git a/src/transports/websocket.js b/src/transports/websocket.js
@@ -126,4 +126,7 @@ module.exports = class Connection extends EventEmitter {
 
     setEncoding(encoding) {
     }
+
+    setEncodingFallback(encoding) {
+    }
 };