Added support for utf8mb4 (emoji support)

Fixes issue q2a#791
NeayiAbandoned · Feb 25, 2020 · 3822807 · 3822807
1 parent 844f864
commit 3822807
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 5 deletions.
diff --git a/qa-config-example.php b/qa-config-example.php
@@ -63,6 +63,15 @@
 
 	define('QA_MYSQL_TABLE_PREFIX', 'qa_');
 
+/*
+	QA_USE_UTF8MB4 allows to use utf8mb4 instead of utf8 - this mainly allows compatibility with emojis.
+	Note that if you enable this setting after your database has been created, you need to change the
+	charset of all the tables in your database (using an export and reimport). See this discussion for
+	more info: https://www.question2answer.org/qa/62412/unicode-10-characters-filtered-out
+*/
+
+	define('QA_USE_UTF8MB4', 'false');
+
 /*
 	If you wish, you can define QA_MYSQL_USERS_PREFIX separately from QA_MYSQL_TABLE_PREFIX.
 	If so, tables containing information about user accounts (not including users' activity and points)

diff --git a/qa-include/db/install.php b/qa-include/db/install.php
@@ -69,6 +69,11 @@ function qa_db_table_definitions()
 	require_once QA_INCLUDE_DIR . 'db/maxima.php';
 	require_once QA_INCLUDE_DIR . 'app/users.php';
 
+	if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+		$collation = 'utf8mb4_bin';
+	else
+		$collation = 'utf_bin';
+
 	/*
 		Important note on character encoding in database and PHP connection to MySQL
 
@@ -109,7 +114,7 @@ function qa_db_table_definitions()
 			'avatarheight' => 'SMALLINT UNSIGNED', // pixel height of stored avatar
 			'passsalt' => 'BINARY(16)', // salt used to calculate passcheck - null if no password set for direct login
 			'passcheck' => 'BINARY(20)', // checksum from password and passsalt - null if no password set for direct login
-			'passhash' => 'VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL', // password_hash
+			'passhash' => 'VARCHAR(255) CHARACTER SET utf8 COLLATE '.$collation.' DEFAULT NULL', // password_hash
 			'level' => 'TINYINT UNSIGNED NOT NULL', // basic, editor, admin, etc...
 			'loggedin' => 'DATETIME NOT NULL', // time of last login
 			'loginip' => 'VARBINARY(16) NOT NULL', // INET6_ATON of IP address of last login
@@ -718,7 +723,10 @@ function qa_db_create_table_sql($rawname, $definition)
 		if (isset($coldef))
 			$querycols .= (strlen($querycols) ? ', ' : '') . (is_int($colname) ? $coldef : ($colname . ' ' . $coldef));
 
-	return 'CREATE TABLE ^' . $rawname . ' (' . $querycols . ') ENGINE=InnoDB CHARSET=utf8';
+	if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+		return 'CREATE TABLE ^' . $rawname . ' (' . $querycols . ') ENGINE=InnoDB CHARSET=utf8mb4';
+	else
+		return 'CREATE TABLE ^' . $rawname . ' (' . $querycols . ') ENGINE=InnoDB CHARSET=utf8';
 }
 
 

diff --git a/qa-include/db/selects.php b/qa-include/db/selects.php
@@ -1216,8 +1216,13 @@ function qa_db_tag_recent_qs_selectspec($voteuserid, $tag, $start, $full = false
 
 	$selectspec = qa_db_posts_basic_selectspec($voteuserid, $full);
 
+	if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+		$collation = 'utf8mb4_bin';
+	else
+		$collation = 'utf8_bin';
+
 	// use two tests here - one which can use the index, and the other which narrows it down exactly - then limit to 1 just in case
-	$selectspec['source'] .= " JOIN (SELECT postid FROM ^posttags WHERE wordid=(SELECT wordid FROM ^words WHERE word=$ AND word=$ COLLATE utf8_bin LIMIT 1) ORDER BY postcreated DESC LIMIT #,#) y ON ^posts.postid=y.postid";
+	$selectspec['source'] .= " JOIN (SELECT postid FROM ^posttags WHERE wordid=(SELECT wordid FROM ^words WHERE word=$ AND word=$ COLLATE $collation LIMIT 1) ORDER BY postcreated DESC LIMIT #,#) y ON ^posts.postid=y.postid";
 	array_push($selectspec['arguments'], $tag, qa_strtolower($tag), $start, $count);
 	$selectspec['sortdesc'] = 'created';
 
@@ -1232,9 +1237,14 @@ function qa_db_tag_recent_qs_selectspec($voteuserid, $tag, $start, $full = false
  */
 function qa_db_tag_word_selectspec($tag)
 {
+	if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+		$collation = 'utf8mb4_bin';
+	else
+		$collation = 'utf8_bin';
+
 	return array(
 		'columns' => array('wordid', 'word', 'tagcount'),
-		'source' => '^words WHERE word=$ AND word=$ COLLATE utf8_bin',
+		'source' => '^words WHERE word=$ AND word=$ COLLATE ' . $collation,
 		'arguments' => array($tag, qa_strtolower($tag)),
 		'single' => true,
 	);

diff --git a/qa-include/util/string.php b/qa-include/util/string.php
@@ -600,6 +600,9 @@ function qa_shorten_string_line($string, $length, $ellipsis = ' ... ')
  */
 function qa_remove_utf8mb4($string)
 {
+	if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+		return $string;
+
 	return preg_replace('%(?:
 		  \xF0[\x90-\xBF][\x80-\xBF]{2}  # planes 1-3
 		| [\xF1-\xF3][\x80-\xBF]{3}      # planes 4-15

diff --git a/qa-src/Database/DbConnection.php b/qa-src/Database/DbConnection.php
@@ -110,7 +110,11 @@ public function connect($failHandler = null)
 			return;
 		}
 
-		$dsn = sprintf('%s:host=%s;dbname=%s;charset=utf8', $this->config['driver'], $this->config['host'], $this->config['database']);
+		if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+			$dsn = sprintf('%s:host=%s;dbname=%s;charset=utf8mb4', $this->config['driver'], $this->config['host'], $this->config['database']);
+		else
+			$dsn = sprintf('%s:host=%s;dbname=%s;charset=utf8', $this->config['driver'], $this->config['host'], $this->config['database']);
+
 		if (isset($this->config['port'])) {
 			$dsn .= ';port=' . $this->config['port'];
 		}
@@ -129,6 +133,9 @@ public function connect($failHandler = null)
 			$this->failError('connect', $ex->getCode(), $ex->getMessage());
 		}
 
+		if (defined('QA_USE_UTF8MB4') && QA_USE_UTF8MB4)
+			$this->pdo->query('SET NAMES utf8mb4');
+
 		qa_report_process_stage('db_connected');
 	}