From f893367e00f618b8b2eddd38db2ac2b5676390b7 Mon Sep 17 00:00:00 2001 From: Tristan Knight Date: Sat, 1 Jun 2024 00:27:12 +0100 Subject: [PATCH] fix: multibyte chars in text lines (#48) --- lua/precognition/horizontal_motions.lua | 18 ++- lua/precognition/init.lua | 30 +++-- lua/precognition/utils.lua | 14 +++ tests/precognition/char_spec.lua | 10 ++ .../precognition/horizontal_motions_spec.lua | 9 ++ tests/precognition/virtline_spec.lua | 103 +++++++++++++++--- 6 files changed, 152 insertions(+), 32 deletions(-) diff --git a/lua/precognition/horizontal_motions.lua b/lua/precognition/horizontal_motions.lua index 42b23d0..abff9b7 100644 --- a/lua/precognition/horizontal_motions.lua +++ b/lua/precognition/horizontal_motions.lua @@ -27,27 +27,26 @@ end ---@param str string ---@param cursorcol integer ----@param _linelen integer +---@param linelen integer ---@param big_word boolean ---@return Precognition.PlaceLoc -function M.next_word_boundary(str, cursorcol, _linelen, big_word) +function M.next_word_boundary(str, cursorcol, linelen, big_word) local offset = cursorcol - local len = vim.fn.strcharlen(str) local char = vim.fn.strcharpart(str, offset - 1, 1) local c_class = utils.char_class(char, big_word) if c_class ~= cc.whitespace then - while utils.char_class(char, big_word) == c_class and offset <= len do + while utils.char_class(char, big_word) == c_class and offset <= linelen do offset = offset + 1 char = vim.fn.strcharpart(str, offset - 1, 1) end end - while utils.char_class(char, big_word) == cc.whitespace and offset <= len do + while utils.char_class(char, big_word) == cc.whitespace and offset <= linelen do offset = offset + 1 char = vim.fn.strcharpart(str, offset - 1, 1) end - if offset > len then + if offset > linelen then return 0 end @@ -114,11 +113,10 @@ end ---@param str string ---@param cursorcol integer ----@param _linelen integer +---@param linelen integer ---@param big_word boolean ---@return Precognition.PlaceLoc -function M.prev_word_boundary(str, cursorcol, _linelen, big_word) - local len = vim.fn.strcharlen(str) +function M.prev_word_boundary(str, cursorcol, linelen, big_word) local offset = cursorcol - 1 local char = vim.fn.strcharpart(str, offset - 1, 1) local c_class = utils.char_class(char, big_word) @@ -141,7 +139,7 @@ function M.prev_word_boundary(str, cursorcol, _linelen, big_word) end end - if offset == nil or offset > len or offset < 0 then + if offset == nil or offset > linelen or offset < 0 then return 0 end return offset + 1 diff --git a/lua/precognition/init.lua b/lua/precognition/init.lua index fa7b128..234370c 100644 --- a/lua/precognition/init.lua +++ b/lua/precognition/init.lua @@ -54,6 +54,10 @@ local M = {} ---@field PrevParagraph Precognition.PlaceLoc ---@field NextParagraph Precognition.PlaceLoc +---@class Precognition.ExtraPadding +---@field start integer +---@field length integer + ---@type Precognition.HintConfig local defaultHintConfig = { Caret = { text = "^", prio = 2 }, @@ -106,8 +110,9 @@ local gutter_group = "precognition_gutter" ---@param marks Precognition.VirtLine ---@param line_len integer +---@param extra_padding Precognition.ExtraPadding ---@return table -local function build_virt_line(marks, line_len) +local function build_virt_line(marks, line_len, extra_padding) if not marks then return {} end @@ -127,20 +132,26 @@ local function build_virt_line(marks, line_len) if existing == " " and existing ~= hint then line_table[col] = hint else -- if the character is not a space, then we need to check the prio - local existingKey + local existing_key for key, value in pairs(config.hints) do if value.text == existing then - existingKey = key + existing_key = key break end end - if existing ~= " " and config.hints[mark].prio > config.hints[existingKey].prio then + if existing ~= " " and config.hints[mark].prio > config.hints[existing_key].prio then line_table[col] = hint end end end end + if #extra_padding > 0 then + for _, padding in ipairs(extra_padding) do + line_table[padding.start] = line_table[padding.start] .. string.rep(" ", padding.length) + end + end + local line = table.concat(line_table) if line:match("^%s+$") then return {} @@ -201,8 +212,8 @@ local function display_marks() if utils.is_blacklisted_buffer(bufnr) then return end - local cursorline, cursorcol = unpack(vim.api.nvim_win_get_cursor(0)) - cursorcol = cursorcol + 1 + local cursorline = vim.fn.line(".") + local cursorcol = vim.fn.charcol(".") if extmark and not dirty then return end @@ -210,6 +221,8 @@ local function display_marks() local tab_width = vim.bo.expandtab and vim.bo.shiftwidth or vim.bo.tabstop local cur_line = vim.api.nvim_get_current_line():gsub("\t", string.rep(" ", tab_width)) local line_len = vim.fn.strcharlen(cur_line) + ---@type Precognition.ExtraPadding[] + local extra_padding = {} -- local after_cursor = vim.fn.strcharpart(cur_line, cursorcol + 1) -- local before_cursor = vim.fn.strcharpart(cur_line, 0, cursorcol - 1) -- local before_cursor_rev = string.reverse(before_cursor) @@ -232,7 +245,10 @@ local function display_marks() Zero = 1, } - local virt_line = build_virt_line(virtual_line_marks, line_len) + --multicharacter padding + utils.add_multibyte_padding(cur_line, extra_padding, line_len) + + local virt_line = build_virt_line(virtual_line_marks, line_len, extra_padding) -- TODO: can we add indent lines to the virt line to match indent-blankline or similar (if installed)? diff --git a/lua/precognition/utils.lua b/lua/precognition/utils.lua index b1c2802..8e1ad53 100644 --- a/lua/precognition/utils.lua +++ b/lua/precognition/utils.lua @@ -49,4 +49,18 @@ function M.create_pad_array(len, str) return pad_array end +---Add extra padding for multi byte character characters +---@param cur_line string +---@param extra_padding Precognition.ExtraPadding[] +---@param line_len integer +function M.add_multibyte_padding(cur_line, extra_padding, line_len) + for i = 1, line_len do + local char = vim.fn.strcharpart(cur_line, i - 1, 1) + local width = vim.fn.strdisplaywidth(char) + if width > 1 then + table.insert(extra_padding, { start = i, length = width - 1 }) + end + end +end + return M diff --git a/tests/precognition/char_spec.lua b/tests/precognition/char_spec.lua index e9041e7..2d1530b 100644 --- a/tests/precognition/char_spec.lua +++ b/tests/precognition/char_spec.lua @@ -51,6 +51,16 @@ describe("big_word classing", function() eq(utils.char_class("@", true), 1) eq(utils.char_class(".", true), 1) end) + + it("can class emoji characters", function() + eq(utils.char_class("🐱", false), 1) + eq(utils.char_class("😸", false), 1) + end) + + it("can class nerdfont characters", function() + eq(utils.char_class("", false), 1) + eq(utils.char_class("", false), 1) + end) end) describe("pad arrays", function() diff --git a/tests/precognition/horizontal_motions_spec.lua b/tests/precognition/horizontal_motions_spec.lua index 5500d5d..dfc28b9 100644 --- a/tests/precognition/horizontal_motions_spec.lua +++ b/tests/precognition/horizontal_motions_spec.lua @@ -304,4 +304,13 @@ describe("edge case", function() eq(16, hm.end_of_word(str, 15, #str, false)) eq(22, hm.end_of_word(str, 21, #str, false)) end) + + it("multibyte characters", function() + local str = "# 💭👀precognition.nvim" + local len = vim.fn.strcharlen(str) + eq(3, hm.next_word_boundary(str, 1, len, false)) + eq(17, hm.prev_word_boundary(str, 18, len, false)) + eq(3, hm.prev_word_boundary(str, 18, len, true)) + eq(3, hm.prev_word_boundary(str, 5, len, false)) + end) end) diff --git a/tests/precognition/virtline_spec.lua b/tests/precognition/virtline_spec.lua index 0bbf0cd..1a2a952 100644 --- a/tests/precognition/virtline_spec.lua +++ b/tests/precognition/virtline_spec.lua @@ -1,5 +1,6 @@ local precognition = require("precognition") local hm = require("precognition.horizontal_motions") +local utils = require("precognition.utils") ---@diagnostic disable-next-line: undefined-field local eq = assert.are.same describe("Build Virtual Line", function() @@ -8,7 +9,7 @@ describe("Build Virtual Line", function() Caret = 4, Dollar = 10, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq(" ^ $", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -17,7 +18,7 @@ describe("Build Virtual Line", function() local marks = { Caret = 4, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq(" ^ ", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -26,7 +27,7 @@ describe("Build Virtual Line", function() local marks = { Dollar = 10, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq(" $", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -35,7 +36,7 @@ describe("Build Virtual Line", function() local marks = { Caret = 1, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq("^ ", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -50,7 +51,7 @@ describe("Build Virtual Line", function() w = 10, Dollar = 50, } - local virtual_line = precognition.build_virt_line(marks, 50) + local virtual_line = precognition.build_virt_line(marks, 50, {}) local line_num = 1 for char in virtual_line[1][1]:gmatch(".") do if line_num == 1 then @@ -84,7 +85,7 @@ describe("Build Virtual Line", function() b = hm.prev_word_boundary(cur_line, cursorcol, line_len, false), Caret = hm.line_start_non_whitespace(cur_line, cursorcol, line_len), Dollar = hm.line_end(cur_line, cursorcol, line_len), - }, line_len) + }, line_len, {}) eq("b e w $", virt_line[1][1]) eq(#line, #virt_line[1][1]) @@ -104,11 +105,83 @@ describe("Build Virtual Line", function() b = hm.prev_word_boundary(cur_line, cursorcol, line_len, false), Caret = hm.line_start_non_whitespace(cur_line, cursorcol, line_len), Dollar = hm.line_end(cur_line, cursorcol, line_len), - }, line_len) + }, line_len, {}) eq(" ^ e w $", virt_line[1][1]) eq(#line, #virt_line[1][1]) end) + + it("can build a line with extra padding", function() + local line = " abc def" + local cursorcol = 5 + local tab_width = vim.bo.expandtab and vim.bo.shiftwidth or vim.bo.tabstop + local cur_line = line:gsub("\t", string.rep(" ", tab_width)) + local line_len = vim.fn.strcharlen(cur_line) + local extra_padding = { { start = 4, length = 4 } } + + local virt_line = precognition.build_virt_line({ + w = hm.next_word_boundary(cur_line, cursorcol, line_len, false), + e = hm.end_of_word(cur_line, cursorcol, line_len, false), + b = hm.prev_word_boundary(cur_line, cursorcol, line_len, false), + Caret = hm.line_start_non_whitespace(cur_line, cursorcol, line_len), + Dollar = hm.line_end(cur_line, cursorcol, line_len), + }, line_len, extra_padding) + + local total_added = 0 + for _, pad in ipairs(extra_padding) do + total_added = total_added + pad.length + end + + eq(" ^ e w $", virt_line[1][1]) + eq(#line + total_added, #virt_line[1][1]) + end) + + it("can build a line with multiple extra padddings", function() + local line = " abc def" + local cursorcol = 5 + local tab_width = vim.bo.expandtab and vim.bo.shiftwidth or vim.bo.tabstop + local cur_line = line:gsub("\t", string.rep(" ", tab_width)) + local line_len = vim.fn.strcharlen(cur_line) + local extra_padding = { { start = 4, length = 4 }, { start = 10, length = 5 } } + + local virt_line = precognition.build_virt_line({ + w = hm.next_word_boundary(cur_line, cursorcol, line_len, false), + e = hm.end_of_word(cur_line, cursorcol, line_len, false), + b = hm.prev_word_boundary(cur_line, cursorcol, line_len, false), + Caret = hm.line_start_non_whitespace(cur_line, cursorcol, line_len), + Dollar = hm.line_end(cur_line, cursorcol, line_len), + }, line_len, extra_padding) + + local total_added = 0 + for _, pad in ipairs(extra_padding) do + total_added = total_added + pad.length + end + + eq(" ^ e w $", virt_line[1][1]) + eq(#line + total_added, #virt_line[1][1]) + end) + + it("example virtual line with emoji", function() + local line = "# 💭👀precognition.nvim" + local cursorcol = 20 + local tab_width = vim.bo.expandtab and vim.bo.shiftwidth or vim.bo.tabstop + local cur_line = line:gsub("\t", string.rep(" ", tab_width)) + local line_len = vim.fn.strcharlen(cur_line) + local extra_padding = {} + + utils.add_multibyte_padding(cur_line, extra_padding, line_len) + + local virt_line = precognition.build_virt_line({ + w = hm.next_word_boundary(cur_line, cursorcol, line_len, false), + e = hm.end_of_word(cur_line, cursorcol, line_len, false), + b = hm.prev_word_boundary(cur_line, cursorcol, line_len, false), + Caret = hm.line_start_non_whitespace(cur_line, cursorcol, line_len), + Dollar = hm.line_end(cur_line, cursorcol, line_len), + }, line_len, extra_padding) + + eq("^ b e", virt_line[1][1]) + eq(vim.fn.strdisplaywidth(line), #virt_line[1][1]) + end) end) describe("Priority", function() @@ -133,7 +206,7 @@ describe("Priority", function() Dollar = 10, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq(" w ", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -159,7 +232,7 @@ describe("Priority", function() Dollar = 10, } - local virtual_line = precognition.build_virt_line(marks, 10) + local virtual_line = precognition.build_virt_line(marks, 10, {}) eq(" w $", virtual_line[1][1]) eq(10, #virtual_line[1][1]) end) @@ -184,7 +257,7 @@ describe("Priority", function() Dollar = 1, } - local virtual_line = precognition.build_virt_line(marks, 1) + local virtual_line = precognition.build_virt_line(marks, 1, {}) eq("$", virtual_line[1][1]) eq(1, #virtual_line[1][1]) @@ -202,7 +275,7 @@ describe("Priority", function() }, }) - virtual_line = precognition.build_virt_line(marks, 1) + virtual_line = precognition.build_virt_line(marks, 1, {}) eq("^", virtual_line[1][1]) eq(1, #virtual_line[1][1]) end) @@ -224,7 +297,7 @@ describe("replacment charcters", function() Caret = 1, } - local virtual_line = precognition.build_virt_line(marks, 1) + local virtual_line = precognition.build_virt_line(marks, 1, {}) eq("x", virtual_line[1][1]) eq(1, #virtual_line[1][1]) end) @@ -244,7 +317,7 @@ describe("replacment charcters", function() Caret = 1, } - local virtual_line = precognition.build_virt_line(marks, 1) + local virtual_line = precognition.build_virt_line(marks, 1, {}) eq("â", virtual_line[1][1]) eq(2, #virtual_line[1][1]) end) @@ -261,7 +334,7 @@ describe("replacment charcters", function() Dollar = 8, } - local virtual_line = precognition.build_virt_line(marks, 8) + local virtual_line = precognition.build_virt_line(marks, 8, {}) eq("0^e w $", virtual_line[1][1]) end) @@ -286,7 +359,7 @@ describe("replacment charcters", function() Dollar = 8, } - local virtual_line = precognition.build_virt_line(marks, 8) + local virtual_line = precognition.build_virt_line(marks, 8, {}) eq("0âe w $", virtual_line[1][1]) end) end)