UTF-8 characters break diagnostics' underline position

[paulkre]

FAQ

• I have checked the FAQ and it didn't resolve my problem.

Issues

• I have checked existing issues and there are no issues with the same problem.

Neovim Version

0.9.1

Dev Version?

• I am using a stable Neovim release version, or if I am using a dev version of Neovim I have confirmed that my issue is reproducible on a stable version.

Operating System

macOS

Minimal Config

-- this template is borrowed from nvim-lspconfig
local on_windows = vim.loop.os_uname().version:match("Windows")

local function join_paths(...)
    local path_sep = on_windows and "\\" or "/"
    local result = table.concat({ ... }, path_sep)
    return result
end

vim.g.loaded_remote_plugins = ""
vim.cmd([[set runtimepath=$VIMRUNTIME]])

local temp_dir = vim.loop.os_getenv("TEMP") or "/tmp"

vim.cmd("set packpath=" .. join_paths(temp_dir, "nvim", "site"))

local package_root = join_paths(temp_dir, "nvim", "site", "pack")
local install_path = join_paths(package_root, "packer", "start", "packer.nvim")
local compile_path = join_paths(install_path, "plugin", "packer_compiled.lua")

local null_ls_config = function()
    local null_ls = require("null-ls")
    -- add only what you need to reproduce your issue
    null_ls.setup({
        sources = {
	    null_ls.builtins.diagnostics.ltrs,
	},
        debug = true,
    })
end

local function load_plugins()
    -- only add other plugins if they are necessary to reproduce the issue
    require("packer").startup({
        {
            "wbthomason/packer.nvim",
            {
                "jose-elias-alvarez/null-ls.nvim",
                requires = { "nvim-lua/plenary.nvim" },
                config = null_ls_config,
            },
        },
        config = {
            package_root = package_root,
            compile_path = compile_path,
        },
    })
end

if vim.fn.isdirectory(install_path) == 0 then
    vim.fn.system({ "git", "clone", "https://github.com/wbthomason/packer.nvim", install_path })
    load_plugins()
    require("packer").sync()
else
    load_plugins()
    require("packer").sync()
end

Steps to Reproduce

Create a txt file with the following content:

UTF-8 characters (e.g. "§¢€£") cause problems: asdf

ltrs is configured and will report an error for the unknown word "asdf". The output of the ltrs command also contains the correct information about the position of the spelling error ("asdf" starts at offset 47). Unfortunately, some internals of null-ls or neovim do not seem to handle UTF-8 content correctly, because this is how the error is being highlighted in the editor:

The line under "asdf" starts 5 characters too early.

Does anyone know where the problem is coming from or how to solve this?

Reproducibility Check

• I confirm that my minimal config is based on the minimal_init.lua template and that my issue is reproducible by running nvim --clean -u minimal_init.lua and following the steps above.

Expected Behavior

The erroneous letters ("asdf") should be underlined correctly.

Actual Behavior

The error highlighting is shifted to the left.

Debug Log

[TRACE Tue Jul 25 13:30:45 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method textDocument/didChange
[TRACE Tue Jul 25 13:30:45 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/generators.lua:21: running generators for method NULL_LS_DIAGNOSTICS
[DEBUG Tue Jul 25 13:30:45 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:321: spawning command "ltrs" at /Users/paulkre/dev/notes with args { "check", "-m", "-r", "--text", 'UTF-8 characters (e.g. "§¢€£") cause problems: asdf\n\n' }
[TRACE Tue Jul 25 13:30:47 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method textDocument/didSave
[TRACE Tue Jul 25 13:30:47 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/generators.lua:21: running generators for method NULL_LS_DIAGNOSTICS_ON_SAVE
[DEBUG Tue Jul 25 13:30:47 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/generators.lua:24: no generators available
[TRACE Tue Jul 25 13:30:48 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:205: error output: nil
[TRACE Tue Jul 25 13:30:48 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:206: output: {
  "language": {
    "code": "en-US",
    "detectedLanguage": {
      "code": "en-US",
      "confidence": 0.99,
      "name": "English (US)",
      "source": "fasttext+commonwords"
    },
    "name": "English (US)"
  },
  "matches": [
    {
      "context": {
        "length": 4,
        "offset": 43,
        "text": "...haracters (e.g. \"§¢€£\") cause problems: asdf  "
      },
      "contextForSureMatch": 0,
      "ignoreForIncompleteSentence": false,
      "length": 4,
      "message": "Possible spelling mistake found.",
      "moreContext": {
        "line_number": 1,
        "line_offset": 47
      },
      "offset": 47,
      "replacements": [
        {
          "value": "ASD"
        },
        {
          "value": "ADF"
        },
        {
          "value": "ASF"
        },
        {
          "value": "JASDF"
        },
        {
          "value": "SDF"
        },
        {
          "value": "... (2 not shown)"
        }
      ],
      "rule": {
        "category": {
          "id": "TYPOS",
          "name": "Possible Typo"
        },
        "description": "Possible spelling mistake",
        "id": "MORFOLOGIK_RULE_EN_US",
        "isPremium": false,
        "issueType": "misspelling",
        "sourceFile": null,
        "subId": null,
        "urls": null
      },
      "sentence": "UTF-8 characters (e.g. \"§¢€£\") cause problems: asdf",
      "shortMessage": "Spelling mistake",
      "type": {
        "typeName": "UnknownWord"
      }
    }
  ],
  "sentenceRanges": [
    [
      0,
      51
    ]
  ],
  "software": {
    "apiVersion": 1,
    "buildDate": "2023-07-19 13:27:09 +0000",
    "name": "LanguageTool",
    "premium": true,
    "premiumHint": "You might be missing errors only the Premium version can find. Contact us at support<at>languagetoolplus.com.",
    "status": "",
    "version": "6.3-SNAPSHOT"
  },
  "warnings": {
    "incompleteResults": false
  }
}

[TRACE Tue Jul 25 13:30:48 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/diagnostics.lua:181: received diagnostics from source 1
[TRACE Tue Jul 25 13:30:48 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/diagnostics.lua:182: { {
    code = "MORFOLOGIK_RULE_EN_US",
    col = 47,
    end_col = 51,
    end_lnum = 0,
    end_row = 1,
    lnum = 0,
    message = "Possible spelling mistake found. Try: “ASD”, “ADF”, “ASF”, “JASDF”, “SDF”, “... (2 not shown)”",
    row = 1,
    severity = 1,
    source = "ltrs"
  } }
[TRACE Tue Jul 25 13:30:49 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method textDocument/didSave
[DEBUG Tue Jul 25 13:30:49 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/diagnostics.lua:164: buffer unchanged; ignoring didSave notification
[TRACE Tue Jul 25 13:30:49 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:106: received LSP request for method shutdown
[TRACE Tue Jul 25 13:30:49 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method exit
[TRACE Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/client.lua:97: starting null-ls client
[TRACE Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:106: received LSP request for method initialize
[DEBUG Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/client.lua:152: unable to notify client for method textDocument/didOpen (client not active): {
  textDocument = {
    uri = "file:///Users/paulkre/dev/notes/test.txt"
  }
}
[TRACE Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method initialized
[TRACE Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/rpc.lua:131: received LSP notification for method textDocument/didOpen
[TRACE Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/generators.lua:21: running generators for method NULL_LS_DIAGNOSTICS_ON_OPEN
[DEBUG Tue Jul 25 13:30:51 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:321: spawning command "ltrs" at /Users/paulkre/dev/notes with args { "check", "-m", "-r", "--text", 'UTF-8 characters (e.g. "§¢€£") cause problems: asdf\n\n' }
[TRACE Tue Jul 25 13:30:52 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:205: error output: nil
[TRACE Tue Jul 25 13:30:52 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/helpers/generator_factory.lua:206: output: {
  "language": {
    "code": "en-US",
    "detectedLanguage": {
      "code": "en-US",
      "confidence": 0.99,
      "name": "English (US)",
      "source": "fasttext+commonwords"
    },
    "name": "English (US)"
  },
  "matches": [
    {
      "context": {
        "length": 4,
        "offset": 43,
        "text": "...haracters (e.g. \"§¢€£\") cause problems: asdf  "
      },
      "contextForSureMatch": 0,
      "ignoreForIncompleteSentence": false,
      "length": 4,
      "message": "Possible spelling mistake found.",
      "moreContext": {
        "line_number": 1,
        "line_offset": 47
      },
      "offset": 47,
      "replacements": [
        {
          "value": "ASD"
        },
        {
          "value": "ADF"
        },
        {
          "value": "ASF"
        },
        {
          "value": "JASDF"
        },
        {
          "value": "SDF"
        },
        {
          "value": "... (2 not shown)"
        }
      ],
      "rule": {
        "category": {
          "id": "TYPOS",
          "name": "Possible Typo"
        },
        "description": "Possible spelling mistake",
        "id": "MORFOLOGIK_RULE_EN_US",
        "isPremium": false,
        "issueType": "misspelling",
        "sourceFile": null,
        "subId": null,
        "urls": null
      },
      "sentence": "UTF-8 characters (e.g. \"§¢€£\") cause problems: asdf",
      "shortMessage": "Spelling mistake",
      "type": {
        "typeName": "UnknownWord"
      }
    }
  ],
  "sentenceRanges": [
    [
      0,
      51
    ]
  ],
  "software": {
    "apiVersion": 1,
    "buildDate": "2023-07-19 13:27:09 +0000",
    "name": "LanguageTool",
    "premium": true,
    "premiumHint": "You might be missing errors only the Premium version can find. Contact us at support<at>languagetoolplus.com.",
    "status": "",
    "version": "6.3-SNAPSHOT"
  },
  "warnings": {
    "incompleteResults": false
  }
}

[TRACE Tue Jul 25 13:30:52 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/diagnostics.lua:181: received diagnostics from source 1
[TRACE Tue Jul 25 13:30:52 2023] /tmp/nvim/site/pack/packer/start/null-ls.nvim/lua/null-ls/diagnostics.lua:182: { {
    code = "MORFOLOGIK_RULE_EN_US",
    col = 47,
    end_col = 51,
    end_lnum = 0,
    end_row = 1,
    lnum = 0,
    message = "Possible spelling mistake found. Try: “ASD”, “ADF”, “ASF”, “JASDF”, “SDF”, “... (2 not shown)”",
    row = 1,
    severity = 1,
    source = "ltrs"
  } }

Help

Yes, but I don't know how to start. I would need guidance

Implementation Help

No response

Requirements

• I have read and followed the instructions above and understand that my issue will be closed if I did not provide the required information.