refactored the take_partial function and use String module functions to handle complex unicode characters/emojis

Author: zookzook

lib/delta.ex

@@ -234,7 +234,7 @@ defmodule Delta do
 
       iex> delta = [Op.insert("01🙋45")]
       iex> Delta.slice(delta, 1, 2)
-      ** (RuntimeError) Encoding failed in take_partial {"1🙋45", %{"insert" => "1🙋45"}, 2, {:incomplete, "1", <<216, 61>>}, {:error, "", <<222, 75, 0, 52, 0, 53>>}}
+      [%{"insert" => "1🙋"}]
   """
   @spec slice(t, non_neg_integer, non_neg_integer) :: t
   def slice(delta, index, len) do
@@ -254,43 +254,38 @@ defmodule Delta do
 
       iex> delta = [Op.insert("01🙋45")]
       iex> Delta.slice_max(delta, 1, 2)
-      [%{"insert" => "1"}]
+      [%{"insert" => "1🙋"}]
   """
   @doc since: "0.2.0"
   @spec slice_max(t, non_neg_integer, non_neg_integer) :: t
   def slice_max(delta, index, len) do
-    {_left, right} = split(delta, index, align: true)
-    {middle, _rest} = split(right, len, align: true)
+    {_left, right} = split(delta, index)
+    {middle, _rest} = split(right, len)
     middle
   end
 
   @doc ~S"""
   Splits delta at the given index.
 
-  ## Options
-
-    * `:align` - when `true`, allow moving index left if
-      we're likely to split a grapheme otherwise.
-
   ## Examples
       iex> delta = [Op.insert("Hello World")]
       iex> Delta.split(delta, 5)
       {[%{"insert" => "Hello"}], [%{"insert" => " World"}]}
 
       iex> delta = [Op.insert("01🙋45")]
       iex> Delta.split(delta, 3, align: true)
-      {[%{"insert" => "01"}], [%{"insert" => "🙋45"}]}
+      {[%{"insert" => "01🙋"}], [%{"insert" => "45"}]}
 
       iex> delta = [Op.insert("a"), Op.insert("b", %{"bold" => true})]
       iex> Delta.split(delta, 3)
       {[%{"insert" => "a"}, %{"insert" => "b", "attributes" => %{"bold" => true}}], []}
   """
-  @spec split(t, non_neg_integer | fun, Keyword.t()) :: {t, t}
-  def split(delta, index, opts \\ [])
+  @spec split(t, non_neg_integer | fun) :: {t, t}
+  def split(delta, index)
 
-  def split(delta, 0, _), do: {[], delta}
+  def split(delta, 0), do: {[], delta}
 
-  def split(delta, index, opts) when is_integer(index) do
+  def split(delta, index) when is_integer(index) do
     do_split(
       [],
       delta,
@@ -303,36 +298,35 @@ defmodule Delta do
           {:cont, index - op_size}
         end
       end,
-      index,
-      opts
+      index
     )
   end
 
-  def split(delta, func, opts) when is_function(func) do
-    do_split([], delta, func, nil, opts)
+  def split(delta, func) when is_function(func) do
+    do_split([], delta, func, nil)
   end
 
-  defp do_split(passed, [], _, _, _), do: {Enum.reverse(passed), []}
+  defp do_split(passed, [], _, _), do: {Enum.reverse(passed), []}
 
-  defp do_split(passed, remaining, func, context, opts) when is_function(func, 1) do
-    do_split(passed, remaining, fn op, _ -> func.(op) end, context, opts)
+  defp do_split(passed, remaining, func, context) when is_function(func, 1) do
+    do_split(passed, remaining, fn op, _ -> func.(op) end, context)
   end
 
-  defp do_split(passed, remaining, func, context, opts) when is_function(func, 2) do
+  defp do_split(passed, remaining, func, context) when is_function(func, 2) do
     [first | remaining] = remaining
 
     case func.(first, context) do
       :cont ->
-        do_split([first | passed], remaining, func, context, opts)
+        do_split([first | passed], remaining, func, context)
 
       {:cont, context} ->
-        do_split([first | passed], remaining, func, context, opts)
+        do_split([first | passed], remaining, func, context)
 
       0 ->
         {Enum.reverse(passed), [first | remaining]}
 
       index ->
-        case Op.take(first, index, opts) do
+        case Op.take(first, index) do
           {left, false} ->
             {Enum.reverse([left | passed]), remaining}
 

lib/delta/op.ex

@@ -1,6 +1,5 @@
 defmodule Delta.Op do
   alias Delta.Attr
-  alias Delta.Utils
   alias Delta.EmbedHandler
 
   @type t :: insert_op | retain_op | delete_op
@@ -171,14 +170,11 @@ defmodule Delta.Op do
       5
 
       iex> Op.text_size("🏴󠁧󠁢󠁳󠁣󠁴󠁿")
-      14
+      1
   """
   @spec text_size(text :: binary) :: non_neg_integer
   def text_size(text) do
-    text
-    |> :unicode.characters_to_binary(:utf8, :utf16)
-    |> byte_size()
-    |> div(2)
+    String.length(text)
   end
 
   @doc ~S"""
@@ -210,22 +206,20 @@ defmodule Delta.Op do
       iex> Op.insert("Hello") |> Op.take(3)
       {%{"insert" => "Hel"}, %{"insert" => "lo"}}
 
-      iex> assert_raise RuntimeError, fn -> Op.insert("🏴󠁧󠁢󠁳󠁣󠁴󠁿") |> Op.take(1) end
-
-      iex> Op.insert("🏴󠁧󠁢󠁳󠁣󠁴󠁿") |> Op.take(1, align: true)
-      {%{"insert" => ""}, %{"insert" => "🏴󠁧󠁢󠁳󠁣󠁴󠁿"}}
+      iex> Op.insert("🏴󠁧󠁢󠁳󠁣󠁴󠁿") |> Op.take(1)
+      {%{"insert" => "🏴󠁧󠁢󠁳󠁣󠁴󠁿"}, false}
   """
-  @spec take(op :: t, length :: non_neg_integer, opts :: Keyword.t()) :: {t, t | boolean}
-  def take(op, length, opts \\ [])
+  @spec take(op :: t, length :: non_neg_integer) :: {t, t | boolean}
+  def take(op, length)
 
-  def take(op = %{"insert" => embed}, _length, _opts) when not is_bitstring(embed) do
+  def take(op = %{"insert" => embed}, _length) when not is_bitstring(embed) do
     {op, false}
   end
 
-  def take(op, length, opts) do
+  def take(op, length) do
     case size(op) - length do
       0 -> {op, false}
-      _ -> take_partial(op, length, opts)
+      _ -> take_partial(op, length)
     end
   end
 
@@ -350,47 +344,21 @@ defmodule Delta.Op do
     {op1, a, op2, b}
   end
 
-  @spec take_partial(t, non_neg_integer, Keyword.t()) :: {t, t}
-  defp take_partial(op, 0, _opts), do: {insert("", op["attributes"]), op}
-
-  defp take_partial(%{"insert" => text} = op, len, opts) do
-    binary = :unicode.characters_to_binary(text, :utf8, :utf16)
-    binary_length = byte_size(binary)
-
-    left =
-      binary
-      |> Kernel.binary_part(0, len * 2)
-      |> :unicode.characters_to_binary(:utf16, :utf8)
-
-    right =
-      binary
-      |> Kernel.binary_part(len * 2, binary_length - len * 2)
-      |> :unicode.characters_to_binary(:utf16, :utf8)
+  @spec take_partial(t, non_neg_integer) :: {t, t}
+  defp take_partial(op, 0), do: {insert("", op["attributes"]), op}
 
-    case {is_binary(left), is_binary(right), Keyword.get(opts, :align, false)} do
-      {true, true, false} ->
-        {insert(left, op["attributes"]), insert(right, op["attributes"])}
-
-      {true, true, true} ->
-        if Utils.slices_likely_cut_emoji?(left, right) do
-          take_partial(op, len - 1, opts)
-        else
-          {insert(left, op["attributes"]), insert(right, op["attributes"])}
-        end
-
-      {_, _, true} ->
-        take_partial(op, len - 1, opts)
-
-      _ ->
-        raise "Encoding failed in take_partial #{inspect({text, op, len, left, right})}"
-    end
+  defp take_partial(%{"insert" => text} = op, len) do
+    length = String.length(text)
+    left = String.slice(text, 0, len)
+    right = String.slice(text, len, length - len)
+    {insert(left, op["attributes"]), insert(right, op["attributes"])}
   end
 
-  defp take_partial(%{"delete" => full} = op, length, _opts) do
+  defp take_partial(%{"delete" => full} = op, length) do
     {delete(length, op["attributes"]), delete(full - length, op["attributes"])}
   end
 
-  defp take_partial(%{"retain" => full} = op, length, _opts) do
+  defp take_partial(%{"retain" => full} = op, length) do
     {retain(length, op["attributes"]), retain(full - length, op["attributes"])}
   end
 

test/delta/delta/compose_test.exs

@@ -2,6 +2,15 @@ defmodule Tests.Delta.Compose do
   use Delta.Support.Case, async: false
   doctest Delta, only: [compose: 2, compose_all: 1]
 
+  describe ".compose/2 (utf-16/32)" do
+    test "take_partial with two 16-bit code units" do
+      a = [Op.insert("Take the 💊💊")]
+      b = [Op.retain(10)]
+
+      assert [%{"insert" => "Take the 💊💊"}] == Delta.compose(a, b)
+    end
+  end
+
   describe ".compose/2 (basic)" do
     test "insert + insert" do
       a = [Op.insert("A")]

test/delta/delta_test.exs

@@ -56,17 +56,17 @@ defmodule Tests.Delta do
 
     test "slice normal emoji" do
       delta = [%{"insert" => "01🙋45"}]
-      assert Delta.slice(delta, 1, 4) == [%{"insert" => "1🙋4"}]
+      assert Delta.slice(delta, 1, 4) == [%{"insert" => "1🙋45"}]
     end
 
     test "slice emoji with zero width joiner" do
       delta = [%{"insert" => "01🙋‍♂️78"}]
-      assert Delta.slice(delta, 1, 7) == [%{"insert" => "1🙋‍♂️7"}]
+      assert Delta.slice(delta, 1, 7) == [%{"insert" => "1🙋‍♂️78"}]
     end
 
     test "slice emoji with joiner and modifer" do
       delta = [%{"insert" => "01🙋🏽‍♂️90"}]
-      assert Delta.slice(delta, 1, 9) == [%{"insert" => "1🙋🏽‍♂️9"}]
+      assert Delta.slice(delta, 1, 9) == [%{"insert" => "1🙋🏽‍♂️90"}]
     end
 
     test "slice with 0 index" do
@@ -121,17 +121,17 @@ defmodule Tests.Delta do
 
     test "slice normal emoji" do
       delta = [%{"insert" => "01🙋45"}]
-      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🙋4"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🙋45"}]
     end
 
     test "slice emoji with zero width joiner" do
       delta = [%{"insert" => "01🙋‍♂️78"}]
-      assert Delta.slice_max(delta, 1, 7) == [%{"insert" => "1🙋‍♂️7"}]
+      assert Delta.slice_max(delta, 1, 7) == [%{"insert" => "1🙋‍♂️78"}]
     end
 
     test "slice emoji with joiner and modifer" do
       delta = [%{"insert" => "01🙋🏽‍♂️90"}]
-      assert Delta.slice_max(delta, 1, 9) == [%{"insert" => "1🙋🏽‍♂️9"}]
+      assert Delta.slice_max(delta, 1, 9) == [%{"insert" => "1🙋🏽‍♂️90"}]
     end
 
     test "slice with 0 index" do
@@ -147,39 +147,36 @@ defmodule Tests.Delta do
     test "slice emoji: codepoint + variation selector" do
       # "01☹️345"
       delta = [%{"insert" => "01\u2639\uFE0F345"}]
-      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1\u2639\uFE0F"}]
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1☹️"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1☹️3"}]
     end
 
     test "slice emoji: codepoint + skin tone modifier" do
       # "01🤵🏽345"
       delta = [%{"insert" => "01\u{1F935}\u{1F3FD}345"}]
-      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1\u{1F935}\u{1F3FD}"}]
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1🤵🏽"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1🤵🏽3"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🤵🏽34"}]
+      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1🤵🏽345"}]
     end
 
     test "slice emoji: codepoint + ZWJ + codepoint" do
       # "01👨‍🏭345"
       delta = [%{"insert" => "01\u{1F468}\u200D\u{1F3ED}345"}]
-      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1"}]
-      assert Delta.slice_max(delta, 1, 6) == [%{"insert" => "1\u{1F468}\u200D\u{1F3ED}"}]
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1👨‍🏭"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1👨‍🏭3"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1👨‍🏭34"}]
+      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1👨‍🏭345"}]
+      assert Delta.slice_max(delta, 1, 6) == [%{"insert" => "1👨‍🏭345"}]
     end
 
     test "slice emoji: flags" do
       # "01🇦🇺345"
       delta = [%{"insert" => "01\u{1F1E6}\u{1F1FA}345"}]
-      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1"}]
-      # "1🇦"
-      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1\u{1F1E6}"}]
-      # "1🇦"
-      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1\u{1F1E6}"}]
-      # "1🇦🇺"
-      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1\u{1F1E6}\u{1F1FA}"}]
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1🇦🇺"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1🇦🇺3"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🇦🇺34"}]
+      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1🇦🇺345"}]
     end
 
     test "slice emoji: tag sequence" do
@@ -188,25 +185,25 @@ defmodule Tests.Delta do
         %{"insert" => "01\u{1F3F4}\u{E0067}\u{E0062}\u{E0073}\u{E0063}\u{E0074}\u{E007F}345"}
       ]
 
-      for len <- 2..14 do
-        assert Delta.slice_max(delta, 1, len) == [%{"insert" => "1"}]
-      end
-
-      assert Delta.slice_max(delta, 1, 15) == [
-               %{"insert" => "1\u{1F3F4}\u{E0067}\u{E0062}\u{E0073}\u{E0063}\u{E0074}\u{E007F}"}
-             ]
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1🏴󠁧󠁢󠁳󠁣󠁴󠁿"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1🏴󠁧󠁢󠁳󠁣󠁴󠁿3"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🏴󠁧󠁢󠁳󠁣󠁴󠁿34"}]
+      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1🏴󠁧󠁢󠁳󠁣󠁴󠁿345"}]
+      assert Delta.slice_max(delta, 1, 6) == [%{"insert" => "1🏴󠁧󠁢󠁳󠁣󠁴󠁿345"}]
     end
 
     test "slice complex emoji" do
       # "01🚵🏻‍♀️345"
       delta = [%{"insert" => "01\u{1F6B5}\u{1F3FB}\u{200D}\u{2640}\u{FE0F}345"}]
-
-      for len <- 2..7 do
-        assert Delta.slice_max(delta, 1, len) == [%{"insert" => "1"}]
-      end
+      assert Delta.slice_max(delta, 1, 2) == [%{"insert" => "1🚵🏻‍♀️"}]
+      assert Delta.slice_max(delta, 1, 3) == [%{"insert" => "1🚵🏻‍♀️3"}]
+      assert Delta.slice_max(delta, 1, 4) == [%{"insert" => "1🚵🏻‍♀️34"}]
+      assert Delta.slice_max(delta, 1, 5) == [%{"insert" => "1🚵🏻‍♀️345"}]
+      assert Delta.slice_max(delta, 1, 6) == [%{"insert" => "1🚵🏻‍♀️345"}]
+      assert Delta.slice_max(delta, 1, 7) == [%{"insert" => "1🚵🏻‍♀️345"}]
 
       assert Delta.slice_max(delta, 1, 8) == [
-               %{"insert" => "1\u{1F6B5}\u{1F3FB}\u{200D}\u{2640}\u{FE0F}"}
+               %{"insert" => "1\u{1F6B5}\u{1F3FB}\u{200D}\u{2640}\u{FE0F}345"}
              ]
     end
   end