add notebook

and dialyxir

Author: vanessaklee

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.12.3-otp-24
-24.1.2
+elixir 1.15.5-otp-26
+erlang 26.0.2

config/config.exs

@@ -1,6 +1,6 @@
 # This file is responsible for configuring your application
 # and its dependencies with the aid of the Mix.Config module.
-use Mix.Config
+import Config
 
 # This configuration is loaded before any dependency and is restricted
 # to this project. If another project depends on this project, this

lib/akin.ex

@@ -3,7 +3,8 @@ defmodule Akin do
   Akin
   =======
 
-  Functions for comparing two strings for similarity using a collection of string comparison algorithms for Elixir. Algorithms can be called independently or in total to return a map of metrics.
+  Functions for comparing two strings for similarity using a collection of string comparison algorithms for Elixir. 
+  Algorithms can be called independently or in total to return a map of metrics.
 
   ## Options
 
@@ -32,7 +33,7 @@ defmodule Akin do
   alias Akin.Corpus
   alias Akin.Names
 
-  @spec compare(binary() | %Corpus{}, binary() | %Corpus{}, keyword()) :: float()
+  @spec compare(binary() | %Corpus{}, binary() | %Corpus{}, keyword()) :: map()
   @doc """
   Compare two strings. Return map of algorithm metrics.
 
@@ -69,6 +70,14 @@ defmodule Akin do
   @doc """
   Compare a string against a list of strings.  Matches are determined by algorithem metrics equal to or higher than the
   `match_at` option. Return a list of strings that are a likely match.
+
+  Future Plans 
+  * _if_ the name part is an initial, give the `initials` score its weight, otherwise reduce it
+  * if the `initials` score is significantly higher than the average of the others, reduce the `initials` score to the average of the others 
+  * add options 
+    * "use_average", "top_three", and/or "average_of_top_three"
+    * "group" to results into strong matches and weak matches
+    * "details" to include the scores in the result list
   """
   def match_names(left, rights, opts \\ default_opts())
 
@@ -90,7 +99,7 @@ defmodule Akin do
     end)
   end
 
-  @spec match_names_metrics(binary(), list(), keyword()) :: float()
+  @spec match_names_metrics(binary(), list(), keyword()) :: list()
   @doc """
   Compare a string against a list of strings. Matches are determined by algorithem metrics equal to or higher than the
   `match_at` option. Return a list of strings that are a likely match and their algorithm metrics.

lib/akin/algorithms/helpers/initials_comparison.ex

@@ -61,10 +61,6 @@ defmodule Akin.Helpers.InitialsComparison do
     Enum.map(lists, fn list -> String.at(list, 0) end)
   end
 
-  defp initials(list) when is_list(list) do
-    Enum.map(list, fn l -> String.at(l, 0) end)
-  end
-
   defp initials(_), do: []
 
   defp actual_initials(list) do

lib/akin/algorithms/levenshtein.ex

@@ -5,6 +5,7 @@ defmodule Akin.Levenshtein do
   @behaviour Akin.Task
   alias Akin.Corpus
 
+  @spec compare(list(), list()) :: float()
   @spec compare(%Corpus{}, %Corpus{}) :: float()
   @spec compare(%Corpus{}, %Corpus{}, Keyword.t()) :: float()
   @doc """
@@ -21,16 +22,16 @@ defmodule Akin.Levenshtein do
 
   def compare([], string), do: length(string)
 
-  def compare(left, right) when is_binary(left) and is_binary(right) do
-    distance = compare(String.graphemes(left), String.graphemes(right))
-    1.0 - distance / Enum.max([String.length(left), String.length(right)])
-  end
-
   def compare(left, right)
       when is_list(left) and is_list(right) do
     rec_lev(left, right, :lists.seq(0, length(right)), 1)
   end
 
+  def compare(left, right) when is_binary(left) and is_binary(right) do
+    distance = compare(String.graphemes(left), String.graphemes(right))
+    1.0 - distance / Enum.max([String.length(left), String.length(right)])
+  end
+
   defp rec_lev([src_head | src_tail], right, distlist, step) do
     rec_lev(src_tail, right, lev_dist(right, distlist, src_head, [step], step), step + 1)
   end

lib/akin/algorithms/phonetic/word.ex

@@ -46,9 +46,8 @@ defmodule Word do
   end
 
   def normalize(input) do
-    String.normalize(input, :nfd)
-    |> String.graphemes()
-    |> Enum.reduce([], fn l, acc -> if Unicode.category(l) == "Mn", do: acc, else: [l | acc] end)
-    |> Enum.reverse()
+    input
+    |> String.normalize(:nfd)
+    |> Unicode.unaccent()
   end
 end

lib/akin/algorithms/substring_set.ex

@@ -39,7 +39,8 @@ defmodule Akin.SubstringSet do
       ) do
     case Strategy.determine(l_string, r_string) do
       :standard ->
-        similarity(l_set, r_set) |> score(opts(opts, :level))
+        similarity(l_set, r_set) 
+        |> score(opts(opts, :level))
 
       {:substring, scale} ->
         score =
@@ -53,10 +54,11 @@ defmodule Akin.SubstringSet do
     end
   end
 
-  defp score(scores, "weak"), do: Enum.max(scores)
-
-  defp score(scores, _) do
-    Enum.sum(scores) / (Enum.count(scores) - 1)
+  defp score(scores, level) do
+    case level do
+      "weak" -> Enum.max(scores)
+      _ -> Enum.sum(scores) / (Enum.count(scores) - 1)
+    end
   end
 
   defp similarity(left, right) do

lib/akin/util.ex

@@ -93,8 +93,6 @@ defmodule Akin.Util do
     |> :unicode.characters_to_nfd_binary()
   end
 
-  defp replace_accents({:error, string, _b}), do: string
-
   defp replace_accents(string) do
     string = String.replace(string, ~r/\W/u, "")
 
@@ -253,54 +251,6 @@ defmodule Akin.Util do
     |> Enum.sort()
   end
 
-
-  # defp list_algorithms(metric, nil, _) do
-  #   Enum.reduce(@typed_algorithms, [], fn {name, m, _}, acc ->
-  #     if metric == m, do: [name | acc], else: = acc
-  #   end)
-  #   |> Enum.sort()
-  # end
-
-  # defp list_algorithms(nil, unit, _) do
-  #   Enum.reduce(@typed_algorithms, [], fn {name, _, u}, acc ->
-  #     if metric == m, do: [name | acc], else: = acc
-  #   end)
-  #   |> Enum.sort()
-  # end
-
-  # defp list_algorithms(nil, nil, []), do: @algorithms
-
-  # defp list_algorithms(metric, unit, []) do
-  #   Enum.filter(@typed_algorithms, fn {_, m, u} ->
-  #     unit == u && metric == m
-  #   end)
-  #   # ["bag_distance", "levenshtein", "jaro_winkler", "jaccard", "tversky", "sorensen_dice"]
-  # end
-
-  # defp list_algorithms(_, _, algorithms) when is_list(algorithms) do
-  #   Enum.filter(algorithms, fn a -> a in @algorithms end)
-  # end
-
-  # defp list_algorithms("string", "partial", []) do
-  #   ["substring_set", "substring_sort", "overlap", "ngram"]
-  # end
-
-  # defp list_algorithms("string", _, []) do
-  #   list_algorithms("string", "whole", []) ++ list_algorithms("string", "partial", [])
-  # end
-
-  # defp list_algorithms("phonetic", "whole", []) do
-  #   ["metaphone", "double_metaphone"]
-  # end
-
-  # defp list_algorithms("phonetic", "partial", []) do
-  #   ["substring_double_metaphone"]
-  # end
-
-  # defp list_algorithms("phonetic", _, []) do
-  #   list_algorithms("phonetic", "whole", []) ++ list_algorithms("phonetic", "partial", [])
-  # end
-
   @spec ngram_tokenize(any, any) :: list
   @doc """
   Tokenizes the input into N-grams (http://en.wikipedia.org/wiki/N-gram).
@@ -323,7 +273,7 @@ defmodule Akin.Util do
 
   def ngram_tokenize(_), do: []
 
-  @spec opts(keyword(), atom()) :: integer() | boolean()
+  @spec opts(keyword(), atom()) :: any()
   @doc """
   Take the value for the key from the options. If not present, use the default value from the default
   options list.

mix.exs

@@ -30,6 +30,7 @@ defmodule Akin.Mixfile do
   defp deps do
     [
       {:credo, "~> 1.0", only: :dev},
+      {:dialyxir, "~> 1.3", only: [:dev], runtime: false},
       {:earmark, "~> 1.3", only: :dev},
       {:excoveralls, "~> 0.10", only: :test},
       {:ex_doc, "~> 0.19", only: :dev},

notebooks/examples.livemd

@@ -0,0 +1,177 @@
+# Akin Examples
+
+## Akin
+
+Akin is a collection of string comparison algorithms for Elixir. Algorithms can be called independently or combined to return a map of metrics. This library was built to facilitiate the disambiguation of names but can be used to compare any two binaries.
+
+## Algorithms
+
+Utilities are provided to return all avialable algorithms.
+
+```elixir
+Akin.Util.list_algorithms()
+```
+
+**Note**: Hamming Distance is excluded as it only compares strings of equal length. To use the Hamming Distance algorithm, call it directly (see: [Independent Algorithms](#independent-algorithms)).
+
+## Combined Algorithms
+
+### Metrics
+
+Results from all algorithms are returned as a map of metrics.
+
+<!-- livebook:{"break_markdown":true} -->
+
+#### Compare Strings
+
+Experiment by changing the value of the strings.
+
+```elixir
+a = "weird"
+b = "wierd"
+
+Akin.compare(a, b)
+```
+
+### Options
+
+Comparison accepts options in a Keyword list.
+
+1. `algorithms`: algorithms to use in comparision. Accepts the name or a keyword list. Default is algorithms/0.
+   1. `metric` - algorithm metric. Default is both
+      * "string": uses string algorithms
+      * "phonetic": uses phonetic algorithms
+   2. `unit` - algorithm unit. Default is both.
+      * "whole": uses algorithms best suited for whole string comparison (distance)
+      * "partial": uses algorithms best suited for partial string comparison (substring)
+2. `level` - level for double phonetic matching. Default is "normal".
+   * "strict": both encodings for each string must match
+   * "strong": the primary encoding for each string must match
+   * "normal": the primary encoding of one string must match either encoding of other string (default)
+   * "weak":   either primary or secondary encoding of one string must match one encoding of other string
+3. `match_at`: an algorith score equal to or above this value is condsidered a match. Default is 0.9
+4. `ngram_size`: number of contiguous letters to split strings into. Default is 2.
+5. `short_length`: qualifies as "short" to recieve a shortness boost. Used by Name Metric. Default is 8.
+6. `stem`: boolean representing whether to compare the stemmed version the strings; uses Stemmer. Default `false`
+
+```elixir
+opts = [algorithms: ["bag_distance", "jaccard", "jaro_winkler"]]
+Akin.compare(a, b, opts)
+```
+
+```elixir
+opts = [algorithms: [metric: "phonetic", unit: "whole"]]
+Akin.compare(a, b, opts)
+```
+
+```elixir
+Akin.compare(a, b, algorithms: [metric: "string", unit: "whole"], ngram_size: 1)
+```
+
+#### n-gram Size
+
+The default ngram size for the algorithms is 2. You can change by setting 
+a value in opts.
+
+```elixir
+opts = [algorithms: ["sorensen_dice"]]
+Akin.compare(a, b, opts)
+```
+
+```elixir
+opts = [algorithms: ["sorensen_dice"], ngram_size: 1]
+Akin.compare(a, b, opts)
+```
+
+#### Match Level
+
+The default match strictness is "normal" You change it by setting 
+a value in opts. Currently it only affects the outcomes of the `substring_set` and
+`double_metaphone` algorithms
+
+```elixir
+left = "Alice in Wonderland"
+right = "Alice's Adventures in Wonderland"
+
+Akin.compare(left, right, algorithms: ["substring_set"])
+```
+
+```elixir
+Akin.compare(left, right, algorithms: ["substring_set"], level: "weak")
+```
+
+```elixir
+left = "which way"
+right = "whitch way"
+
+Akin.compare(left, right, algorithms: ["double_metaphone"], level: "weak")
+```
+
+```elixir
+Akin.compare(left, right, algorithms: ["double_metaphone"], level: "strict")
+```
+
+#### Stems
+
+Compare the stemmed version of two strings.
+
+```elixir
+not_gerund = "write"
+gerund = "writing"
+
+Akin.compare(not_gerund, gerund, algorithms: ["bag_distance", "double_metaphone"])
+```
+
+```elixir
+Akin.compare(not_gerund, gerund, algorithms: ["bag_distance", "double_metaphone"], stem: true)
+```
+
+### Preprocessing
+
+Before being compared, strings are converted to downcase and unicode standard, whitespace is standardized, nontext (like punctuation & emojis) is replaced, and accents are converted. The string is then composed into a struct representing the corpus of data used by the comparison algorithms.
+
+```elixir
+name = "Alice Liddell"
+
+Akin.Util.compose(name)
+```
+
+### Accents
+
+```elixir
+name_a = "Hubert Łępicki"
+
+Akin.Util.compose(name_a)
+```
+
+```elixir
+name_b = "Hubert Lepicki"
+
+Akin.compare(name_a, name_b)
+```
+
+### Phonemes
+
+```elixir
+Akin.phonemes(name)
+```
+
+```elixir
+Akin.phonemes("wonderland")
+```
+
+## Independent Algorithms
+
+Each algorithm can be called directly. Module names are camelcased versions of the the snakecased algorithm names returned by `list_algorithms/0`.
+
+```elixir
+a = Akin.Util.compose("weird")
+b = Akin.Util.compose("wierd")
+Akin.BagDistance.compare(a, b)
+```
+
+Hamming Distance is excluded from `list_algorithms/0` and the combined algorithm metrics as it only compares strings of equal length. To use the Hamming Distance algorithm, call it directly.
+
+```elixir
+Akin.Hamming.compare("weird", "wierd")
+```