Merge pull request #16 from luistrivelatto/master

Fixes #15, #17, #18, and does other improvements

Author: comigor

.github/actions/dart-test/Dockerfile

@@ -1,4 +1,4 @@
-FROM cirrusci/flutter
+FROM cirrusci/flutter:beta
 
 USER root
 

CHANGELOG.md

@@ -1,5 +1,16 @@
 # CHANGELOG
 
+## 0.3.0
+- Improve search results on weighted search to combine scores from all keys
+- Improve search results on single-keyed search, making it consistent with non-weighted search
+- Add parameter to ignore tokens smaller than a certain length when searching
+- Add normalization of WeightedKey weights
+- Fix bug where results returned from search all had arrayIndex = -1
+- Fix bug where the token scores didn't count towards the result score
+
+## 0.2.5
+- Fix bug for search that started or ended with whitespace when tokenize option is true
+
 ## 0.2.4
 - Bump dependencies, fix CI
 

lib/data/fuzzy_options.dart

@@ -9,15 +9,17 @@ class WeightedKey<T> {
     @required this.name,
     @required this.getter,
     @required this.weight,
-  }) : assert(weight >= 0 && weight <= 1);
+  }) : assert(weight > 0, 'Weight should be positive and non-zero');
 
   /// Name of this getter
   final String name;
 
   /// Getter to a specifc string inside item
   final String Function(T obj) getter;
 
-  /// Weight of this getter
+  /// Weight of this getter. When passing a list of WeightedKey to FuzzyOptions,
+  /// the weight can be any positive number; FuzzyOptions normalizes it on
+  /// construction.
   final double weight;
 }
 
@@ -28,17 +30,20 @@ int _defaultSortFn<T>(Result<T> a, Result<T> b) => a.score.compareTo(b.score);
 
 /// Options for performing a fuzzy search
 class FuzzyOptions<T> {
-  /// Instantiate an options object
+  /// Instantiate an options object.
+  /// The `keys` list requires a positive number (they'll be normalized upon
+  /// instantiation). If any weight is not positive, throws an ArgumentError.
   FuzzyOptions({
     this.location = 0,
     this.distance = 100,
     this.threshold = 0.6,
     this.maxPatternLength = 32,
     this.isCaseSensitive = false,
     Pattern tokenSeparator,
+    this.minTokenCharLength = 1,
     this.findAllMatches = false,
     this.minMatchCharLength = 1,
-    this.keys = const [],
+    List<WeightedKey<T>> keys = const [],
     this.shouldSort = true,
     SorterFn<T> sortFn,
     this.tokenize = false,
@@ -47,6 +52,7 @@ class FuzzyOptions<T> {
     this.shouldNormalize = false,
   })  : tokenSeparator =
             tokenSeparator ?? RegExp(r' +', caseSensitive: isCaseSensitive),
+        keys = _normalizeWeights(keys),
         sortFn = sortFn ?? _defaultSortFn;
 
   /// Approximately where in the text is the pattern expected to be found?
@@ -72,6 +78,9 @@ class FuzzyOptions<T> {
   /// Regex used to separate words when searching. Only applicable when `tokenize` is `true`.
   final Pattern tokenSeparator;
 
+  /// Ignore tokens with length smaller than this. Only applicable when `tokenize` is `true`.
+  final int minTokenCharLength;
+
   /// When true, the algorithm continues searching to the end of the input even if a perfect
   /// match is found before the end of the same input.
   final bool findAllMatches;
@@ -112,6 +121,7 @@ class FuzzyOptions<T> {
         maxPatternLength: options?.maxPatternLength ?? maxPatternLength,
         isCaseSensitive: options?.isCaseSensitive ?? isCaseSensitive,
         tokenSeparator: options?.tokenSeparator ?? tokenSeparator,
+        minTokenCharLength: options?.minTokenCharLength ?? minTokenCharLength,
         findAllMatches: options?.findAllMatches ?? findAllMatches,
         minMatchCharLength: options?.minMatchCharLength ?? minMatchCharLength,
         keys: options?.keys ?? keys,
@@ -122,4 +132,22 @@ class FuzzyOptions<T> {
         verbose: options?.verbose ?? verbose,
         shouldNormalize: options?.shouldNormalize ?? shouldNormalize,
       );
+
+  static List<WeightedKey<T>> _normalizeWeights<T>(List<WeightedKey<T>> keys) {
+    if (keys.isEmpty) {
+      return [];
+    }
+
+    var weightSum = keys
+        .map((key) => key.weight)
+        .fold<double>(0, (previousValue, element) => previousValue + element);
+
+    return keys
+        .map((key) => WeightedKey<T>(
+              name: key.name,
+              getter: key.getter,
+              weight: key.weight / weightSum,
+            ))
+        .toList();
+  }
 }

lib/data/result.dart

@@ -81,7 +81,8 @@ class ResultDetails<T> {
   /// Score of this result
   final double score;
 
-  /// nScore of this result (?)
+  /// nScore of this result. It's the weighted score of the match, when it's
+  /// a weighted search (i.e. uses WeightedKeys).
   double nScore;
 
   /// Indexes of matched patterns on the value

lib/fuzzy.dart

@@ -67,7 +67,9 @@ class Fuzzy<T> {
 
     if (options.tokenize) {
       // Tokenize on the separator
-      final tokens = pattern.split(options.tokenSeparator);
+      final tokens = pattern.split(options.tokenSeparator)
+        ..removeWhere((token) => token.isEmpty)
+        ..removeWhere((token) => token.length < options.minTokenCharLength);
       for (var i = 0, len = tokens.length; i < len; i += 1) {
         tokenSearchers.add(Bitap(tokens[i], options: options));
       }
@@ -136,7 +138,6 @@ class Fuzzy<T> {
 
   List<Result<T>> _analyze({
     String key = '',
-    int arrayIndex = -1,
     String value,
     T record,
     int index,
@@ -153,7 +154,7 @@ class Fuzzy<T> {
     }
 
     var exists = false;
-    var averageScore = -1;
+    var averageScore = -1.0;
     var numTextMatches = 0;
 
     final mainSearchResult = fullSearcher.search(value.toString());
@@ -190,8 +191,8 @@ class Fuzzy<T> {
         }
       }
 
-      final averageScore =
-          scores.fold(0, (memo, score) => memo + score) / scores.length;
+      averageScore =
+          scores.fold<double>(0, (memo, score) => memo + score) / scores.length;
 
       _log('Token score average: $averageScore');
     }
@@ -218,7 +219,7 @@ class Fuzzy<T> {
         // existingResult.score, bitapResult.score
         existingResult.matches.add(ResultDetails<T>(
           key: key,
-          arrayIndex: arrayIndex,
+          arrayIndex: index,
           value: value,
           score: finalScore,
           matchedIndices: mainSearchResult.matchedIndices,
@@ -230,7 +231,7 @@ class Fuzzy<T> {
           matches: [
             ResultDetails<T>(
               key: key,
-              arrayIndex: arrayIndex,
+              arrayIndex: index,
               value: value,
               score: finalScore,
               matchedIndices: mainSearchResult.matchedIndices,
@@ -254,29 +255,40 @@ class Fuzzy<T> {
   void _computeScore(Map<String, double> weights, List<Result<T>> results) {
     _log('\n\nComputing score:\n');
 
+    if (weights.length <= 1) {
+      _computeScoreNoWeights(results);
+    } else {
+      _computeScoreWithWeights(weights, results);
+    }
+  }
+
+  void _computeScoreNoWeights(List<Result<T>> results) {
     for (var i = 0, len = results.length; i < len; i += 1) {
       final matches = results[i].matches;
-      final scoreLen = matches.length;
+      var bestScore = matches.map((m) => m.score).fold<double>(
+          1.0, (previousValue, element) => min(previousValue, element));
+      results[i].score = bestScore;
+    }
+  }
 
+  void _computeScoreWithWeights(
+      Map<String, double> weights, List<Result<T>> results) {
+    for (var i = 0, len = results.length; i < len; i += 1) {
       var currScore = 1.0;
-      var bestScore = 1.0;
 
-      for (var j = 0; j < scoreLen; j += 1) {
-        final weight = weights[matches[j].key] ?? 1.0;
-        final score = weight == 1.0
-            ? matches[j].score
-            : (matches[j].score == 0.0 ? 0.001 : matches[j].score);
+      for (var match in results[i].matches) {
+        var weight = weights[match.key];
+        assert(weight != null);
+
+        // We don't use 0 so that the weight differences don't get zeroed out
+        final score = match.score == 0.0 ? 0.001 : match.score;
         final nScore = score * weight;
 
-        if (weight != 1) {
-          bestScore = min(bestScore, nScore);
-        } else {
-          matches[j].nScore = nScore;
-          currScore *= nScore;
-        }
+        match.nScore = nScore;
+        currScore *= nScore;
       }
 
-      results[i].score = bestScore == 1.0 ? currScore : bestScore;
+      results[i].score = currScore;
     }
   }
 

pubspec.yaml

@@ -1,5 +1,5 @@
 name: fuzzy
-version: 0.2.4
+version: 0.3.0
 
 description: >
   Fuzzy search in Dart. initially translated from Fuse.js.

test/fixtures/games.dart

@@ -0,0 +1,36 @@
+class Game {
+  Game({this.tournament, this.stage});
+
+  final String tournament;
+  final String stage;
+
+  @override
+  String toString() => '$tournament $stage';
+}
+
+final customGameList = [
+  Game(
+    tournament: 'WorldCup',
+    stage: 'Groups',
+  ),
+  Game(
+    tournament: 'WorldCup',
+    stage: 'Semi-finals',
+  ),
+  Game(
+    tournament: 'WorldCup',
+    stage: 'Final',
+  ),
+  Game(
+    tournament: 'ChampionsLeague',
+    stage: 'Groups',
+  ),
+  Game(
+    tournament: 'ChampionsLeague',
+    stage: 'Semi-finals',
+  ),
+  Game(
+    tournament: 'ChampionsLeague',
+    stage: 'Final',
+  ),
+];