Skip to content

Commit 4022eb2

Browse files
isoosisoos
committed
Control search index's text match scope through (local) latencies.
1 parent d96e254 commit 4022eb2

File tree

6 files changed

+275
-67
lines changed

6 files changed

+275
-67
lines changed

‎app/lib/search/mem_index.dart

+94-65
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,7 @@ class InMemoryPackageIndex {
226226
packageScores,
227227
parsedQueryText,
228228
includeNameMatches: (query.offset ?? 0) == 0,
229+
textMatchExtent: query.textMatchExtent,
229230
);
230231

231232
final nameMatches = textResults?.nameMatches;
@@ -287,7 +288,9 @@ class InMemoryPackageIndex {
287288
boundedList(indexedHits, offset: query.offset, limit: query.limit);
288289

289290
late List<PackageHit> packageHits;
290-
if (textResults != null && (textResults.topApiPages?.isNotEmpty ?? false)) {
291+
if (TextMatchExtent.shouldMatchApi(query.textMatchExtent) &&
292+
textResults != null &&
293+
(textResults.topApiPages?.isNotEmpty ?? false)) {
291294
packageHits = indexedHits.map((ps) {
292295
final apiPages = textResults.topApiPages?[ps.index]
293296
// TODO(https://github.com/dart-lang/pub-dev/issues/7106): extract title for the page
@@ -305,6 +308,7 @@ class InMemoryPackageIndex {
305308
nameMatches: nameMatches,
306309
topicMatches: topicMatches,
307310
packageHits: packageHits,
311+
errorMessage: textResults?.errorMessage,
308312
);
309313
}
310314

@@ -332,61 +336,82 @@ class InMemoryPackageIndex {
332336
IndexedScore<String> packageScores,
333337
String? text, {
334338
required bool includeNameMatches,
339+
required int? textMatchExtent,
335340
}) {
341+
if (text == null || text.isEmpty) {
342+
return null;
343+
}
344+
336345
final sw = Stopwatch()..start();
337-
if (text != null && text.isNotEmpty) {
338-
final words = splitForQuery(text);
339-
if (words.isEmpty) {
340-
for (var i = 0; i < packageScores.length; i++) {
341-
packageScores.setValue(i, 0);
342-
}
343-
return _TextResults.empty();
344-
}
346+
final words = splitForQuery(text);
347+
if (words.isEmpty) {
348+
packageScores.fillRange(0, packageScores.length, 0);
349+
return _TextResults.empty();
350+
}
345351

346-
bool aborted = false;
352+
final matchName = TextMatchExtent.shouldMatchName(textMatchExtent);
353+
if (!matchName) {
354+
packageScores.fillRange(0, packageScores.length, 0);
355+
return _TextResults.empty(
356+
errorMessage:
357+
'Search index in reduced mode: unable to match query text.');
358+
}
347359

348-
bool checkAborted() {
349-
if (!aborted && sw.elapsed > _textSearchTimeout) {
350-
aborted = true;
351-
_logger.info(
352-
'[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.');
353-
}
354-
return aborted;
360+
bool aborted = false;
361+
bool checkAborted() {
362+
if (!aborted && sw.elapsed > _textSearchTimeout) {
363+
aborted = true;
364+
_logger.info(
365+
'[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.');
355366
}
367+
return aborted;
368+
}
369+
370+
Set<String>? nameMatches;
371+
if (includeNameMatches && _documentsByName.containsKey(text)) {
372+
nameMatches ??= <String>{};
373+
nameMatches.add(text);
374+
}
375+
376+
// Multiple words are scored separately, and then the individual scores
377+
// are multiplied. We can use a package filter that is applied after each
378+
// word to reduce the scope of the later words based on the previous results.
379+
/// However, API docs search should be filtered on the original list.
380+
final indexedPositiveList = packageScores.toIndexedPositiveList();
356381

357-
Set<String>? nameMatches;
358-
if (includeNameMatches && _documentsByName.containsKey(text)) {
382+
final matchDescription =
383+
TextMatchExtent.souldMatchDescription(textMatchExtent);
384+
final matchReadme = TextMatchExtent.shouldMatchReadme(textMatchExtent);
385+
final matchApi = TextMatchExtent.shouldMatchApi(textMatchExtent);
386+
387+
for (final word in words) {
388+
if (includeNameMatches && _documentsByName.containsKey(word)) {
359389
nameMatches ??= <String>{};
360-
nameMatches.add(text);
390+
nameMatches.add(word);
361391
}
362392

363-
// Multiple words are scored separately, and then the individual scores
364-
// are multiplied. We can use a package filter that is applied after each
365-
// word to reduce the scope of the later words based on the previous results.
366-
/// However, API docs search should be filtered on the original list.
367-
final indexedPositiveList = packageScores.toIndexedPositiveList();
368-
369-
for (final word in words) {
370-
if (includeNameMatches && _documentsByName.containsKey(word)) {
371-
nameMatches ??= <String>{};
372-
nameMatches.add(word);
373-
}
393+
_scorePool.withScore(
394+
value: 0.0,
395+
fn: (wordScore) {
396+
_packageNameIndex.searchWord(word,
397+
score: wordScore, filterOnNonZeros: packageScores);
374398

375-
_scorePool.withScore(
376-
value: 0.0,
377-
fn: (wordScore) {
378-
_packageNameIndex.searchWord(word,
379-
score: wordScore, filterOnNonZeros: packageScores);
399+
if (matchDescription) {
380400
_descrIndex.searchAndAccumulate(word, score: wordScore);
401+
}
402+
if (matchReadme) {
381403
_readmeIndex.searchAndAccumulate(word,
382404
weight: 0.75, score: wordScore);
383-
packageScores.multiplyAllFrom(wordScore);
384-
},
385-
);
386-
}
405+
}
406+
packageScores.multiplyAllFrom(wordScore);
407+
},
408+
);
409+
}
387410

388-
final topApiPages =
389-
List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
411+
final topApiPages =
412+
List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
413+
414+
if (matchApi) {
390415
const maxApiPageCount = 2;
391416
if (!checkAborted()) {
392417
_apiSymbolIndex.withSearchWords(words, weight: 0.70, (symbolPages) {
@@ -420,29 +445,28 @@ class InMemoryPackageIndex {
420445
}
421446
});
422447
}
448+
}
423449

424-
// filter results based on exact phrases
425-
final phrases = extractExactPhrases(text);
426-
if (!aborted && phrases.isNotEmpty) {
427-
for (var i = 0; i < packageScores.length; i++) {
428-
if (packageScores.isNotPositive(i)) continue;
429-
final doc = _documents[i];
430-
final matchedAllPhrases = phrases.every((phrase) =>
431-
doc.package.contains(phrase) ||
432-
doc.description!.contains(phrase) ||
433-
doc.readme!.contains(phrase));
434-
if (!matchedAllPhrases) {
435-
packageScores.setValue(i, 0);
436-
}
450+
// filter results based on exact phrases
451+
final phrases = extractExactPhrases(text);
452+
if (!aborted && phrases.isNotEmpty) {
453+
for (var i = 0; i < packageScores.length; i++) {
454+
if (packageScores.isNotPositive(i)) continue;
455+
final doc = _documents[i];
456+
final matchedAllPhrases = phrases.every((phrase) =>
457+
(matchName && doc.package.contains(phrase)) ||
458+
(matchDescription && doc.description!.contains(phrase)) ||
459+
(matchReadme && doc.readme!.contains(phrase)));
460+
if (!matchedAllPhrases) {
461+
packageScores.setValue(i, 0);
437462
}
438463
}
439-
440-
return _TextResults(
441-
topApiPages,
442-
nameMatches: nameMatches?.toList(),
443-
);
444464
}
445-
return null;
465+
466+
return _TextResults(
467+
topApiPages,
468+
nameMatches: nameMatches?.toList(),
469+
);
446470
}
447471

448472
List<IndexedPackageHit> _rankWithValues(
@@ -521,15 +545,20 @@ class InMemoryPackageIndex {
521545
class _TextResults {
522546
final List<List<MapEntry<String, double>>?>? topApiPages;
523547
final List<String>? nameMatches;
548+
final String? errorMessage;
524549

525-
factory _TextResults.empty() => _TextResults(
526-
null,
527-
nameMatches: null,
528-
);
550+
factory _TextResults.empty({String? errorMessage}) {
551+
return _TextResults(
552+
null,
553+
nameMatches: null,
554+
errorMessage: errorMessage,
555+
);
556+
}
529557

530558
_TextResults(
531559
this.topApiPages, {
532560
required this.nameMatches,
561+
this.errorMessage,
533562
});
534563
}
535564

‎app/lib/search/search_service.dart

+50-1
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,9 @@ class ServiceSearchQuery {
165165
final int? offset;
166166
final int? limit;
167167

168+
/// The scope/depth of text matching.
169+
final int? textMatchExtent;
170+
168171
ServiceSearchQuery._({
169172
this.query,
170173
TagsPredicate? tagsPredicate,
@@ -173,6 +176,7 @@ class ServiceSearchQuery {
173176
this.order,
174177
this.offset,
175178
this.limit,
179+
this.textMatchExtent,
176180
}) : parsedQuery = ParsedQueryText.parse(query),
177181
tagsPredicate = tagsPredicate ?? TagsPredicate(),
178182
publisherId = publisherId?.trimToNull();
@@ -185,6 +189,7 @@ class ServiceSearchQuery {
185189
int? minPoints,
186190
int offset = 0,
187191
int? limit = 10,
192+
int? textMatchExtent,
188193
}) {
189194
final q = query?.trimToNull();
190195
return ServiceSearchQuery._(
@@ -195,6 +200,7 @@ class ServiceSearchQuery {
195200
order: order,
196201
offset: offset,
197202
limit: limit,
203+
textMatchExtent: textMatchExtent,
198204
);
199205
}
200206

@@ -210,6 +216,8 @@ class ServiceSearchQuery {
210216
int.tryParse(uri.queryParameters['minPoints'] ?? '0') ?? 0;
211217
final offset = int.tryParse(uri.queryParameters['offset'] ?? '0') ?? 0;
212218
final limit = int.tryParse(uri.queryParameters['limit'] ?? '0') ?? 0;
219+
final textMatchExtent =
220+
int.tryParse(uri.queryParameters['textMatchExtent'] ?? '');
213221

214222
return ServiceSearchQuery.parse(
215223
query: q,
@@ -219,6 +227,7 @@ class ServiceSearchQuery {
219227
minPoints: minPoints,
220228
offset: max(0, offset),
221229
limit: max(_minSearchLimit, limit),
230+
textMatchExtent: textMatchExtent,
222231
);
223232
}
224233

@@ -229,6 +238,7 @@ class ServiceSearchQuery {
229238
SearchOrder? order,
230239
int? offset,
231240
int? limit,
241+
int? textMatchExtent,
232242
}) {
233243
return ServiceSearchQuery._(
234244
query: query ?? this.query,
@@ -238,6 +248,7 @@ class ServiceSearchQuery {
238248
minPoints: minPoints,
239249
offset: offset ?? this.offset,
240250
limit: limit ?? this.limit,
251+
textMatchExtent: textMatchExtent ?? this.textMatchExtent,
241252
);
242253
}
243254

@@ -251,6 +262,8 @@ class ServiceSearchQuery {
251262
'minPoints': minPoints.toString(),
252263
'limit': limit?.toString(),
253264
'order': order?.name,
265+
if (textMatchExtent != null)
266+
'textMatchExtent': textMatchExtent.toString(),
254267
};
255268
map.removeWhere((k, v) => v == null);
256269
return map;
@@ -277,7 +290,8 @@ class ServiceSearchQuery {
277290
_hasOnlyFreeText &&
278291
_isNaturalOrder &&
279292
_hasNoOwnershipScope &&
280-
!_isFlutterFavorite;
293+
!_isFlutterFavorite &&
294+
TextMatchExtent.shouldMatchApi(textMatchExtent);
281295

282296
bool get considerHighlightedHit => _hasOnlyFreeText && _hasNoOwnershipScope;
283297
bool get includeHighlightedHit => considerHighlightedHit && offset == 0;
@@ -295,6 +309,41 @@ class ServiceSearchQuery {
295309
}
296310
}
297311

312+
/// The scope (depth) of the text matching.
313+
abstract class TextMatchExtent {
314+
/// No text search is done.
315+
/// Requests with text queries will return a failure message.
316+
static final int none = 10;
317+
318+
/// Text search is on package names.
319+
static final int name = 20;
320+
321+
/// Text search is on package names, descriptions and topic tags.
322+
static final int description = 30;
323+
324+
/// Text search is on names, descriptions, topic tags and readme content.
325+
static final int readme = 40;
326+
327+
/// Text search is on names, descriptions, topic tags, readme content and API symbols.
328+
static final int api = 50;
329+
330+
/// No value was given, assuming default behavior of including everything.
331+
static final int unspecified = 99;
332+
333+
/// Text search is on package names.
334+
static bool shouldMatchName(int? value) => (value ?? unspecified) >= name;
335+
336+
/// Text search is on package names, descriptions and topic tags.
337+
static bool souldMatchDescription(int? value) =>
338+
(value ?? unspecified) >= description;
339+
340+
/// Text search is on names, descriptions, topic tags and readme content.
341+
static bool shouldMatchReadme(int? value) => (value ?? unspecified) >= readme;
342+
343+
/// Text search is on names, descriptions, topic tags, readme content and API symbols.
344+
static bool shouldMatchApi(int? value) => (value ?? unspecified) >= api;
345+
}
346+
298347
class QueryValidity {
299348
final String? rejectReason;
300349

‎app/lib/service/entrypoint/search.dart

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class SearchCommand extends Command {
4343
);
4444
registerScopeExitCallback(index.close);
4545

46-
registerSearchIndex(IsolateSearchIndex(index));
46+
registerSearchIndex(LatencyAwareSearchIndex(IsolateSearchIndex(index)));
4747

4848
void scheduleRenew() {
4949
scheduleMicrotask(() async {

0 commit comments

Comments
 (0)