Metrics: categories, empty predictions, tests, fnmr@fmr

**CHANGELOG**

* Added support of empty predictions to the retrieval metrics. For example, it may be useful when we cut retrieval results by distance threshold).
* Moved categories handling to functional metrics from `EmbeddingMetrics` class, also updated `.md` example to show how to deal with categories.
* Added `calc_fnmr_at_fmr_rr`, removed `extract_pos_neg_dists` and `calc_fnmr_at_fmr_from_matrices`. Returned `fnmr@fmr` to `EmbeddingMetrics` (there was a todo).

TESTS
* Moved tests that use old formats of retrieval metrics to a separate folder: `...test_metrics/test_outdated/...`.
* Added a few new tests on retrieval metrics: test handling categories and empty predictions.
* Added test on `calc_fnmr_at_fmr_rr`.
* Added test that `EmbeddingMetrics` calculate all expected metrics.

Author: AlekseySh

Makefile

@@ -59,12 +59,12 @@ wandb_login:
 
 .PHONY: run_all_tests
 run_all_tests: download_mock_dataset wandb_login
-	export PYTORCH_ENABLE_MPS_FALLBACK=1; pytest --disable-warnings -sv tests
+	export PYTORCH_ENABLE_MPS_FALLBACK=1; export PYTHONPATH=.; pytest --disable-warnings -sv tests
 	pytest --disable-warnings --doctest-modules --doctest-continue-on-failure -sv oml
 
 .PHONY: run_short_tests
 run_short_tests: download_mock_dataset
-	export PYTORCH_ENABLE_MPS_FALLBACK=1; pytest --disable-warnings -sv -m "not long and not needs_optional_dependency" tests
+	export PYTORCH_ENABLE_MPS_FALLBACK=1; export PYTHONPATH=.; pytest --disable-warnings -sv -m "not long and not needs_optional_dependency" tests
 	pytest --disable-warnings --doctest-modules --doctest-continue-on-failure -sv oml
 
 .PHONY: test_converters
@@ -122,6 +122,7 @@ clean:
 	find . -type d -name "ml-runs" -exec rm -r {} +
 	find . -type d -name "logs" -exec rm -r {} +
 	find . -type d -name ".ipynb_checkpoints" -exec rm -r {} +
+	find . -type d -name ".hydra" -exec rm -r {} +
 	find . -type f -name "*.log" -exec rm {} +
 	find . -type f -name "*predictions.json" -exec rm {} +
 	rm -rf docs/build

README.md

@@ -334,6 +334,8 @@ for batch in tqdm(train_loader):
 [comment]:vanilla-validation-start
 ```python
 
+import numpy as np
+
 from oml.datasets import ImageQueryGalleryLabeledDataset
 from oml.inference import inference
 from oml.metrics import calc_retrieval_metrics_rr
@@ -345,17 +347,19 @@ from oml.registry.transforms import get_transforms_for_pretrained
 extractor = ViTExtractor.from_pretrained("vits16_dino")
 transform, _ = get_transforms_for_pretrained("vits16_dino")
 
-_, df_val = download_mock_dataset(global_paths=True)
+_, df_val = download_mock_dataset(global_paths=True, df_name="df_with_category.csv")
 dataset = ImageQueryGalleryLabeledDataset(df_val, transform=transform)
 
+# you can optionally provide categories to have category wise metrics
+query_categories = np.array(dataset.extra_data["category"])[dataset.get_query_ids()]
+
 embeddings = inference(extractor, dataset, batch_size=4)
 
 rr = RetrievalResults.compute_from_embeddings(embeddings, dataset, n_items_to_retrieve=5)
-metrics = calc_retrieval_metrics_rr(rr, map_top_k=(3, 5), precision_top_k=(5,), cmc_top_k=(3,))
+metrics = calc_retrieval_metrics_rr(rr, query_categories, map_top_k=(3, 5), precision_top_k=(5,), cmc_top_k=(3,))
 
 print(rr, "\n", metrics)
-rr.visualize(query_ids=[2, 1], dataset=dataset).show()
-
+rr.visualize(query_ids=[2, 1], dataset=dataset)
 ```
 [comment]:vanilla-validation-end
 </p>

docs/readme/examples_source/extractor/val.md

@@ -5,6 +5,8 @@
 [comment]:vanilla-validation-start
 ```python
 
+import numpy as np
+
 from oml.datasets import ImageQueryGalleryLabeledDataset
 from oml.inference import inference
 from oml.metrics import calc_retrieval_metrics_rr
@@ -16,17 +18,19 @@ from oml.registry.transforms import get_transforms_for_pretrained
 extractor = ViTExtractor.from_pretrained("vits16_dino")
 transform, _ = get_transforms_for_pretrained("vits16_dino")
 
-_, df_val = download_mock_dataset(global_paths=True)
+_, df_val = download_mock_dataset(global_paths=True, df_name="df_with_category.csv")
 dataset = ImageQueryGalleryLabeledDataset(df_val, transform=transform)
 
+# you can optionally provide categories to have category wise metrics
+query_categories = np.array(dataset.extra_data["category"])[dataset.get_query_ids()]
+
 embeddings = inference(extractor, dataset, batch_size=4)
 
 rr = RetrievalResults.compute_from_embeddings(embeddings, dataset, n_items_to_retrieve=5)
-metrics = calc_retrieval_metrics_rr(rr, map_top_k=(3, 5), precision_top_k=(5,), cmc_top_k=(3,))
+metrics = calc_retrieval_metrics_rr(rr, query_categories, map_top_k=(3, 5), precision_top_k=(5,), cmc_top_k=(3,))
 
 print(rr, "\n", metrics)
-rr.visualize(query_ids=[2, 1], dataset=dataset).show()
-
+rr.visualize(query_ids=[2, 1], dataset=dataset)
 ```
 [comment]:vanilla-validation-end
 </p>

docs/source/contents/metrics.rst

@@ -26,9 +26,9 @@ calc_retrieval_metrics
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: oml.functional.metrics.calc_retrieval_metrics
 
-calc_topological_metrics
+calc_retrieval_metrics_rr
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. autofunction:: oml.functional.metrics.calc_topological_metrics
+.. autofunction:: oml.metrics.embeddings.calc_retrieval_metrics_rr
 
 calc_cmc
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -46,6 +46,14 @@ calc_fnmr_at_fmr
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: oml.functional.metrics.calc_fnmr_at_fmr
 
+calc_fnmr_at_fmr_rr
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: oml.metrics.embeddings.calc_fnmr_at_fmr_rr
+
+calc_topological_metrics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: oml.functional.metrics.calc_topological_metrics
+
 calc_pcf
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: oml.functional.metrics.calc_pcf

oml/functional/knn.py

@@ -2,6 +2,7 @@
 
 import torch
 from torch import BoolTensor, FloatTensor, LongTensor
+from tqdm.auto import tqdm
 
 from oml.const import BS_KNN
 from oml.utils.misc_torch import pairwise_dist
@@ -53,7 +54,7 @@ def batched_knn(
     gt_ids = []
 
     # we do batching over first (queries) dimension
-    for i in range(0, nq, bs):
+    for i in tqdm(range(0, nq, bs), desc="Finding nearest neighbors."):
         distances_b = pairwise_dist(x1=embeddings_query[i : i + bs, :], x2=embeddings_gallery, p=2)
         ids_query_b = ids_query[i : i + bs]