Skip to content

Commit 4b42654

Browse files
cwlacewes-gobrielsys-vdms
authoredJan 27, 2025
Update faiss version in both Dockerfiles and any required API updates (#251)
* Update faiss version in both Dockerfiles * update Faiss version to latest and add IP metric to Flat HNSW * Automated updates: Format and/or coverage --------- Co-authored-by: s-gobriel <sameh.gobriel@intel.com> Co-authored-by: sys_vdms <sys_vdms@intel.com>
1 parent 97d7167 commit 4b42654

8 files changed

+90
-6
lines changed
 

‎.github/coverage/cpp.develop.coverage_report.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ src/vcl/CustomVCL.cc 51 22 43% 55,57-58,60-63
3434
src/vcl/DescriptorSet.cc 209 155 74% 65,69-70,93-94,115-116,133,135,137,190-193,196,221-222,224-225,228-231,240-244,256,268,319-320,323,325-328,331,345-346,348-350,354-356,358,365-367,369-370,373-374
3535
src/vcl/DescriptorSetData.cc 55 47 85% 48,58,64,67,114,116-118
3636
src/vcl/Exception.cc 7 6 85% 38
37-
src/vcl/FaissDescriptorSet.cc 206 177 85% 83,115-116,132,167,187-188,204-205,224-225,238-239,245,258-259,261,272-273,279,303-304,306-307,309,372-373,379,397
37+
src/vcl/FaissDescriptorSet.cc 209 177 84% 83,115-116,132,167,187-188,204-205,224-225,238-239,245,258-259,261,272-273,279,303-304,306-307,309,372-373,379,394-396,400
3838
src/vcl/FlinngDescriptorSet.cc 149 109 73% 60-66,89,109-111,113-114,118-121,124,126,128,130,132,134-137,140-141,143-144,170-171,176-177,182,206,208,228,248,279
3939
src/vcl/Image.cc 910 689 75% 62,73-74,76-78,81-84,86,92,101,122-123,125,132-133,135,147,165,170,193,196-199,223,246,249-252,264,273,276-279,291,323,326-329,341,347,349-352,360-362,369,393-396,415,417,425,427,432,436,441,445,459,462,467-468,471-472,474,490,500,513,531,553-556,594,605-606,608,615,619,624,627-630,658-660,712,757-758,809,838-842,844-850,852,854-855,896,899-900,939-940,944-945,966,985-986,988,1028-1030,1032-1036,1038-1042,1044-1048,1050-1054,1056-1060,1062-1065,1088,1109,1128-1136,1147-1148,1167-1186,1198-1199,1207,1218,1220-1222,1224-1226,1228,1242,1246-1247,1249,1254-1255,1257,1278,1282,1285,1292,1307,1313,1322,1336,1361,1379,1462,1481
4040
src/vcl/KeyFrame.cc 303 244 80% 58,62,86,90,95,97,102,105-107,109-111,113,119,139,148,154,172,186,190,216,220,224,235,239,249,255,274,284,288,307,315,341,345,347,359,367,369,394,396,405,430,442,449,465,469,478,483,495,500,507,514,518,525,541,547,557,563
@@ -57,5 +57,5 @@ utils/src/comm/Exception.cc 6 0 0% 35-40
5757
utils/src/stats/SystemStats.cc 250 249 99% 453
5858
utils/src/timers/TimerMap.cc 82 75 91% 126,151,153,155-158
5959
------------------------------------------------------------------------------
60-
TOTAL 10243 6572 64%
60+
TOTAL 10246 6572 64%
6161
------------------------------------------------------------------------------

‎.github/coverage/cpp.develop.coverage_value.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
64.1609
1+
64.1421

‎.github/scripts/Dockerfile.checkin

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ WORKDIR /dependencies
6969
ENV AUTOCONF_VERSION="2.71" \
7070
AWS_SDK_VERSION="1.11.336" \
7171
CMAKE_VERSION="v3.28.5" \
72-
FAISS_VERSION="v1.7.4" \
72+
FAISS_VERSION="v1.9.0" \
7373
LIBEDIT_VERSION="20230828-3.1" \
7474
OPENCV_VERSION="4.9.0" \
7575
PEG_VERSION="0.1.19" \

‎docker/base/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ WORKDIR /dependencies
6969
ENV AUTOCONF_VERSION="2.71" \
7070
AWS_SDK_VERSION="1.11.336" \
7171
CMAKE_VERSION="v3.28.5" \
72-
FAISS_VERSION="v1.7.4" \
72+
FAISS_VERSION="v1.9.0" \
7373
LIBEDIT_VERSION="20230828-3.1" \
7474
OPENCV_VERSION="4.9.0" \
7575
PEG_VERSION="0.1.19" \
@@ -188,7 +188,7 @@ RUN apt-get update -y && apt-get upgrade -y && \
188188
libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libzip-dev openjdk-17-jdk-headless \
189189
procps && \
190190
apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
191-
apt-get clean && rm -rf /var/lib/apt/lists/* && \
191+
apt-get clean && rm -rf /var/lib/apt/lists/* && \
192192
echo "/usr/local/lib" >> /etc/ld.so.conf.d/all-libs.conf && ldconfig && \
193193
python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "protobuf==4.${PROTOBUF_VERSION}" \
194194
"coverage>=7.3.1" "cryptography>=42.0.7"

‎src/vcl/FaissDescriptorSet.cc

+3
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,9 @@ FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet(
391391
if (metric == L2) {
392392
_index = new faiss::IndexHNSWFlat(dim, hnsw_M, faiss::METRIC_L2);
393393
((faiss::IndexHNSWFlat *)_index)->hnsw.efConstruction = 96;
394+
} else if (metric == IP) {
395+
_index = new faiss::IndexHNSWFlat(dim, hnsw_M, faiss::METRIC_INNER_PRODUCT);
396+
((faiss::IndexHNSWFlat *)_index)->hnsw.efConstruction = 96;
394397
} else {
395398
// only metric L2 is supported for HNSWFLAT for FAISS v1.7.4
396399
// newer version of Faiss e.g. V1.8.0 supports I.P. metric for HNSW

‎tests/unit_tests/DescriptorSetAdd_test.cc

+45
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,51 @@ TEST(Descriptors_Add, add_hnswflatl2_100d_2add) {
343343
delete[] xb;
344344
}
345345

346+
TEST(Descriptors_Add, add_hnswflatip_100d) {
347+
348+
// test to add 100 descriptors of 100D each
349+
// descriptors are created by varying an init with a cyclic value
350+
// init init ... init (D times)
351+
// init.11 init.11 ... init.11 (D times)
352+
// ...
353+
// init.nb-1 init.nb-1 ... init.nb-1 (D times)
354+
// hence, nearest neigbor of any query descriptor are the IDs that is next to
355+
// the query ID
356+
357+
int d = 100;
358+
int nb = 100; // we are using 2 decimal points for I.P. nb is maximum 100
359+
float *xb = generate_desc_inner_product_increase(d, nb);
360+
361+
std::string index_filename = "dbs/add_hnswflatip_100d";
362+
VCL::DescriptorSet index(index_filename, unsigned(d), VCL::FaissHNSWFlat);
363+
364+
std::vector<long> classes(nb);
365+
366+
for (auto &str : classes) {
367+
str = 1;
368+
}
369+
370+
index.add(xb, nb, classes);
371+
372+
std::vector<float> distances;
373+
std::vector<long> desc_ids;
374+
index.search(xb, 1, 4, desc_ids, distances);
375+
376+
int exp = 0;
377+
for (auto &desc : desc_ids) {
378+
EXPECT_EQ(desc, exp++);
379+
}
380+
381+
// Check that the distance of k neighbor is always less than k+1 neighbor
382+
for (int i = 0; i < distances.size() - 1; ++i) {
383+
EXPECT_LT(distances[i], distances[i + 1]);
384+
}
385+
386+
index.store();
387+
388+
delete[] xb;
389+
}
390+
346391
// Flinng Tests
347392

348393
TEST(Descriptors_Add, add_flinngIP_100d) {

‎tests/unit_tests/helpers.cc

+31
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,37 @@ float *generate_desc_linear_increase(int d, int nb, float init) {
213213
return xb;
214214
}
215215

216+
// Functions to generate descriptors close in inner product metric
217+
// for example for d=3, nb=5 and init=1.0f
218+
// the generated vectors will be
219+
// Vector 0: [1.00, 1.00, 1.00]
220+
// Vector 1: [1.11, 1.11, 1.11]
221+
// Vector 2: [1.23, 1.23, 1.23]
222+
// Vector 3: [1.36, 1.36, 1.36]
223+
// Vector 4: [1.50, 1.50, 1.50]
224+
// in the IP domain, closest vector to descriptor K is (K+1)
225+
226+
void generate_desc_inner_product_increase(int d, int nb, float *xb,
227+
float init) {
228+
float val = init;
229+
for (int i = 0; i < nb; ++i) {
230+
for (int j = 0; j < d; ++j) {
231+
xb[i * d + j] = val;
232+
}
233+
val += 0.1f + (i * 0.01f); // Increase increment as index increases
234+
}
235+
}
236+
237+
float *generate_desc_inner_product_increase(int d, int nb, float init) {
238+
float *xb = new float[d * nb];
239+
generate_desc_inner_product_increase(d, nb, xb, init);
240+
return xb;
241+
}
242+
243+
// Functions to create a distribution of descriptors
244+
// resulting descriptors are clustered around cluster heads
245+
// nearest neighbors when the cluster head is used for query should be
246+
// the remaining descriptors within the cluster (with no strict order)
216247
void generate_desc_normal_cluster(int d, int nb, float *xb, float init,
217248
int cluster_size, float clusterhead_std,
218249
float cluster_std) {

‎tests/unit_tests/helpers.h

+5
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ void generate_desc_linear_increase(int d, int nb, float *xb, float init = 0);
6060

6161
float *generate_desc_linear_increase(int d, int nb, float init = 0);
6262

63+
void generate_desc_inner_product_increase(int d, int nb, float *xb,
64+
float init = 1.0);
65+
66+
float *generate_desc_inner_product_increase(int d, int nb, float init = 1.0);
67+
6368
void generate_desc_normal_cluster(int d, int nb, float *xb, float init = 0,
6469
int cluster_size = 5,
6570
float clusterhead_std = 1.0,

0 commit comments

Comments
 (0)