AER-6846 - Abandon secondary index population on system memory stop-writes and indexes-memory-budget for in-memory indexes.

Co-authored-by: Suresh Togas <[email protected]>

Author: srivarun91

as/include/base/datamodel.h

@@ -864,6 +864,7 @@ typedef struct as_namespace_s {
 
 	bool			stop_writes;
 	bool			hwm_breached;
+	bool			memory_breached; // stop-writes-sys-memory-pct or indexes-memory-budget
 
 	uint64_t		non_expirable_objects;
 

as/include/sindex/sindex.h

@@ -105,6 +105,7 @@ typedef struct as_sindex_s {
 
 	bool readable; // false while building sindex
 	bool dropped;
+	bool error;
 	uint32_t n_jobs;
 
 	uint64_t keys_per_bval;

as/src/base/nsup.c

@@ -933,9 +933,11 @@ eval_stop_writes(as_namespace* ns, bool cold_starting)
 	char* at = reasons;
 
 	uint32_t cfg_pct = as_load_uint32(&ns->stop_writes_sys_memory_pct);
+	bool memory_breached = false;
 
 	// Note - not >= since sys_memory_pct is rounded up, not down.
 	if (cfg_pct != 0 && sys_memory_pct > cfg_pct) {
+		memory_breached = true;
 		at = stpcpy(at, "sys-memory & ");
 	}
 
@@ -944,9 +946,12 @@ eval_stop_writes(as_namespace* ns, bool cold_starting)
 	uint64_t ixs_memory_budget = as_load_uint64(&ns->indexes_memory_budget);
 
 	if (ixs_memory_budget != 0 && ixs_sz > ixs_memory_budget) {
+		memory_breached = true;
 		at = stpcpy(at, "indexes-memory & ");
 	}
 
+	ns->memory_breached = memory_breached;
+
 	// Note - configured value 0 automatically switches this off. Also, not <=
 	// because avail vs. used, though data_avail_pct is rounded down.
 	if (data_avail_pct < ns->storage_stop_writes_avail_pct) {

as/src/sindex/populate.c

@@ -419,6 +419,15 @@ populate_reduce_cb(as_index_ref* r_ref, void* udata)
 		return false;
 	}
 
+	if (! as_namespace_sindex_persisted(ns) && ns->memory_breached) {
+		cf_warning(AS_SINDEX, "{%s} populating sindex %s - aborted due to memory limit breach",
+				ns->name, si->iname);
+		si->error = true; // an error, once set, can't be cleared until restart
+		*cbi->p_aborted = true;
+		as_record_done(r_ref, ns);
+		return false;
+	}
+
 	if (++cbi->n_reduced == PROGRESS_RESOLUTION) {
 		cbi->n_reduced = 0;
 

as/src/sindex/sindex.c

@@ -552,7 +552,10 @@ as_sindex_list_str(const as_namespace* ns, bool b64, cf_dyn_buf* db)
 			}
 		}
 
-		if (si->readable) {
+		if (si->error) {
+			cf_dyn_buf_append_string(db, ":state=ERROR");
+		}
+		else if (si->readable) {
 			cf_dyn_buf_append_string(db, ":state=RW");
 		}
 		else {