build: tools-2643 add make shared, builds asbackup as a dynamic lib (#66

) * build: add make shared, builds asbackup as a dynamic lib * feat: expose restore_run and backup_run for dynamic lib use * tools-2667 set s3 configs in run functions Co-authored-by: Jesse S <[email protected]> --------- Co-authored-by: Jesse S <[email protected]>
aerospike · Sep 18, 2023 · 217d27b · 217d27b
1 parent 5ec01ff
commit 217d27b
Show file tree

Hide file tree

Showing 8 changed files with 252 additions and 130 deletions.
diff --git a/Makefile b/Makefile
@@ -48,6 +48,15 @@ ifdef M1_HOME_BREW
   OPENSSL_PREFIX = /opt/homebrew/opt/openssl
 endif
 
+ifeq ($(OS),Darwin)
+  DYNAMIC_SUFFIX = dylib
+  DYNAMIC_FLAG = -dynamiclib
+else
+  DYNAMIC_SUFFIX = so
+  DYNAMIC_FLAG = -shared
+endif
+DYNAMIC_OPTIONS =
+
 CC ?= cc
 
 DWARF := $(shell $(CC) -Wall -Wextra -O2 -o /tmp/asflags_$${$$} src/flags.c; \
@@ -279,6 +288,9 @@ BACKUP := $(DIR_BIN)/asbackup
 RESTORE := $(DIR_BIN)/asrestore
 TOML := $(DIR_TOML)/libtoml.a
 
+BACKUP_DYNAMIC := $(DIR_BIN)/asbackup.$(DYNAMIC_SUFFIX)
+RESTORE_DYNAMIC := $(DIR_BIN)/asrestore.$(DYNAMIC_SUFFIX)
+
 SRCS := $(BACKUP_SRC) $(RESTORE_SRC)
 OBJS := $(BACKUP_OBJ) $(RESTORE_OBJ)
 DEPS := $(BACKUP_DEP) $(RESTORE_DEP)
@@ -320,11 +332,24 @@ TEST_DEPS := $(sort $(TEST_DEPS))
 .PHONY: all
 all: $(BINS)
 
+# used as a pre-requisite for make shared
+# this rule is not meant for manual use by a user
+.PHONY: _set_dynamic_options
+_set_dynamic_options: $(TOML)
+	$(eval DYNAMIC_OPTIONS = -fPIC)
+
+# builds asbackup and asrestore as shared libraries
+# asbackup is designed as a standalone exe, use at your own risk
+# run this with the same options you would use in a normal build
+.PHONY: shared
+shared: _set_dynamic_options $(BACKUP_DYNAMIC) $(RESTORE_DYNAMIC)
+	$(eval DYNAMIC_OPTIONS =)
+
 .PHONY: clean
 clean:
 	$(MAKE) -C $(DIR_TOML) clean
 	$(MAKE) -C $(DIR_C_CLIENT) clean
-	rm -f $(DEPS) $(OBJS) $(BINS) $(TEST_OBJS) $(TEST_DEPS) $(TEST_BINS)
+	rm -f $(DEPS) $(OBJS) $(BINS) $(TEST_OBJS) $(TEST_DEPS) $(TEST_BINS) $(BACKUP_DYNAMIC) $(RESTORE_DYNAMIC)
 	if [ -d $(DIR_OBJ) ]; then rmdir $(DIR_OBJ); fi
 	if [ -d $(DIR_BIN) ]; then rmdir $(DIR_BIN); fi
 	if [ -d $(DIR_TEST_OBJ) ]; then rm -r $(DIR_TEST_OBJ); fi
@@ -367,17 +392,23 @@ $(DIR_BIN):
 	mkdir $(DIR_BIN)
 
 $(DIR_OBJ)/%_c.o: $(DIR_SRC)/%.c | $(DIR_OBJ)
-	$(CC) $(CFLAGS) -MMD -o $@ -c $(INCLUDES) $<
+	$(CC) $(DYNAMIC_OPTIONS) $(CFLAGS) -MMD -o $@ -c $(INCLUDES) $<
 
 $(DIR_OBJ)/%_cc.o: $(DIR_SRC)/%.cc | $(DIR_OBJ)
-	$(CXX) $(CXXFLAGS) -MMD -o $@ -c $(INCLUDES) $<
+	$(CXX) $(DYNAMIC_OPTIONS) $(CXXFLAGS) -MMD -o $@ -c $(INCLUDES) $<
 
 $(BACKUP): $(BACKUP_OBJ) $(TOML) $(C_CLIENT_LIB) | $(DIR_BIN)
 	$(CXX) $(LDFLAGS) -o $(BACKUP) $(BACKUP_OBJ) $(LIBRARIES)
 
 $(RESTORE): $(RESTORE_OBJ) $(TOML) $(C_CLIENT_LIB) | $(DIR_BIN)
 	$(CXX) $(LDFLAGS) -o $(RESTORE) $(RESTORE_OBJ) $(LIBRARIES)
 
+$(BACKUP_DYNAMIC): $(BACKUP_OBJ) $(TOML) $(C_CLIENT_LIB) | $(DIR_BIN)
+	$(CXX) $(DYNAMIC_FLAG) $(LDFLAGS) -o $(BACKUP_DYNAMIC) $(BACKUP_OBJ) $(LIBRARIES)
+
+$(RESTORE_DYNAMIC): $(RESTORE_OBJ) $(TOML) $(C_CLIENT_LIB) | $(DIR_BIN)
+	$(CXX) $(DYNAMIC_FLAG) $(LDFLAGS) -o $(RESTORE_DYNAMIC) $(RESTORE_OBJ) $(LIBRARIES)
+
 $(TOML):
 	$(MAKE) -C $(DIR_TOML)
 

diff --git a/README.md b/README.md
@@ -41,7 +41,7 @@ apt-get update
 apt-get install build-essential libssl-dev libuv1-dev libcurl4-openssl-dev libzstd-dev
 
 # for aws-sdk-cpp build
-apt-get install cmake
+apt-get install cmake pkg-config zlib1g-dev
 
 # download aws sdk
 git clone https://github.com/aws/aws-sdk-cpp.git
@@ -96,7 +96,7 @@ mkdir build
 cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DBUILD_SHARED_LIBS=ON -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=lib
 make -C build
 
-# install aws static sdk
+# install aws dynamic sdk
 cd build
 make install
 cd ../..

diff --git a/include/backup.h b/include/backup.h
@@ -71,6 +71,9 @@ extern "C" {
 // Estimate total backup file sizes with 99.9% confidence.
 #define BACKUP_FILE_ESTIMATE_CONFIDENCE_LEVEL 0.999
 
+#define RUN_BACKUP_SUCCESS ((void*) 0)
+#define RUN_BACKUP_FAILURE ((void*) -1lu)
+
 /*
  * The struct used to maintain state information about a backup file which was
  * not completely filled from a backup task
@@ -87,6 +90,7 @@ typedef struct queued_backup_fd {
 //
 
 int32_t backup_main(int32_t argc, char **argv);
+backup_status_t* backup_run(backup_config_t* conf);
 
 /*
  * Returns the backup config/status struct being used by the currently running

diff --git a/include/restore.h b/include/restore.h
@@ -66,6 +66,7 @@ extern "C" {
 
 // The interval for logging per-thread timing stats.
 #define STAT_INTERVAL 10
+#define RUN_RESTORE_FAILURE ((void*) -1lu)
 
 /*
  * The backup file information pushed to the job queue and picked up by the restore threads.
@@ -142,6 +143,7 @@ typedef enum {
 //
 
 int32_t restore_main(int32_t argc, char **argv);
+restore_status_t* restore_run(restore_config_t *conf);
 
 #ifdef __cplusplus
 }

diff --git a/src/backup.c b/src/backup.c
@@ -50,15 +50,11 @@ typedef struct backup_globals {
 // of the vector).
 static as_vector g_globals;
 
-#define RUN_BACKUP_SUCCESS ((void*) 0)
-#define RUN_BACKUP_FAILURE ((void*) -1lu)
-
-
 //==========================================================
 // Forward Declarations.
 //
 
-static backup_status_t* run_backup(backup_config_t* conf);
+static backup_status_t* start_backup(backup_config_t* conf);
 
 typedef struct distr_stats {
 	uint64_t total;
@@ -195,6 +191,7 @@ static void show_estimate(FILE* mach_fd, uint64_t* samples, uint32_t n_samples,
 		uint64_t rec_count_estimate, io_write_proxy_t* fd);
 static void sig_hand(int32_t sig);
 static void no_op(int32_t sig);
+static void set_s3_configs(const backup_config_t*);
 
 
 //==========================================================
@@ -220,7 +217,7 @@ backup_main(int32_t argc, char **argv)
 		goto cleanup;
 	}
 
-	backup_status_t* status = run_backup(&conf);
+	backup_status_t* status = start_backup(&conf);
 	if (status == RUN_BACKUP_SUCCESS) {
 		res = EXIT_SUCCESS;
 	}
@@ -234,14 +231,37 @@ backup_main(int32_t argc, char **argv)
 
 cleanup:
 	file_proxy_cloud_shutdown();
-
 	as_vector_destroy(&g_globals);
-
 	ver("Exiting with status code %d", res);
-
 	return res;
 }
 
+/*
+ * FOR USE WITH ASBACKUP AS A LIBRARY (Use at your own risk)
+ *
+ * Runs a backup job with the given configuration. This method is not thread
+ * safe and should not be called multiple times in parallel, as it uses global
+ * variables to handle signal interruption.
+ * 
+ * The passed in backup config must be destroyed by the caller using backup_config_destroy()
+ * To enable C client logging, call enable_client_log() before calling this function
+ * 
+ * Returns the backup_status struct used during the run which must be freed by the
+ * caller using backup_status_destroy(), then free().
+ * Only free the return value if it is != RUN_BACKUP_FAILURE || != RUN_BACKUP_SUCCESS
+ */
+backup_status_t*
+backup_run(backup_config_t* conf) {
+	as_vector_init(&g_globals, sizeof(backup_globals_t), 1);
+
+	backup_status_t* status = start_backup(conf);
+
+	file_proxy_cloud_shutdown();
+	as_vector_destroy(&g_globals);
+
+	return status;
+}
+
 backup_config_t*
 get_g_backup_conf(void)
 {
@@ -272,7 +292,7 @@ get_g_backup_status(void)
  * caller).
  */
 static backup_status_t*
-run_backup(backup_config_t* conf)
+start_backup(backup_config_t* conf)
 {
 	int32_t res = EXIT_FAILURE;
 	bool do_backup_save_state = false;
@@ -281,6 +301,8 @@ run_backup(backup_config_t* conf)
 
 	push_backup_globals(conf, NULL);
 
+	set_s3_configs(conf);
+
 	if (conf->remove_artifacts) {
 
 		if (conf->output_file != NULL) {
@@ -491,14 +513,14 @@ run_backup(backup_config_t* conf)
 
 			bool cur_silent_val = g_silent;
 			g_silent = true;
-			backup_status_t* estimate_status = run_backup(estimate_conf);
+			backup_status_t* estimate_status = start_backup(estimate_conf);
 			g_silent = cur_silent_val;
 
 			backup_config_destroy(estimate_conf);
 			cf_free(estimate_conf);
 
 			// re-enable signal handling, since it was disabled at the end of
-			// the estimate run in run_backup.
+			// the estimate run in start_backup.
 			set_sigaction(sig_hand);
 
 			if (estimate_status == RUN_BACKUP_FAILURE) {
@@ -2558,3 +2580,23 @@ no_op(int32_t sig)
 	(void) sig;
 }
 
+static void
+set_s3_configs(const backup_config_t* conf)
+{
+	if (conf->s3_region != NULL) {
+		s3_set_region(conf->s3_region);
+	}
+
+	if (conf->s3_profile != NULL) {
+		s3_set_profile(conf->s3_profile);
+	}
+
+	if (conf->s3_endpoint_override != NULL) {
+		s3_set_endpoint(conf->s3_endpoint_override);
+	}
+
+	s3_set_max_async_downloads(conf->s3_max_async_downloads);
+	s3_set_max_async_uploads(conf->s3_max_async_uploads);
+	s3_set_connect_timeout_ms(conf->s3_connect_timeout);
+	s3_set_log_level(conf->s3_log_level);
+}
diff --git a/src/backup_config.c b/src/backup_config.c
@@ -744,23 +744,6 @@ backup_config_init(int argc, char* argv[], backup_config_t* conf)
 		return BACKUP_CONFIG_INIT_FAILURE;
 	}
 
-	if (conf->s3_region != NULL) {
-		s3_set_region(conf->s3_region);
-	}
-
-	if (conf->s3_profile != NULL) {
-		s3_set_profile(conf->s3_profile);
-	}
-
-	if (conf->s3_endpoint_override != NULL) {
-		s3_set_endpoint(conf->s3_endpoint_override);
-	}
-
-	s3_set_max_async_downloads(conf->s3_max_async_downloads);
-	s3_set_max_async_uploads(conf->s3_max_async_uploads);
-	s3_set_connect_timeout_ms(conf->s3_connect_timeout);
-	s3_set_log_level(conf->s3_log_level);
-
 	if (conf->estimate) {
 		if (conf->filter_exp != NULL || conf->node_list != NULL ||
 				conf->mod_after > 0 || conf->mod_before > 0 || conf->ttl_zero ||