Skip to content

Commit d864c9d

Browse files
shaharuk-ybshaharuk-yb
authored
Perf-data-loader: improved loader script and support for user defined enum types (#143)
* use table schema to fetch table details, improved perf-data-loader wrapper file, improved load order
1 parent b264719 commit d864c9d

File tree

4 files changed

+325
-147
lines changed

4 files changed

+325
-147
lines changed

‎perf-data-loader‎

Lines changed: 132 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ LOAD_ONLY=false
99
GEN_LOAD_ORDER=false
1010
JAR_PATH=""
1111

12+
# ANSI color codes
13+
GREEN="\033[32m"
14+
RED="\033[31m"
15+
RESET="\033[0m"
16+
1217
# Function to display help
1318
function display_help {
1419
echo "Usage: $0 --config <config_file> --table-name <table_name> --rows <rows> [--gen-config-only] [--load-only] [--gen-load-order]"
@@ -48,6 +53,49 @@ function find_jar {
4853
exit 1
4954
}
5055

56+
# Function to show a fake progress bar while a command is running
57+
show_progress_bar() {
58+
local CMD_PID=$1
59+
local SECONDS=0
60+
61+
while kill -0 "$CMD_PID" 2>/dev/null; do
62+
printf "\r["
63+
for ((i=0; i<50; i++)); do
64+
if (( i < (SECONDS % 50) )); then
65+
printf "="
66+
else
67+
printf " "
68+
fi
69+
done
70+
printf "] %d%%" $((SECONDS % 50 * 2))
71+
sleep 0.1
72+
done
73+
}
74+
75+
# Function to check command success or failure
76+
# Function to check command success or failure
77+
check_command_status() {
78+
local CMD_PID=$1
79+
local SUCCESS_MSG=$2
80+
local FAILURE_MSG=$3
81+
local LOG_FILE=$4
82+
83+
wait "$CMD_PID"
84+
CMD_EXIT_CODE=$?
85+
# Clear the progress bar line
86+
printf "\r\033[K"
87+
if [ $CMD_EXIT_CODE -eq 0 ]; then
88+
# Ensure the progress bar finishes at 100% on success
89+
printf "\r[==================================================] 100%%\n"
90+
echo -e "${GREEN}$SUCCESS_MSG${RESET}"
91+
else
92+
# Avoid showing 100% if the command fails
93+
echo -e "${RED}$FAILURE_MSG${RESET}"
94+
echo -e "${RED}\nConsole log:${RESET}"
95+
cat "$LOG_FILE"
96+
fi
97+
}
98+
5199
# Parse arguments
52100
while [[ $# -gt 0 ]]; do
53101
case $1 in
@@ -104,8 +152,23 @@ if [ "$LOAD_ONLY" = true ]; then
104152
echo "Error: --config parameter is required with --load-only."
105153
display_help
106154
fi
107-
echo "Loading the data into the database now!"
108-
java -jar "$JAR_PATH" -b featurebench -c "$CONFIG" --load=True
155+
156+
# Create a temporary file to capture output and errors
157+
LOG_FILE=$(mktemp)
158+
echo "Loading the data into the database now..."
159+
# Start the Java command in the background
160+
java -jar "$JAR_PATH" -b featurebench -c "$CONFIG" --load=True > "$LOG_FILE" 2>&1 &
161+
CMD_PID=$!
162+
# Show the progress bar while the command is running
163+
show_progress_bar "$CMD_PID"
164+
# Check if the command was successful and print the appropriate message
165+
check_command_status "$CMD_PID" \
166+
"Data load to the table \`${TABLE_NAME}\` is successful." \
167+
"Failed to load data into the table: ${TABLE_NAME}." \
168+
"$LOG_FILE"
169+
170+
# Clean up the log file
171+
rm "$LOG_FILE"
109172
exit 0
110173
fi
111174

@@ -114,8 +177,22 @@ if [ "$GENERATE_ONLY" = true ]; then
114177
echo "Error: --config, --table-name, and --rows parameters are required with --gen-config-only."
115178
display_help
116179
fi
117-
echo "Generating loader file for the table"
118-
java -jar "$JAR_PATH" -b perf-dataloader -c "$CONFIG" -p tableName="$TABLE_NAME" -p rows="$ROWS" --load=True
180+
# Create a temporary file to capture output and errors
181+
LOG_FILE=$(mktemp)
182+
echo "Generating loader file for the table..."
183+
# Start the Java command in the background
184+
java -jar "$JAR_PATH" -b perf-dataloader -c "$CONFIG" -p tableName="$TABLE_NAME" -p rows="$ROWS" --load=True > "$LOG_FILE" 2>&1 &
185+
CMD_PID=$!
186+
# Show the progress bar while the command is running
187+
show_progress_bar "$CMD_PID"
188+
# Check if the command was successful and print the appropriate message
189+
check_command_status "$CMD_PID" \
190+
"Loader file generated successfully: ${TABLE_NAME}_loader.yaml" \
191+
"Failed to generate loader file." \
192+
"$LOG_FILE"
193+
194+
# Clean up the log file
195+
rm "$LOG_FILE"
119196
exit 0
120197
fi
121198

@@ -126,18 +203,33 @@ if [ "$GEN_LOAD_ORDER" = true ]; then
126203
fi
127204
# Create a temporary file
128205
TEMP_CONFIG=$(mktemp)
206+
# Create a temporary file to capture output and errors
207+
LOG_FILE=$(mktemp)
129208
# Copy the content of the original CONFIG file to the temporary file
130209
cp "$CONFIG" "$TEMP_CONFIG"
131210

132211
# Append text to the temporary config file (you can customize the text as needed)
133212
echo >> "$TEMP_CONFIG"
134213
echo "gen-db-load-order: true" >> "$TEMP_CONFIG"
135-
echo "Generating load order based on the provided config file"
136-
java -jar "$JAR_PATH" -b perf-dataloader -c "$TEMP_CONFIG" -p tableName="dummy" -p rows="1" --load=True
137-
214+
echo "Generating load order based on the provided config file..."
215+
java -jar "$JAR_PATH" -b perf-dataloader -c "$TEMP_CONFIG" -p tableName="dummy" -p rows="1" --load=True > "$LOG_FILE" 2>&1 &
216+
CMD_PID=$!
217+
# Show the progress bar while the command is running
218+
show_progress_bar "$CMD_PID"
219+
# Check if the command was successful and print the appropriate message
220+
check_command_status "$CMD_PID" \
221+
"Load order generated successfully in file \`load_order.json\` and is as follows: " \
222+
"Failed to generate load order for the database." \
223+
"$LOG_FILE"
224+
# If successful, print the contents of the load order file
225+
if [ $CMD_EXIT_CODE -eq 0 ]; then
226+
cat "load_order.json"
227+
echo ""
228+
fi
138229
# Clean up: delete the temporary config file
139230
rm -f "$TEMP_CONFIG"
140-
231+
# Clean up the log file
232+
rm "$LOG_FILE"
141233
exit 0
142234
fi
143235

@@ -149,10 +241,38 @@ if [ -z "$CONFIG" ] || [ -z "$TABLE_NAME" ] || [ -z "$ROWS" ]; then
149241
fi
150242

151243
# If no specific option is provided, do both generate and load with default config file location
152-
echo "Generating loader file for the table"
153-
java -jar "$JAR_PATH" -b perf-dataloader -c "$CONFIG" -p tableName="$TABLE_NAME" -p rows="$ROWS" --load=True
244+
# Create a temporary file to capture output and errors
245+
LOG_FILE=$(mktemp)
246+
echo "Generating loader file for the table..."
247+
# Start the Java command in the background
248+
java -jar "$JAR_PATH" -b perf-dataloader -c "$CONFIG" -p tableName="$TABLE_NAME" -p rows="$ROWS" --load=True > "$LOG_FILE" 2>&1 &
249+
CMD_PID=$!
250+
# Show the progress bar while the command is running
251+
show_progress_bar "$CMD_PID"
252+
# Check if the command was successful and print the appropriate message
253+
check_command_status "$CMD_PID" \
254+
"Loader file generated successfully: ${TABLE_NAME}_loader.yaml" \
255+
"Failed to generate loader file." \
256+
"$LOG_FILE"
257+
258+
# Clean up the log file
259+
rm "$LOG_FILE"
154260

155261
DEFAULT_CONFIG="${TABLE_NAME}_loader.yaml"
156262

157-
echo "Loading the data into the database now!"
158-
java -jar "$JAR_PATH" -b featurebench -c "$DEFAULT_CONFIG" --load=True
263+
# Create a temporary file to capture output and errors
264+
LOG_FILE=$(mktemp)
265+
echo "Loading the data into the database now..."
266+
# Start the Java command in the background
267+
java -jar "$JAR_PATH" -b featurebench -c "$DEFAULT_CONFIG" --load=True > "$LOG_FILE" 2>&1 &
268+
CMD_PID=$!
269+
# Show the progress bar while the command is running
270+
show_progress_bar "$CMD_PID"
271+
# Check if the command was successful and print the appropriate message
272+
check_command_status "$CMD_PID" \
273+
"Data load to the table \`${TABLE_NAME}\` is successful." \
274+
"Failed to load data into the table: ${TABLE_NAME}." \
275+
"$LOG_FILE"
276+
277+
# Clean up the log file
278+
rm "$LOG_FILE"

0 commit comments

Comments
 (0)