Script updates, bug updates

Author: mariadb-RoelVandePaar

BUGS/MDEV-22444.sql

@@ -13,3 +13,7 @@ USE test;
 SET IN_PREDICATE_CONVERSION_THRESHOLD=2;
 CREATE TABLE t(c BIGINT NOT NULL);
 SELECT * FROM t WHERE c IN (CURDATE(),ADDDATE(CURDATE(),'a')) ORDER BY c;
+
+SET @@in_predicate_conversion_threshold=2;
+CREATE TABLE t (a INT KEY) ENGINE=InnoDB;
+SELECT 1 FROM t WHERE ROW(a, (a,a)) IN ((1, (1,1)),(2, (2,2)));

BUGS/MDEV-29178.sql

@@ -0,0 +1,19 @@
+INSTALL SONAME 'ha_federatedx.so';
+CREATE USER federatedx@localhost IDENTIFIED BY '';
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET '../socket.sock',DATABASE 'test',user 'federatedx',PASSWORD'');
+CREATE TABLE t1 (c INT KEY,c2 BLOB,c3 TEXT) ENGINE=InnoDB;
+CREATE TABLE t2 CONNECTION='srv/t1' ENGINE=FEDERATED;
+START TRANSACTION;
+DELETE FROM t2 WHERE other=0;
+CREATE TABLE t2 (d INT) ENGINE=FEDERATED;
+INSERT INTO t2 VALUES (1);
+
+INSTALL SONAME 'ha_federatedx.so';
+CREATE USER federatedx@localhost IDENTIFIED BY '';
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET '../socket.sock',DATABASE 'test',user 'federatedx',PASSWORD'');
+CREATE TABLE t2 (c INT KEY,c2 BLOB) ENGINE=InnoDB;
+CREATE TABLE t CONNECTION='srv/t2' ENGINE=FEDERATED;
+SET autocommit=OFF;
+INSERT INTO t VALUES (1);
+CREATE TABLE t (a INT KEY,b CHAR(1)) ENGINE=FEDERATED;
+INSERT INTO t (c) VALUES (1);

BUGS/MDEV-29502.sql

@@ -15,3 +15,11 @@ CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET '../socket.sock', D
 CREATE TABLE t (c INT) ENGINE=InnoDB;
 CREATE TABLE t1 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv", TABLE "t"';
 SELECT * FROM t1 ORDER BY CAST(c AS INET6);
+
+INSTALL PLUGIN Spider SONAME 'ha_spider.so';
+CREATE USER Spider@localhost IDENTIFIED BY 'PWD123';
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET '../socket.sock', DATABASE 'test', USER 'Spider', PASSWORD 'PWD123');
+CREATE TABLE t (c1 INT) ENGINE=InnoDB;
+CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv", TABLE "t"' COMMENT='TABLE "st"' PARTITION BY LIST COLUMNS (c1) (PARTITION p1 DEFAULT COMMENT='srv "d"' ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv", TABLE "t"');
+INSERT INTO t1 VALUES (0xA9DA);
+SELECT COUNT(*) AS total_rows, MIN(c1) AS min_value, MAX(c1) FROM t1;

BUGS/MDEV-29758.sql

@@ -0,0 +1,3 @@
+INSTALL SONAME 'ha_federatedx.so';
+CREATE TABLE t1 (c1 INT) ENGINE=InnoDB;
+CREATE TABLE tf ENGINE=FEDERATED CONNECTION='mysql://[email protected]:12133/test/t1' ENGINE=FEDERATED AS SELECT 1 AS a;

federatedpreload.sql

@@ -0,0 +1,9 @@
+INSTALL SONAME 'ha_federatedx.so';
+CREATE USER federatedx@localhost IDENTIFIED BY 'PWD';
+GRANT ALL ON test.* TO federatedx@localhost;
+FLUSH PRIVILEGES;
+CREATE SERVER srv FOREIGN DATA wrapper mysql options (socket '../socket.sock', DATABASE 'test', USER 'federatedx', PASSWORD 'PWD');
+CREATE TABLE t2 (c1 INT PRIMARY KEY,c2 BLOB, c3 TEXT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0,NULL,'a');
+CREATE TABLE t1 CONNECTION='srv/t2' ENGINE=FEDERATED;
+SELECT * FROM t1;

homedir_scripts/gomd

@@ -2,13 +2,14 @@
 
 # For the BASEDIR, either specifying -opt or -dbg is fine, the script will auto-compute the other directory
 # A single opt or dbg workdir instance will tend to take 0.4-0.8 GB of space
-BASEDIR=/test/MD221022-mariadb-10.11.1-linux-x86_64-opt
+BASEDIR=/test/MD190922-mariadb-10.10.2-linux-x86_64-dbg
 #SLEEP_MD071022-mariadb-10.11.0-linux-x86_64-opt
 #MDEV-27233_MD011022-mariadb-10.7.7-linux-x86_64-dbg
 RUNSOPT=2
 RUNSDBG=2
 #CONF=pquery-run-MD-spider.conf
-CONF=pquery-run-MD-spider-all.conf
+#CONF=pquery-run-MD-spider-all.conf
+CONF=pquery-run-MD-federated.conf
 #CONF=pquery-run-MD.conf
 #CONF=pquery-run-MD-allb.conf
 

known_bugs.strings

@@ -133,6 +133,9 @@ SIGSEGV|_int_free|root_free|free_root|free_tmp_table
 !is_set() || (m_status == DA_OK_BULK && is_bulk_op())|SIGABRT|Diagnostics_area::set_ok_status|my_ok|Sql_cmd_delete::delete_from_single_table|Sql_cmd_delete::execute_inner     ## MDEV-28883
 
 ##### CURRENT BUGS ##### Search key: Mac ###
+sp > last_savepoint()|SIGABRT|federatedx_io_mysql::savepoint_set|federatedx_txn::sp_acquire|federatedx_txn::txn_begin|ha_federatedx::external_lock                             ## MDEV-29178
+sp > last_savepoint()|SIGABRT|federatedx_io_mysql::savepoint_set|federatedx_txn::sp_acquire|federatedx_txn::stmt_begin|ha_federatedx::external_lock                            ## MDEV-29178
+0|SIGABRT|Protocol::end_statement|dispatch_command|do_command|do_handle_one_connection                                                                                         ## MDEV-29758
 SIGSEGV|spider_string::length|spider_link_get_key|my_hash_key|hashcmp                                                                                                          ## MDEV-29854
 !strstr(table->name.m_name, "/FTS_") || purge_sys.must_wait_FTS()|SIGABRT|trx_t::commit|ha_innobase::delete_table|hton_drop_table|ha_delete_table                              ## MDEV-29853
 SIGSEGV|__strcmp_avx2|sp_process_definer|mysql_create_routine|mysql_execute_command                                                                                            ## MDEV-29852
@@ -192,6 +195,8 @@ SIGSEGV|Binary_string::q_append|spider_string::q_append|spider_db_mbase_util::op
 SIGSEGV|Binary_string::q_append|spider_string::q_append|spider_db_mbase_util::open_item_func|spider_mbase_handler::append_list_item_select                                     ## MDEV-29502
 SIGSEGV|Static_binary_string::q_append|spider_string::q_append|spider_db_mbase_util::open_item_func|spider_db_open_item_func                                                   ## MDEV-29502
 SIGSEGV|Static_binary_string::q_append|spider_string::q_append|spider_db_mbase_util::open_item_func|spider_mbase_handler::append_list_item_select                              ## MDEV-29502
+SIGSEGV|Binary_string::q_append|spider_string::q_append|spider_db_mbase_row::append_to_str|spider_db_mbase_row::append_to_str                                                  ## MDEV-29502
+SIGSEGV|Static_binary_string::q_append|spider_string::q_append|spider_db_mbase_row::append_to_str|spider_db_mbase_row::append_to_str                                           ## MDEV-29502
 share->tdc->flushed|SIGABRT|close_all_tables_for_name|handle_alter_part_error|fast_alter_partition_table|mysql_alter_table                                                     ## MDEV-29503
 !trx_free || !trx->locked_connections|SIGABRT|spider_free_trx_conn|spider_free_trx_alloc|spider_free_trx|spider_close_connection                                               ## MDEV-29484
 SIGSEGV|spider_db_mbase_util::append_table|spider_db_mbase_util::append_tables_top_down|spider_db_mbase_util::append_embedding_tables|spider_db_mbase_util::append_table       ## MDEV-29163
@@ -940,6 +945,9 @@ SIGSEGV|Item::val_uint|handle_select|execute_sqlcom_select|mysql_execute_command
 SIGSEGV|__strlen_avx2|Json_writer::add_str|Json_value_helper::add_str|Json_writer_object::add                                                                                  ## MDEV-22444
 SIGSEGV|__strlen_avx2|String_list::append_str|Explain_index_use::set|st_join_table::save_explain_data                                                                          ## MDEV-22444
 SIGSEGV|Field_iterator_table::create_item|Field_iterator_table_ref::create_item|insert_fields|setup_wild                                                                       ## MDEV-22444
+SIGSEGV|Item::operator new|Type_handler_json_common::make_json_valid_expr|JOIN::transform_in_predicates_into_in_subq|JOIN::optimize_inner                                      ## MDEV-22444
+SIGABRT|__cxa_pure_virtual|Item::real_type|Value_source::Context::Context|Item_func_in::propagate_equal_fields                                                                 ## MDEV-22444
+0|SIGABRT|Type_handler_row::subquery_type_allows_materialization|cmp_row_types|Item_func_in::in_predicate_to_in_subs_transformer|Item_func::transform                          ## MDEV-22444
 templ->mbmaxlen > templ->mbminlen || templ->mysql_col_len == len || (field_no == templ->icp_rec_field_no && field->prefix_len > 0)|SIGABRT|row_sel_field_store_in_mysql_format_func|row_sel_store_mysql_field|row_sel_store_mysql_rec|row_search_mvcc ## MDEV-22395
 0|SIGABRT|stored_field_cmp_to_item|convert_const_to_int|Item_func::convert_const_compared_to_int_field|Item_func::setup_args_and_comparator                                    ## MDEV-22385
 SIGSEGV|mysql_alter_table|Sql_cmd_alter_table::execute|mysql_execute_command|mysql_parse                                                                                       ## MDEV-22381

new_text_string.sh

@@ -284,10 +284,11 @@ rm -f /tmp/${RANDF}.gdb1
 
 # Stack catch
 IMPROVE_FLAG=''
-grep --binary-files=text -A100 'signal handler called' /tmp/${RANDF}.gdb2 | grep --binary-files=text -vE 'std::terminate.*from|__GI_raise |__GI_abort |__assert_fail_base |__GI___assert_fail |memmove|memcpy|\?\? \(\)|\(gdb\)|signal handler called' | sed 's|^#[0-9]\+[ \t]\+||' | sed 's|(.*) at ||;s|:[ 0-9]\+$||' > /tmp/${RANDF}.gdb4
+FRAMES_FILTER='std::terminate.*from|__GI_raise |__GI_abort |__assert_fail_base |__GI___assert_fail |memmove|memcpy|\?\? \(\)|\(gdb\)|signal handler called|uw_update_context_1|uw_init_context_1|_Unwind_Resume'
+grep --binary-files=text -A100 'signal handler called' /tmp/${RANDF}.gdb2 | grep --binary-files=text -vEi "${FRAMES_FILTER}" | sed 's|^#[0-9]\+[ \t]\+||' | sed 's|(.*) at ||;s|:[ 0-9]\+$||' > /tmp/${RANDF}.gdb4
 if [ "$(wc -m /tmp/${RANDF}.gdb4 2>/dev/null | sed 's| .*||')" == "0" ]; then
   # 'signal handler called' was not found, try another method
-  grep --binary-files=text -m1 -A100 '^#0' /tmp/${RANDF}.gdb2 | grep --binary-files=text -vE 'std::terminate.*from|__GI_raise |__GI_abort |__assert_fail_base |__GI___assert_fail |memmove|memcpy|\?\? \(\)|\(gdb\)|signal handler called' | sed 's|^#[0-9]\+[ \t]\+||' | sed 's|(.*) at ||;s|:[ 0-9]\+$||' > /tmp/${RANDF}.gdb4
+  grep --binary-files=text -m1 -A100 '^#0' /tmp/${RANDF}.gdb2 | grep --binary-files=text -vEi "${FRAMES_FILTER}" | sed 's|^#[0-9]\+[ \t]\+||' | sed 's|(.*) at ||;s|:[ 0-9]\+$||' > /tmp/${RANDF}.gdb4
 fi
 if [ "$(wc -m /tmp/${RANDF}.gdb4 2>/dev/null | sed 's| .*||')" == "0" ]; then
   # '#0' was not found, which is unlikely to happen, but improvements may be possible, set flag
@@ -314,7 +315,7 @@ fi
 rm -f /tmp/${RANDF}.gdb4
 
 # Grap first 4 frames, if they exist, and add to TEXT
-FRAMES="$(cat /tmp/${RANDF}.gdb3 | head -n4 | sed 's| [^ ]\+$||' | tr '\n' '|' | sed 's/|$/\n/')"
+FRAMES="$(cat /tmp/${RANDF}.gdb3 | head -n4 | sed 's| [^ ]\+$||;s|[ ]*(.*||' | tr '\n' '|' | sed 's/|$/\n/')"
 rm -f /tmp/${RANDF}.gdb3
 if [ ! -z "${FRAMES}" ]; then
   if [ ${FRAMESONLY} -eq 1 -o -z "${TEXT}" ]; then

pquery-go-expert.sh

@@ -9,7 +9,7 @@
 # MULTI_THREADS is set to 3
 # MULTI_THREADS_INCREASE is set to 3
 # MULTI_THREADS_MAX is set to 9
-# STAGE1_LINES is set to 7  # This was previously 13, which is better for stable systems, as it will allow reducer to continue towards auto pquery-go-expert.sh. Auto-ge (i.e. all other stages after stage 1, as can also be set/done by calling ~/ge) is good for non-sporadic issues as it will drop any uncessary lines after stage 1 in stage 2. However, it is not good for sporadic issues as this will leave regularly 5-10 lines in the testcase which are not needed, requiring one to lower the number of STAGE1_LINES and re-running reducer. The tradeoff however is that one needs to be more diligent in checking runs and regularly CTRL+C > ~/ge for trials which have reduced to a large number of lines. To reach some 'ideal' tradeoff between the two, for the moment 5 was chosen. Updated to 7 on 25/9/21 to better cater for tests which exactly require 5 lines. This will still require some trials (i.e. the non-sporadic ones) to be CTRL+C > ~/ge'd, and some trials (the sporadic ones) where STAGE1_LINES needs to be set lower. Ideally, at some point, an auto-restart-reducers script may be best where FORCE_SKIPV is tested and changes are made based on the result. Thinking about it, it may be better to include this functionality in reducer itself. The risk is that the issue does not reproduce even on 50 threads (auto-increased). To counter this, another type of STAGE V may be implemented; one which executes the testcase up to 10000 times. This is slow however, and then perhas the current system is best; rely on the skill of the engineer to see difference between sporadic/non-sporadic.
+# STAGE1_LINES is set to 13  # This was previously 13, which is better for stable systems, as it will allow reducer to continue towards auto pquery-go-expert.sh. Auto-ge (i.e. all other stages after stage 1, as can also be set/done by calling ~/ge) is good for non-sporadic issues as it will drop any uncessary lines after stage 1 in stage 2. However, it is not good for sporadic issues as this will leave regularly 5-10 lines in the testcase which are not needed, requiring one to lower the number of STAGE1_LINES and re-running reducer. The tradeoff however is that one needs to be more diligent in checking runs and regularly CTRL+C > ~/ge for trials which have reduced to a large number of lines. To reach some 'ideal' tradeoff between the two, for the moment 5 was chosen. Updated to 7 on 25/9/21 to better cater for tests which exactly require 5 lines. This will still require some trials (i.e. the non-sporadic ones) to be CTRL+C > ~/ge'd, and some trials (the sporadic ones) where STAGE1_LINES needs to be set lower. Ideally, at some point, an auto-restart-reducers script may be best where FORCE_SKIPV is tested and changes are made based on the result. Thinking about it, it may be better to include this functionality in reducer itself. The risk is that the issue does not reproduce even on 50 threads (auto-increased). To counter this, another type of STAGE V may be implemented; one which executes the testcase up to 10000 times. This is slow however, and then perhas the current system is best; rely on the skill of the engineer to see difference between sporadic/non-sporadic.
 # INPUTFILE is auto-optimized towards latest sql trace inc _out* handling
 # The effect of FORCE_SKIPV=1 is that reducer will skip the verify stage, start reduction immediately, using 3 threads (MULTI_THREADS=3), and never increases the set amount of
 # threads (result of using FORCE_SKIPV=1). Note that MULTI_THREADS_INCREASE is only relevant for non-FORCE_SKIPV runs, more on why this is changed then below.