Wordlist, rules: Drop the dupe check against the previous word

This check dates back to 1990s when our primary target was descrypt with
its length truncation, its computation was much slower, and we didn't have
the generic dupe suppressor.  In the suppressor, a word matching the
previous one would be an L1 cache hit, so we don't gain much by
pre-checking for that special case.  And if the suppressor is disabled,
that's probably because we don't want any overhead like that.

Besides, this check didn't behave correctly along with an external filter()
possibly modifying the word (a bug that also dates back to 1990s).

OTOH, we do not currently support the suppressor on top of stacked rules
(we do before stacked rules), so we do lose a bit of functionality with
this change.

Author: solardiz

src/pp.c

@@ -1192,11 +1192,6 @@ void do_prince_crack(struct db_main *db, const char *wordlist, int rules)
   setmode (fileno (stdout), O_BINARY);
   #endif
 #else
-  union {
-    char buffer[LINE_BUFFER_SIZE];
-    ARCH_WORD dummy;
-  } aligned;
-  char *last = aligned.buffer;
   int loopback = (options.flags & FLG_PRINCE_LOOPBACK) ? 1 : 0;
   int mask_mult = MAX(1, mask_num_qw);
   int our_fmt_len = (db->format->params.plaintext_length + ((mask_mult - 1) * mask_add_len)) / mask_mult - mask_add_len;
@@ -1371,7 +1366,7 @@ void do_prince_crack(struct db_main *db, const char *wordlist, int rules)
     do {
       char *rule;
 
-      if ((rule = rules_reject(prerule, -1, last, db)))
+      if ((rule = rules_reject(prerule, -1, db)))
       {
         list_add(rule_list, rule);
         active_rules++;
@@ -2334,8 +2329,7 @@ void do_prince_crack(struct db_main *db, const char *wordlist, int rules)
               do {
                 char *word;
 
-                if ((word = rules_apply(pw_buf, rule->data, -1, last))) {
-                  last = word;
+                if ((word = rules_apply(pw_buf, rule->data, -1))) {
 #if HAVE_REXGEN
                   if (regex) {
                     if ((jtr_done = do_regex_hybrid_crack(db, regex, word,

src/rules.c

@@ -107,12 +107,10 @@ static struct {
  */
 	int pass;
 /*
- * Some rule commands may temporarily double the length, and we skip a few
- * machine words to avoid cache bank conflicts when copying data between the
- * buffers.  We need three buffers because some rule commands require separate
- * input and output buffers and we also need a buffer either for leaving the
- * previous mangled word intact for a subsequent comparison (in wordlist mode)
- * or for switching between two input words (in "single crack" mode).
+ * Some rule commands may temporarily double the length.
+ * We need three buffers because some rule commands require separate input and
+ * output buffers and we also need a buffer for switching between two input
+ * words in "single crack" mode.
  * rules_apply() tries to minimize data copying, and thus it may return a
  * pointer to any of the three buffers.
  *
@@ -576,7 +574,7 @@ int rules_init_stack(char *ruleset, rule_stack *stack_ctx,
 		do {
 			rule_number++;
 
-			if ((rule = rules_reject(prerule, -1, NULL, db))) {
+			if ((rule = rules_reject(prerule, -1, db))) {
 				list_add(stack_ctx->stack_rule, rule);
 				active_rules++;
 
@@ -642,7 +640,7 @@ void rules_init(struct db_main *db, int max_length)
 	rules_stacked_after = (options.flags & (FLG_RULES_CHK | FLG_SINGLE_CHK | FLG_BATCH_CHK)) && (options.flags & FLG_RULES_STACK_CHK);
 }
 
-char *rules_reject(char *rule, int split, char *last, struct db_main *db)
+char *rules_reject(char *rule, int split, struct db_main *db)
 {
 	static char out_rule[RULE_BUFFER_SIZE];
 
@@ -748,15 +746,15 @@ char *rules_reject(char *rule, int split, char *last, struct db_main *db)
 accept:
 	rules_pass--;
 	strnzcpy(out_rule, rule - 1, sizeof(out_rule));
-	rules_apply(safe_null_string, out_rule, split, last);
+	rules_apply(safe_null_string, out_rule, split);
 	rules_pass++;
 
 	return out_rule;
 }
 
 #define STACK_MAXLEN (rules_stacked_after ? RULE_WORD_SIZE : rules_max_length)
 
-char *rules_apply(char *word_in, char *rule, int split, char *last)
+char *rules_apply(char *word_in, char *rule, int split)
 {
 	union {
 		char aligned[PLAINTEXT_BUFFER_SIZE];
@@ -776,8 +774,6 @@ char *rules_apply(char *word_in, char *rule, int split, char *last)
 		memory = word = word_in;
 
 	in = buffer[0][STAGE];
-	if (in == last)
-		in = buffer[2][STAGE];
 
 	length = 0;
 	while (length < RULE_WORD_SIZE) {
@@ -797,8 +793,6 @@ char *rules_apply(char *word_in, char *rule, int split, char *last)
 		REJECT
 
 	alt = buffer[1][STAGE];
-	if (alt == last)
-		alt = buffer[2][STAGE];
 
 /*
  * This assumes that RULE_WORD_SIZE is small enough that length can't reach or
@@ -1771,22 +1765,6 @@ char *rules_apply(char *word_in, char *rule, int split, char *last)
 		length = strlen(in);
 	}
 
-	if (last) {
-		if (length > STACK_MAXLEN)
-			length = STACK_MAXLEN;
-		if (length >= ARCH_SIZE - 1) {
-			if (*(ARCH_WORD *)in != *(ARCH_WORD *)last)
-				return in;
-			if (strcmp(&in[ARCH_SIZE - 1], &last[ARCH_SIZE - 1]))
-				return in;
-			return NULL;
-		}
-		if (last[length])
-			return in;
-		if (memcmp(in, last, length))
-			return in;
-		return NULL;
-	}
 	return in;
 
 out_which:
@@ -1849,11 +1827,6 @@ int rules_advance_stack(rule_stack *ctx, int quiet)
  */
 char *rules_process_stack(char *key, rule_stack *ctx)
 {
-	static union {
-		char buf[LINE_BUFFER_SIZE];
-		ARCH_WORD dummy;
-	} aligned;
-	static char *last = aligned.buf;
 	char *word;
 
 	if (!ctx->rule) {
@@ -1865,8 +1838,7 @@ char *rules_process_stack(char *key, rule_stack *ctx)
 
 	rules_stacked_after = 0;
 
-	if ((word = rules_apply(key, ctx->rule->data, -1, last)))
-		last = word;
+	word = rules_apply(key, ctx->rule->data, -1);
 
 	rules_stacked_after = !!(options.flags & (FLG_RULES_CHK | FLG_SINGLE_CHK | FLG_BATCH_CHK));
 
@@ -1878,11 +1850,6 @@ char *rules_process_stack(char *key, rule_stack *ctx)
  */
 char *rules_process_stack_all(char *key, rule_stack *ctx)
 {
-	static union {
-		char buf[LINE_BUFFER_SIZE];
-		ARCH_WORD dummy;
-	} aligned;
-	static char *last = aligned.buf;
 	char *word;
 
 	if (!ctx->rule) {
@@ -1896,10 +1863,8 @@ char *rules_process_stack_all(char *key, rule_stack *ctx)
 	rules_stacked_after = 0;
 
 	while (ctx->rule) {
-		if ((word = rules_apply(key, ctx->rule->data, -1, last))) {
-			last = word;
+		if ((word = rules_apply(key, ctx->rule->data, -1)))
 			return word;
-		} else
 		if ((ctx->rule = ctx->rule->next)) {
 			rules_stacked_number++;
 			if (!stack_rules_mute)
@@ -1943,7 +1908,7 @@ static int rules_check(struct rpp_context *start, int split)
 
 	rules_pass = -1; /* rules_reject() will turn this into -2 */
 	while ((rule = rpp_next(&ctx))) {
-		rules_reject(rule, split, NULL, NULL);
+		rules_reject(rule, split, NULL);
 		if (rules_errno) break;
 
 		if (ctx.input) rules_line = ctx.input->number;
@@ -1981,7 +1946,7 @@ static void rules_load_normalized_list(struct cfg_line *pLine)
 			/*
 			 * this will 'reduce' the rule by stripping no-op's.
 			 */
-			char *rule = rules_reject(pLine->data, -1, NULL, NULL);
+			char *rule = rules_reject(pLine->data, -1, NULL);
 			if (rule) {
 				rules_normalize_add_line(rule, pLine->id);
 				++rules_tmp_dup_removal_cnt;

src/rules.h

@@ -60,11 +60,8 @@ extern void rules_init(struct db_main *db, int max_length);
  *
  * split == 0	"single crack" mode rules allowed
  * split < 0	"single crack" mode rules are invalid
- *
- * last may specify which internal buffer must not be touched.
  */
-extern char *rules_reject(char *rule, int split, char *last,
-	struct db_main *db);
+extern char *rules_reject(char *rule, int split, struct db_main *db);
 
 /*
  * Applies rule to a word. Returns the updated word, or NULL if rejected or
@@ -73,13 +70,8 @@ extern char *rules_reject(char *rule, int split, char *last,
  * split > 0	"single crack" mode, split is the second word's position
  * split == 0	"single crack" mode, only one word
  * split < 0	other cracking modes, "single crack" mode rules are invalid
- *
- * If last is non-NULL, it should be the previous mangled word and it is
- * assumed to be properly aligned for ARCH_WORD accesses (pointers returned by
- * rules_apply() are properly aligned).  If the new mangled word matches the
- * previous one, it will be rejected (rules_apply() will return NULL).
  */
-extern char *rules_apply(char *word, char *rule, int split, char *last);
+extern char *rules_apply(char *word, char *rule, int split);
 
 /*
  * Similar to rules_check(), but displays a message and does not return on

src/single.c

@@ -674,7 +674,7 @@ static int single_process_pw(struct db_salt *salt, struct db_password *pw,
 	do {
 		if (first == global_head)
 			first_global = 1;
-		if ((key = rules_apply(first->data, rule, 0, NULL)))
+		if ((key = rules_apply(first->data, rule, 0)))
 		if (ext_filter(key))
 		if (single_add_key(salt, key, 0))
 			return 1;
@@ -701,7 +701,7 @@ static int single_process_pw(struct db_salt *salt, struct db_password *pw,
 				strnzcpy(pair, first->data, RULE_WORD_SIZE);
 				strnzcat(pair, second->data, RULE_WORD_SIZE);
 
-				if ((key = rules_apply(pair, rule, split, NULL)))
+				if ((key = rules_apply(pair, rule, split)))
 				if (ext_filter(key))
 				if (single_add_key(salt, key, 0))
 					return 1;
@@ -716,7 +716,7 @@ static int single_process_pw(struct db_salt *salt, struct db_password *pw,
 				pair[1] = 0;
 				strnzcat(pair, second->data, RULE_WORD_SIZE);
 
-				if ((key = rules_apply(pair, rule, 1, NULL)))
+				if ((key = rules_apply(pair, rule, 1)))
 				if (ext_filter(key))
 				if (single_add_key(salt, key, 0))
 					return 1;
@@ -817,7 +817,7 @@ static void single_run(void)
 				}
 			}
 
-			if (!(rule = rules_reject(prerule, 0, NULL, single_db))) {
+			if (!(rule = rules_reject(prerule, 0, single_db))) {
 				rule_number++;
 				if (options.verbosity >= VERB_DEFAULT &&
 				    !rules_mute &&

src/wordlist.c

@@ -303,7 +303,7 @@ static double get_progress(void)
 	        (rule_count * size * mask_mult));
 }
 
-static char *dummy_rules_apply(char *word, char *rule, int split, char *last)
+static char *dummy_rules_apply(char *word, char *rule, int split)
 {
 	return word;
 }
@@ -459,18 +459,17 @@ static MAYBE_INLINE int wbuf_unique(char *line)
 void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 {
 	union {
-		char buffer[2][LINE_BUFFER_SIZE + CACHE_BANK_SHIFT];
+		char buffer[LINE_BUFFER_SIZE];
 #if MGETL_HAS_SIMD
 		vtype dummy;
 #else
 		ARCH_WORD dummy;
 #endif
 	} aligned;
-	char *line = aligned.buffer[0];
-	char *last = aligned.buffer[1];
+	char *line = aligned.buffer;
 	struct rpp_context ctx;
 	char *prerule="", *rule="", *word="";
-	char *(*apply)(char *word, char *rule, int split, char *last) = NULL;
+	char *(*apply)(char *word, char *rule, int split) = NULL;
 	int dist_switch=0;
 	uint64_t my_words=0, their_words=0, my_words_left=0;
 	int64_t i, file_len = 0;
@@ -1085,11 +1084,6 @@ void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 	if (rules)
 		prerule = rpp_next(&ctx);
 
-/* A string that can't be produced by fgetl(). */
-	last = aligned.buffer[1];
-	last[0] = '\n';
-	last[1] = 0;
-
 	dist_rules = 0;
 	dist_switch = rule_count; /* never */
 	my_words = ~0UL; /* all */
@@ -1151,7 +1145,7 @@ void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 				    for_node > options.node_max)
 					goto next_rule;
 			}
-			if ((rule = rules_reject(prerule, -1, last, db))) {
+			if ((rule = rules_reject(prerule, -1, db))) {
 				if (strcmp(prerule, rule)) {
 					if (!rules_mute)
 					log_event("- Rule #%d: '%.100s'"
@@ -1186,8 +1180,7 @@ void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 				}
 			}
 			loop_line_no++;
-			if ((word = apply(joined->data, rule, -1, last))) {
-				last = word;
+			if ((word = apply(joined->data, rule, -1))) {
 #if HAVE_REXGEN
 				if (regex) {
 					if (do_regex_hybrid_crack(db, regex,
@@ -1255,8 +1248,7 @@ void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 #endif
 			line_number++;
 
-			if ((word = apply(line, rule, -1, last))) {
-				last = word;
+			if ((word = apply(line, rule, -1))) {
 #if HAVE_REXGEN
 				if (regex) {
 					if (do_regex_hybrid_crack(db, regex,
@@ -1326,16 +1318,9 @@ void do_wordlist_crack(struct db_main *db, const char *name, int rules)
 							goto next_word;
 					}
 					line[length] = 0;
-
-					if (!strcmp(line, last))
-						goto next_word;
 				}
 
-				if ((word = apply(line, rule, -1, last))) {
-					if (rules)
-						last = word;
-					else
-						strcpy(last, word);
+				if ((word = apply(line, rule, -1))) {
 #if HAVE_REXGEN
 					if (regex) {
 						if (do_regex_hybrid_crack(