gufi_rollup --delete-below <level>

users probably won't be querying starting at a low level, so delete
rollup data under provided level to save storage space

can combine with --min-level to limit storing rollup data between 2 levels

Author: calccrypto

include/bf.h

@@ -171,6 +171,7 @@ extern "C" {
 #define FLAG_GROUP_MEM     2000
 #define FLAG_GROUP_XATTRS  3000
 #define FLAG_GROUP_INC     4000
+#define FLAG_GROUP_ROLLUP  5000
 
 /* miscellaneous flags */
 
@@ -218,35 +219,31 @@ extern "C" {
 #define FLAG_TERSE_LONG "terse"
 #define FLAG_TERSE {FLAG_TERSE_LONG, no_argument, NULL, FLAG_TERSE_SHORT}
 
-#define FLAG_ROLLUP_LIMIT_SHORT (FLAG_GROUP_MISC + 11)
-#define FLAG_ROLLUP_LIMIT_LONG "limit"
-#define FLAG_ROLLUP_LIMIT {FLAG_ROLLUP_LIMIT_LONG, required_argument, NULL, FLAG_ROLLUP_LIMIT_SHORT}
-
-#define FLAG_DONT_REPROCESS_SHORT (FLAG_GROUP_MISC + 12)
+#define FLAG_DONT_REPROCESS_SHORT (FLAG_GROUP_MISC + 11)
 #define FLAG_DONT_REPROCESS_LONG "dont-reprocess"
 #define FLAG_DONT_REPROCESS {FLAG_DONT_REPROCESS_LONG, no_argument, NULL, FLAG_DONT_REPROCESS_SHORT}
 
-#define FLAG_NEWLINE_SHORT (FLAG_GROUP_MISC + 13)
+#define FLAG_NEWLINE_SHORT (FLAG_GROUP_MISC + 12)
 #define FLAG_NEWLINE_LONG "newline"
 #define FLAG_NEWLINE {FLAG_NEWLINE_LONG, required_argument, NULL, FLAG_NEWLINE_SHORT}
 
-#define FLAG_SUPPRESS_NEWLINE_SHORT (FLAG_GROUP_MISC + 14)
+#define FLAG_SUPPRESS_NEWLINE_SHORT (FLAG_GROUP_MISC + 13)
 #define FLAG_SUPPRESS_NEWLINE_LONG "suppress-newline"
 #define FLAG_SUPPRESS_NEWLINE {FLAG_SUPPRESS_NEWLINE_LONG, no_argument, NULL, FLAG_SUPPRESS_NEWLINE_SHORT}
 
-#define FLAG_DIR_MATCH_UID_SHORT (FLAG_GROUP_MISC + 15)
+#define FLAG_DIR_MATCH_UID_SHORT (FLAG_GROUP_MISC + 14)
 #define FLAG_DIR_MATCH_UID_LONG "dir-match-uid"
 #define FLAG_DIR_MATCH_UID {FLAG_DIR_MATCH_UID_LONG, optional_argument, NULL, FLAG_DIR_MATCH_UID_SHORT}
 
-#define FLAG_DIR_MATCH_GID_SHORT (FLAG_GROUP_MISC + 16)
+#define FLAG_DIR_MATCH_GID_SHORT (FLAG_GROUP_MISC + 15)
 #define FLAG_DIR_MATCH_GID_LONG "dir-match-gid"
 #define FLAG_DIR_MATCH_GID {FLAG_DIR_MATCH_GID_LONG, optional_argument, NULL, FLAG_DIR_MATCH_GID_SHORT}
 
-#define FLAG_PRINT_EACCES_SHORT (FLAG_GROUP_MISC + 17)
+#define FLAG_PRINT_EACCES_SHORT (FLAG_GROUP_MISC + 16)
 #define FLAG_PRINT_EACCES_LONG "print-eacces"
 #define FLAG_PRINT_EACCES {FLAG_PRINT_EACCES_LONG, no_argument, NULL, FLAG_PRINT_EACCES_SHORT}
 
-#define FLAG_NO_PRINT_SQL_ON_ERR_SHORT (FLAG_GROUP_MISC + 18)
+#define FLAG_NO_PRINT_SQL_ON_ERR_SHORT (FLAG_GROUP_MISC + 17)
 #define FLAG_NO_PRINT_SQL_ON_ERR_LONG "no-print-sql-on-err"
 #define FLAG_NO_PRINT_SQL_ON_ERR {FLAG_NO_PRINT_SQL_ON_ERR_LONG, no_argument, NULL, FLAG_NO_PRINT_SQL_ON_ERR_SHORT}
 
@@ -308,6 +305,16 @@ extern "C" {
 #define FLAG_SUSPECT_STAT_LONG "suspect-stat"
 #define FLAG_SUSPECT_STAT {FLAG_SUSPECT_STAT_LONG, no_argument, NULL, FLAG_SUSPECT_STAT_SHORT}
 
+/* gufi_rollup flags */
+
+#define FLAG_ROLLUP_LIMIT_SHORT (FLAG_GROUP_ROLLUP + 0)
+#define FLAG_ROLLUP_LIMIT_LONG "limit"
+#define FLAG_ROLLUP_LIMIT {FLAG_ROLLUP_LIMIT_LONG, required_argument, NULL, FLAG_ROLLUP_LIMIT_SHORT}
+
+#define FLAG_ROLLUP_DELETE_BELOW_SHORT (FLAG_GROUP_ROLLUP + 1)
+#define FLAG_ROLLUP_DELETE_BELOW_LONG "delete-below"
+#define FLAG_ROLLUP_DELETE_BELOW {FLAG_ROLLUP_DELETE_BELOW_LONG, required_argument, NULL, FLAG_ROLLUP_DELETE_BELOW_SHORT}
+
 /* required at the end of every flag list */
 #define FLAG_END {NULL, 0, NULL, 0}
 
@@ -514,7 +521,11 @@ struct input {
     str_t format;
 
     /* only used by rollup */
-    size_t rollup_entries_limit;
+    struct {
+        size_t entries_limit;
+        size_t delete_below;
+        int attach_flag;    /* SQLITE_OPEN_READONLY or SQLITE_OPEN_READWRITE; set by --delete-below */
+    } rollup;
 
     /* trie containing directory basenames to skip during tree traversal */
     trie_t *skip;

src/bf.c

@@ -123,6 +123,8 @@ struct input *input_init(struct input *in) {
         in->output                  = STDOUT;
         in->output_buffer_size      = 4096;
         in->open_flags              = SQLITE_OPEN_READONLY;   // default to read-only opens
+        in->rollup.delete_below     = -1;
+        in->rollup.attach_flag      = SQLITE_OPEN_READONLY;
         in->skip                    = trie_alloc();
 
         /* always skip . and .. */
@@ -251,7 +253,6 @@ void print_help(const char* prog_name,
             case FLAG_PATH_LIST_SHORT:               printf("      --path-list <filename>        File containing paths at single level to walk (not including starting path). If --min-level > 0, prepend each line of the file with the index path."); break;
             case FLAG_FORMAT_SHORT:                  printf("      --format <FORMAT>             use the specified FORMAT instead of the default; output a newline after each use of FORMAT"); break;
             case FLAG_TERSE_SHORT:                   printf("      --terse                       print the information in terse form"); break; /* output from stat --help */
-            case FLAG_ROLLUP_LIMIT_SHORT:            printf("      --limit <count>               Highest number of files/links in a directory allowed to be rolled up"); break;
             case FLAG_DONT_REPROCESS_SHORT:          printf("      --dont-reprocess              if a directory was previously processed, skip descending the subtree"); break;
             case FLAG_NEWLINE_SHORT:                 printf("      --newline <c>                 character used to separate lines (default: '\\n') [use 0 for NULL character]"); break;
             case FLAG_SUPPRESS_NEWLINE_SHORT:        printf("      --suppress-newline            do not print the line separator"); break;
@@ -277,6 +278,10 @@ void print_help(const char* prog_name,
             case FLAG_SUSPECT_TIME_SHORT:            printf("      --suspect-time <s>            time in seconds since epoch for suspect comparision"); break;
             case FLAG_SUSPECT_STAT_SHORT:            printf("      --suspect-stat                if an entry is suspect, stat it to get timestamps to compare against suspecttime"); break;
 
+            /* gufi_rollup flags */
+            case FLAG_ROLLUP_LIMIT_SHORT:            printf("      --limit <count>               Highest number of files/links in a directory allowed to be rolled up"); break;
+            case FLAG_ROLLUP_DELETE_BELOW_SHORT:     printf("      --delete-below <level>        Delete rollup data under (not including) this level"); break;
+
             default:                                 printf("print_help(): unrecognized option '%c'", (char)options->val); break;
         }
         options++;
@@ -331,7 +336,6 @@ void show_input(struct input* in, int retval) {
     printf("in.format_set               = %d\n",            in->format_set);
     printf("in.format                   = '%s'\n",          in->format.data);
     printf("in.terse                    = %d\n",            in->terse);
-    printf("in.rollup_entries_limit     = %zu\n",           in->rollup_entries_limit);
     printf("in.dont_reprocess           = %d\n",            in->dont_reprocess);
     printf("in.newline                  = '%c'\n",          in->newline);
     printf("in.suppress_newline         = %d\n",            in->suppress_newline);
@@ -350,13 +354,18 @@ void show_input(struct input* in, int retval) {
 
     printf("in.process_xattrs           = %d\n",            in->process_xattrs);
 
-    /* gufi_incremental_update */
+    /* gufi_incremental_update flags */
     printf("in.insuspect                = '%s'\n",          in->insuspect.data);
     printf("in.suspectfile              = '%d'\n",          in->suspectfile);
     printf("in.suspectmethod            = '%d'\n",          in->suspectmethod);
     printf("in.suspecttime              = '%d'\n",          in->suspecttime);
     printf("in.suspectstat              = '%d'\n",          in->suspectstat);
 
+    /* gufi_rollup flags */
+    printf("in.rollup.entries_limit     = %zu\n",           in->rollup.entries_limit);
+    printf("in.rollup.delete_below      = %zu\n",           in->rollup.delete_below);
+    printf("in.rollup.attach_flag       = %d\n",            in->rollup.attach_flag);
+
     printf("retval                      = %d\n",            retval);
     printf("\n");
 }
@@ -623,11 +632,6 @@ int parse_cmd_line(int                  argc,
                 in->terse = 1;
                 break;
 
-            case FLAG_ROLLUP_LIMIT_SHORT:
-                INSTALL_SIZE(&in->rollup_entries_limit, optarg, (size_t) 0, (size_t) -1,
-                             "--" FLAG_ROLLUP_LIMIT_LONG, &retval);
-                break;
-
             case FLAG_DONT_REPROCESS_SHORT:
                 in->dont_reprocess = 1;
                 break;
@@ -703,7 +707,7 @@ int parse_cmd_line(int                  argc,
                 in->process_xattrs = 1; /* all xattr flags point to the same variable */
                 break;
 
-            /* gufi_incremental_update */
+            /* gufi_incremental_update flags */
 
             case FLAG_SUSPECT_FILE_SHORT:
                 INSTALL_STR(&in->insuspect, optarg);
@@ -725,6 +729,19 @@ int parse_cmd_line(int                  argc,
                 in->suspectstat = 1;
                 break;
 
+            /* gufi_rollup flags */
+
+            case FLAG_ROLLUP_LIMIT_SHORT:
+                INSTALL_SIZE(&in->rollup.entries_limit, optarg, (size_t) 0, (size_t) -1,
+                             "--" FLAG_ROLLUP_LIMIT_LONG, &retval);
+                break;
+
+            case FLAG_ROLLUP_DELETE_BELOW_SHORT:
+                INSTALL_SIZE(&in->rollup.delete_below, optarg, (size_t) 0, (size_t) -1,
+                             "--" FLAG_ROLLUP_DELETE_BELOW_LONG, &retval);
+                in->rollup.attach_flag = SQLITE_OPEN_READWRITE;
+                break;
+
             case '?':
                 /* getopt_long prints an error message and returns '?' when there is a problem. */
                 retval = -1;