casync-tool: gc verb

Fixes #43.

Author: keszybz

README.md

@@ -158,7 +158,9 @@ but there are other systems that use similar algorithms, in particular:
 ## Maintenance
 
 ```
-# casync gc /var/lib/backup.castr /home/lennart.caidx /home/foobar.caidx ... (NOT IMPLEMENTED YET)
+# casync gc /home/lennart-20170101.caidx /home/lennart-20170102.caidx /home/lennart-20170103.caidx
+# casync gc --backup /var/lib/backup/backup.castr /home/lennart-*.caidx
+
 # casync make /home/lennart.catab /home/lennart (NOT IMPLEMENTED)
 ```
 

doc/casync.rst

@@ -16,6 +16,7 @@ Synopsis
 | **casync** [*OPTIONS*...] digest [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
 | **casync** [*OPTIONS*...] mount [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
 | **casync** [*OPTIONS*...] mkdev [*BLOB* | *BLOB_INDEX*] [*NODE*]
+| **casync** [*OPTIONS*...] gc *BLOB_INDEX* | *ARCHIVE_INDEX* ...
 
 Description
 -----------
@@ -42,8 +43,8 @@ The metadata included in the archive is controlled by the ``--with-*`` and
 ``--without-*`` options.
 
 |
-| **casync** extract [*ARCHIVE* | *ARCHIVE_INDEX*] [*DIRECTORY*]
-| **casync** extract *BLOB_INDEX* *FILE* | *DEVICE*
+| **casync** **extract** [*ARCHIVE* | *ARCHIVE_INDEX*] [*DIRECTORY*]
+| **casync** **extract** *BLOB_INDEX* *FILE* | *DEVICE*
 
 This will extract the contents of a .catar archive or .caidx index
 into the specified *DIRECTORY*, or the contents specified by *BLOB_INDEX*
@@ -54,7 +55,7 @@ The metadata replayed from the archive is controlled by the ``--with-*`` and
 ``--without-*`` options.
 
 |
-| **casync** list [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
+| **casync** **list** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
 
 This will list all the files and directories in the specified .catar
 archive or .caidx index, or the directory. The argument is optional,
@@ -68,7 +69,7 @@ The output includes the permission mask and file names::
   -rw-r--r-- TODO
 
 |
-| **casync** mtree [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
+| **casync** **mtree** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
 
 This is similar to **list**, but includes information about each entry in the
 key=value format defined by BSD mtree(5)::
@@ -79,7 +80,7 @@ key=value format defined by BSD mtree(5)::
   TODO type=file mode=0644 size=2395 uid=0 gid=0 time=1498175562.000000000 sha256digest=316f11a03c08ec39f0328ab1f7446bd048507d3fbeafffe7c32fad4942244b7d
 
 |
-| **casync** stat [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*] [*PATH*]
+| **casync** **stat** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*] [*PATH*]
 
 This will show detailed information about a file or directory *PATH*, as found
 in either *ARCHIVE* or *ARCHIVE_INDEX* or underneath *DIRECTORY*. Both arguments
@@ -99,7 +100,7 @@ Example output::
      Group: zbyszek (1000)
 
 |
-| **casync** digest [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
+| **casync** **digest** [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
 
 This will compute and print the checksum of the argument.
 The argument is optional and defaults to the current directory::
@@ -111,13 +112,13 @@ The argument is optional and defaults to the current directory::
   d1698b0c4c27163284abea5d1e369b92e89dd07cb74378638849800e0406baf7
 
 |
-| **casync** mount [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
+| **casync** **mount** [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
 
 This will mount the specified .catar archive or .caidx index at the
 specified *PATH*, using the FUSE protocol.
 
 |
-| **casync** mkdev [*BLOB* | *BLOB_INDEX*] [*NODE*]
+| **casync** **mkdev** [*BLOB* | *BLOB_INDEX*] [*NODE*]
 
 This will create a block device *NODE* with the contents specified
 by the .caibx *BLOB_INDEX* or just the file or block device *BLOB*,
@@ -134,13 +135,24 @@ Example::
 
 When ``casync mkdev`` is killed, the device is destroyed.
 
+|
+| **casync** **gc** *ARCHIVE_INDEX* | *BLOB_INDEX* ...
+
+This will remove all chunks that are not used by one of the specified indices
+(one or more blob and archive indices can be given). If ``--store`` is not
+given, the default store for the first index will be used.
+
+This command can be used to prune unused chunks from a shared chunk
+store.
+
 Options
 -------
 
 General options:
 
 --help, -h                      Show terse help output
 --verbose, -v                   Show terse status information during runtime
+--dry-run, -n                   Only print what would be removed with **gc**
 --store=PATH                    The primary chunk store to use
 --extra-store=<PATH>            Additional chunk store to look for chunks in
 --chunk-size=<[MIN:]AVG[:MAX]>  The minimal/average/maximum number of bytes in a chunk

src/castore.c

@@ -1,12 +1,14 @@
 /* SPDX-License-Identifier: LGPL-2.1+ */
 
+#include <dirent.h>
 #include <fcntl.h>
 #include <lzma.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include "castore.h"
 #include "def.h"
+#include "dirent-util.h"
 #include "realloc-buffer.h"
 #include "rm-rf.h"
 #include "util.h"
@@ -34,6 +36,14 @@ struct CaStore {
         uint64_t n_request_bytes;
 };
 
+struct CaStoreIterator {
+        CaStore *store;
+
+        DIR *rootdir;
+        struct dirent *subdir_de;
+        DIR *subdir;
+};
+
 CaStore* ca_store_new(void) {
         CaStore *store;
 
@@ -312,3 +322,88 @@ int ca_store_set_digest_type(CaStore *s, CaDigestType type) {
 
         return 0;
 }
+
+CaStoreIterator* ca_store_iterator_new(CaStore *store) {
+        CaStoreIterator *it;
+
+        it = new0(CaStoreIterator, 1);
+        if (!it)
+                return NULL;
+
+        it->store = store;
+
+        return it;
+}
+
+CaStoreIterator* ca_store_iterator_unref(CaStoreIterator *iter) {
+        if (!iter)
+                return NULL;
+
+        if (iter->rootdir)
+                closedir(iter->rootdir);
+        if (iter->subdir)
+                closedir(iter->subdir);
+        return mfree(iter);
+}
+
+int ca_store_iterator_next(
+                CaStoreIterator *iter,
+                int *rootdir_fd,
+                const char **subdir,
+                int *subdir_fd,
+                const char **chunk) {
+
+        struct dirent *de;
+
+        if (!iter->rootdir) {
+                iter->rootdir = opendir(iter->store->root);
+                if (!iter->rootdir)
+                        return -errno;
+        }
+
+        while (true) {
+                if (!iter->subdir) {
+                        int fd;
+
+                        errno = 0;
+                        iter->subdir_de = readdir(iter->rootdir);
+                        if (!iter->subdir_de) {
+                                if (errno > 0)
+                                        return -errno;
+                                return 0; /* done */
+                        }
+
+                        fd = openat(dirfd(iter->rootdir), iter->subdir_de->d_name,
+                                    O_RDONLY|O_CLOEXEC|O_DIRECTORY);
+                        if (fd < 0) {
+                                if (errno == EISDIR)
+                                        continue;
+                                return -errno;
+                        }
+
+                        iter->subdir = fdopendir(fd);
+                        if (!iter->subdir) {
+                                safe_close(fd);
+                                return -errno;
+                        }
+                }
+
+                FOREACH_DIRENT_ALL(de, iter->subdir, return -errno) {
+                        if (!dirent_is_file_with_suffix(de, ".cacnk"))
+                                continue;
+
+                        if (rootdir_fd)
+                                *rootdir_fd = dirfd(iter->rootdir);
+                        if (subdir)
+                                *subdir = iter->subdir_de->d_name;
+                        if (subdir_fd)
+                                *subdir_fd = dirfd(iter->subdir);
+                        if (chunk)
+                                *chunk = de->d_name;
+                        return 1; /* success */
+                }
+
+                assert_se(closedir(iter->subdir) == 0);
+                iter->subdir = NULL;
+        }
+}

src/castore.h

@@ -8,10 +8,14 @@
 #include "cautil.h"
 
 typedef struct CaStore CaStore;
+typedef struct CaStoreIterator CaStoreIterator;
 
 CaStore* ca_store_new(void);
 CaStore *ca_store_new_cache(void);
 CaStore* ca_store_unref(CaStore *store);
+static inline void ca_store_unrefp(CaStore **store) {
+        ca_store_unref(*store);
+}
 
 int ca_store_set_path(CaStore *store, const char *path);
 int ca_store_set_compression(CaStore *store, CaChunkCompression c);
@@ -26,4 +30,16 @@ int ca_store_get_request_bytes(CaStore *s, uint64_t *ret);
 
 int ca_store_set_digest_type(CaStore *s, CaDigestType type);
 
+CaStoreIterator* ca_store_iterator_new(CaStore *store);
+CaStoreIterator* ca_store_iterator_unref(CaStoreIterator *iter);
+static inline void ca_store_iterator_unrefp(CaStoreIterator **iter) {
+        ca_store_iterator_unref(*iter);
+}
+int ca_store_iterator_next(
+                CaStoreIterator *iter,
+                int *rootdir_fd,
+                const char **subdir,
+                int *subdir_fd,
+                const char **chunk);
+
 #endif

src/casync-tool.c

@@ -21,6 +21,7 @@
 #include "caremote.h"
 #include "castore.h"
 #include "casync.h"
+#include "gc.h"
 #include "notify.h"
 #include "parse-util.h"
 #include "signal-handler.h"
@@ -35,6 +36,7 @@ static enum arg_what {
         _WHAT_INVALID = -1,
 } arg_what = _WHAT_INVALID;
 static bool arg_verbose = false;
+static bool arg_dry_run = false;
 static bool arg_exclude_nodump = true;
 static bool arg_exclude_submounts = false;
 static bool arg_reflink = true;
@@ -74,6 +76,8 @@ static void help(void) {
                "  -h --help                  Show this help\n"
                "     --version               Show brief version information\n"
                "  -v --verbose               Show terse status information during runtime\n"
+               "  -n --dry-run               When garbage collecting, only print what would\n"
+               "                             be done\n"
                "     --store=PATH            The primary chunk store to use\n"
                "     --extra-store=PATH      Additional chunk store to look for chunks in\n"
                "     --chunk-size=[MIN:]AVG[:MAX]\n"
@@ -312,6 +316,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "help",              no_argument,       NULL, 'h'                   },
                 { "version",           no_argument,       NULL, ARG_VERSION           },
                 { "verbose",           no_argument,       NULL, 'v'                   },
+                { "dry-run",           no_argument,       NULL, 'n'                   },
                 { "store",             required_argument, NULL, ARG_STORE             },
                 { "extra-store",       required_argument, NULL, ARG_EXTRA_STORE       },
                 { "chunk-size",        required_argument, NULL, ARG_CHUNK_SIZE        },
@@ -345,7 +350,7 @@ static int parse_argv(int argc, char *argv[]) {
         if (getenv_bool("CASYNC_VERBOSE") > 0)
                 arg_verbose = true;
 
-        while ((c = getopt_long(argc, argv, "hv", options, NULL)) >= 0) {
+        while ((c = getopt_long(argc, argv, "hvn", options, NULL)) >= 0) {
 
                 switch (c) {
 
@@ -361,6 +366,10 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_verbose = true;
                         break;
 
+                case 'n':
+                        arg_dry_run = true;
+                        break;
+
                 case ARG_STORE: {
                         char *p;
 
@@ -3807,6 +3816,67 @@ static int verb_udev(int argc, char *argv[]) {
         return 0;
 }
 
+static int verb_gc(int argc, char *argv[]) {
+        int i, r;
+        _cleanup_(ca_chunk_collection_unrefp) CaChunkCollection *coll = NULL;
+        _cleanup_(ca_store_unrefp) CaStore *store = NULL;
+
+        if (argc < 2) {
+                fprintf(stderr, "Expected at least one argument.\n");
+                return -EINVAL;
+        }
+
+        coll = ca_chunk_collection_new();
+        if (!coll)
+                return log_oom();
+
+        /* This sets the same store for all indices, based on the first index. */
+        r = set_default_store(argv[1]);
+        if (r < 0)
+                return r;
+
+        if (!arg_store) {
+                fprintf(stderr, "Failed to determine store, use --store= to set store.\n");
+                return -EINVAL;
+        }
+
+        store = ca_store_new();
+        if (!store)
+                return log_oom();
+
+        r = ca_store_set_path(store, arg_store);
+        if (r < 0) {
+                fprintf(stderr, "Set to set store to \"%s\": %s", arg_store, strerror(-r));
+                return r;
+        }
+
+        for (i = 1; i < argc; i++) {
+                const char *path = argv[i];
+
+                r = ca_chunk_collection_add_index(coll, path);
+                if (r < 0)
+                        return r;
+        }
+
+        {
+                size_t usage, size;
+
+                assert_se(ca_chunk_collection_usage(coll, &usage) == 0);
+                assert_se(ca_chunk_collection_size(coll, &size) == 0);
+                if (arg_verbose)
+                        printf("Chunk store usage: %zu references, %zu chunks\n", usage, size);
+        }
+
+        r = ca_gc_cleanup_unused(store, coll,
+                                 arg_verbose * CA_GC_VERBOSE |
+                                 arg_dry_run * CA_GC_DRY_RUN);
+        if (r < 0)
+                fprintf(stderr, "Chunk cleanup failed: %s\n", strerror(-r));
+
+        return r;
+}
+
+
 static int dispatch_verb(int argc, char *argv[]) {
         int r;
 
@@ -3836,6 +3906,8 @@ static int dispatch_verb(int argc, char *argv[]) {
                 r = verb_push(argc, argv);
         else if (streq(argv[0], "udev")) /* "Secret" verb, only to be called by the udev nbd rules */
                 r = verb_udev(argc, argv);
+        else if (streq(argv[0], "gc"))
+                r = verb_gc(argc, argv);
         else {
                 fprintf(stderr, "Unknown verb '%s'. (Invoke '%s --help' for a list of available verbs.)\n", argv[0], program_invocation_short_name);
                 r = -EINVAL;