gufi_du

implements du with
    --apparent-size
    --block-size SIZE
    --total
    --summarize

only includes file/link blocks/size
    directory blocks/size are not included

requires the treesummary table to be present
    if no rows are returned from gufi_query, will print warning
    works with rollups

changed -a to an enum
    AND -> RUN_ON_ROW  (0) (default)
    OR  -> RUN_SE      (1)
    new:   RUN_TSE     (2)

added totblocks to summary and treesummary tables

gufi_rollup now copies treesummary data upwards

Author: calccrypto

docs/latex/CMakeLists.txt

@@ -138,6 +138,7 @@ set(gufi_administrator_inputs
 set(gufi_user_inputs
     sections/introduction.tex
     sections/user_env.tex
+    sections/gufi_du.tex
     sections/gufi_find.tex
     sections/gufi_ls.tex
     sections/gufi_stats.tex

docs/latex/GUFIText.sty

@@ -104,6 +104,7 @@
 \newcommand{\processdir}{\texttt{processdir}\xspace}
 \newcommand{\querydb}{\texttt{querydb}\xspace}
 \newcommand{\querydbs}{\texttt{querydbs}\xspace}
+\newcommand{\gufidu}{\texttt{gufi\_du}\xspace}
 \newcommand{\gufifind}{\texttt{gufi\_find}\xspace}
 \newcommand{\gufils}{\texttt{gufi\_ls}\xspace}
 \newcommand{\gufisqlite}{\texttt{gufi\_sqlite3}\xspace}

docs/latex/gufi_user.tex

@@ -65,6 +65,7 @@
 % Useful packages
 \usepackage{graphicx}
 \usepackage{hyperref}
+\usepackage{tabularx}
 \usepackage{textcomp}
 \usepackage{verbatim}
 \usepackage{GUFIText}
@@ -91,6 +92,7 @@
 \include{sections/license}
 \include{sections/introduction}
 \include{sections/user_env}
+\include{sections/gufi_du}
 \include{sections/gufi_find}
 \include{sections/gufi_ls}
 \include{sections/gufi_stat}

docs/latex/sections/gufi_du.tex

@@ -0,0 +1,105 @@
+% This file is part of GUFI, which is part of MarFS, which is released
+% under the BSD license.
+%
+%
+% Copyright (c) 2017, Los Alamos National Security (LANS), LLC
+% All rights reserved.
+%
+% Redistribution and use in source and binary forms, with or without modification,
+% are permitted provided that the following conditions are met:
+%
+% 1. Redistributions of source code must retain the above copyright notice, this
+% list of conditions and the following disclaimer.
+%
+% 2. Redistributions in binary form must reproduce the above copyright notice,
+% this list of conditions and the following disclaimer in the documentation and/or
+% other materials provided with the distribution.
+%
+% 3. Neither the name of the copyright holder nor the names of its contributors
+% may be used to endorse or promote products derived from this software without
+% specific prior written permission.
+%
+% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+% ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+% WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+% IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+% INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+% DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+% LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+% OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+%
+%
+% From Los Alamos National Security, LLC:
+% LA-CC-15-039
+%
+% Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
+% Copyright 2017. Los Alamos National Security, LLC. This software was produced
+% under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
+% Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
+% the U.S. Department of Energy. The U.S. Government has rights to use,
+% reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR LOS
+% ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+% ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is
+% modified to produce derivative works, such modified software should be
+% clearly marked, so as not to confuse it with the version available from
+% LANL.
+%
+% THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
+% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+% THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+% ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
+% CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+% EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+% OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+% IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+% OF SUCH DAMAGE.
+
+
+
+\section{\gufidu}
+\gufidu replicates a subset of the functionality provided by the POSIX
+command du.
+
+\subsection{Notes}
+\begin{itemize}
+\item \gufidu requires the index to have treesummary data in at least
+  the directory that is passed in. If only some directories have
+  treesummary data, \gufidu without arguments will only report sizes
+  from those directories.
+  \begin{itemize}
+  \item Run \gufitreesummary, \gufitreesummaryall, or \gufirollup
+    before calling \gufidu.
+  \end{itemize}
+\item \gufidu output does not include the block count or size of
+  directories.
+\end{itemize}
+
+\subsection{Flags}
+\begin{table} [h!]
+  \centering
+  \begin{tabularx}{\textwidth}{| l | X |}
+    \hline
+    Flags & Functionality \\
+    \hline
+    -{}-help & displays help menu \\
+    \hline
+    -v, -{}-version & show program's version number and exit \\
+    \hline
+    -{}-apparent-size & print apparent sizes, rather than disk usage;
+    although the apparent size is usually smaller, it may be larger
+    due to holes in ('sparse') files, internal fragmentation, indirect
+    blocks, and the like \\
+    \hline
+    -B, -{}-block-size & scale sizes by SIZE before printing them \\
+    \hline
+    -c, -{}-total & produce a grand total \\
+    \hline
+    -s, -{}-summarize & display only a total for each argument \\
+    \hline
+  \end{tabularx}
+  \caption{\label{fig:gufi_du flags}{gufi\_du Flags and Functionality}}
+\end{table}

docs/latex/sections/gufi_ls.tex

@@ -68,7 +68,7 @@ \section{\gufils}
 \subsection{Flags}
 \begin{table} [h!]
   \centering
-  \begin{tabular}{| l | r |}
+  \begin{tabular}{| l | l |}
     \hline
     Flags & Functionality \\
     \hline

examples/gengidsummaryavoidentriesscan

@@ -73,7 +73,7 @@ THREADS="${2:-1}"
 # if not, fall back to pulling from entries
 
 echo "create gid summary records in summary tables in ${INDEX}"
-"${BFQ}" -n "${THREADS}" -a -w \
+"${BFQ}" -n "${THREADS}" -a 1 -w \
     -S "DELETE FROM summary WHERE rectype == 2; INSERT INTO summary (gid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, rectype) SELECT gid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, 2 FROM summary WHERE rectype == 0 AND mingid == maxgid" \
     -E "INSERT INTO summary (gid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, rectype) SELECT gid, SUM(CASE WHEN type == 'f' THEN 1 ELSE 0 END), SUM(CASE WHEN type == 'l' THEN 1 ELSE 0 END), MIN(uid), MAX(uid), MIN(gid), MAX(gid), MIN(size), MAX(size), SUM(CASE WHEN size <= 1024 then 1 ELSE 0 END), SUM(CASE WHEN size > 1024 then 1 ELSE 0 END), SUM(CASE WHEN size <= 1048576 then 1 ELSE 0 END), SUM(CASE WHEN size > 1048576 then 1 ELSE 0 END), SUM(CASE WHEN size > 1073741824 then 1 ELSE 0 END), SUM(CASE WHEN size > 1099511627776 then 1 ELSE 0 END), SUM(size), MIN(ctime), MAX(ctime), MIN(mtime), MAX(mtime), MIN(atime), MAX(atime), MIN(blocks), MAX(blocks), SUM(LENGTH(xattr_names) - LENGTH(REPLACE(xattr_names, char(31), ''))), MIN(crtime), MAX(crtime), MIN(ossint1), MAX(ossint1), SUM(ossint1), MIN(ossint2), MAX(ossint2), SUM(ossint2), MIN(ossint3), MAX(ossint3), SUM(ossint3), MIN(ossint4), MAX(ossint4), SUM(ossint4), 2 FROM entries WHERE NOT EXISTS (SELECT 1 FROM summary WHERE rectype == 2) GROUP BY gid;" \
     "${INDEX}"

examples/genuidsummaryavoidentriesscan

@@ -73,7 +73,7 @@ THREADS="${2:-1}"
 # if not, fall back to pulling from entries
 
 echo "create uid summary records in summary tables in ${INDEX}"
-"${BFQ}" -n "${THREADS}" -a -w \
+"${BFQ}" -n "${THREADS}" -a 1 -w \
     -S "DELETE FROM summary WHERE rectype == 1; INSERT INTO summary (uid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, rectype) SELECT uid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, 1 FROM summary WHERE rectype == 0 AND minuid == maxuid;" \
     -E "INSERT INTO summary (uid, totfiles, totlinks, minuid, maxuid, mingid, maxgid, minsize, maxsize, totltk, totmtk, totltm, totmtm, totmtg, totmtt, totsize, minctime, maxctime, minmtime, maxmtime, minatime, maxatime, minblocks, maxblocks, totxattr, mincrtime, maxcrtime, minossint1, maxossint1, totossint1, minossint2, maxossint2, totossint2, minossint3, maxossint3, totossint3, minossint4, maxossint4, totossint4, rectype) SELECT uid, SUM(CASE WHEN type == 'f' THEN 1 ELSE 0 END), SUM(CASE WHEN type == 'l' THEN 1 ELSE 0 END), MIN(uid), MAX(uid), MIN(gid), MAX(gid), MIN(size), MAX(size), SUM(CASE when size <= 1024 then 1 ELSE 0 END), SUM(CASE when size > 1024 then 1 ELSE 0 END), SUM(CASE when size <= 1048576 then 1 ELSE 0 END), SUM(CASE when size > 1048576 then 1 ELSE 0 END), SUM(CASE when size > 1073741824 then 1 ELSE 0 END), SUM(CASE when size > 1099511627776 then 1 ELSE 0 END), SUM(size), MIN(ctime), MAX(ctime), MIN(mtime), MAX(mtime), MIN(atime), MAX(atime), MIN(blocks), MAX(blocks), SUM(LENGTH(xattr_names) - LENGTH(REPLACE(xattr_names, char(31), ''))), MIN(crtime), MAX(crtime), MIN(ossint1), MAX(ossint1), SUM(ossint1), MIN(ossint2), MAX(ossint2), SUM(ossint2), MIN(ossint3), MAX(ossint3), SUM(ossint3), MIN(ossint4), MAX(ossint4), SUM(ossint4), 1 FROM entries WHERE NOT EXISTS (SELECT 1 FROM summary WHERE rectype == 1) GROUP BY uid;" \
     "${INDEX}"

include/bf.h

@@ -111,6 +111,7 @@ struct sum {
   long long int maxatime;
   long long int minblocks;
   long long int maxblocks;
+  long long int totblocks; /* files only - does not include directory/self */
   long long int totxattr;
   long long int totsubdirs;
   long long int maxsubdirfiles;
@@ -133,10 +134,11 @@ struct sum {
   long long int totextdbs; /* only tracked in treesummary, not summary */
 };
 
-typedef enum AndOr {
-    AND,
-    OR,
-} AndOr_t;
+typedef enum AFlag {
+    RUN_ON_ROW = 0,   /* if returned row, run next SQL, else stop (continue descent) (default) */
+    RUN_SE     = 1,   /* skip T, run S and E whether or not a row was returned (old -a) */
+    RUN_TSE    = 2,   /* run T, S, and E whether or not a row was returned */
+} AFlag_t;
 
 typedef enum OutputMethod {
     STDOUT,    /* default */
@@ -212,7 +214,7 @@ struct input {
    char delim;
    int  buildindex;
    size_t maxthreads;
-   AndOr_t andor;
+   AFlag_t process_sql;       /* what to do if an SQL statement returns/doesn't return 1 row */
    int  insertdir;                // added for bfwreaddirplus2db
    int  insertfl;                 // added for bfwreaddirplus2db
    int  suspectd;                 // added for bfwreaddirplus2db for how to default suspect directories 0 - not supsect 1 - suspect

include/dbutils.h

@@ -109,7 +109,7 @@ extern const char ENTRIES_INSERT[];
 /* directory metadata + aggregate data */
 #define SUMMARY           "summary"
 #define SUMMARY_SCHEMA(name, extra_cols)            \
-    "CREATE TABLE " name "(" extra_cols "name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64);"
+    "CREATE TABLE " name "(" extra_cols "name TEXT, type TEXT, inode TEXT, mode INT64, nlink INT64, uid INT64, gid INT64, size INT64, blksize INT64, blocks INT64, atime INT64, mtime INT64, ctime INT64, linkname TEXT, xattr_names BLOB, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totblocks INT16, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, rectype INT64, pinode TEXT, isroot INT64, rollupscore INT64);"
 extern const char SUMMARY_CREATE[];
 
 /* view of summary table with rollups */
@@ -136,7 +136,7 @@ extern const char VRPENTRIES_CREATE[];
 /* aggregate data of tree starting at current directory */
 #define TREESUMMARY       "treesummary"
 #define TREESUMMARY_SCHEMA(name, extra_cols)        \
-    "CREATE TABLE " name "(" extra_cols "inode TEXT, pinode TEXT, totsubdirs INT64, maxsubdirfiles INT64, maxsubdirlinks INT64, maxsubdirsize INT64, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, totextdbs INT64, rectype INT64, uid INT64, gid INT64);"
+    "CREATE TABLE " name "(" extra_cols "inode TEXT, pinode TEXT, totsubdirs INT64, maxsubdirfiles INT64, maxsubdirlinks INT64, maxsubdirsize INT64, totfiles INT64, totlinks INT64, minuid INT64, maxuid INT64, mingid INT64, maxgid INT64, minsize INT64, maxsize INT64, totzero INT64, totltk INT64, totmtk INT64, totltm INT64, totmtm INT64, totmtg INT64, totmtt INT64, totsize INT64, minctime INT64, maxctime INT64, minmtime INT64, maxmtime INT64, minatime INT64, maxatime INT64, minblocks INT64, maxblocks INT64, totblocks INT64, totxattr INT64, depth INT64, mincrtime INT64, maxcrtime INT64, minossint1 INT64, maxossint1 INT64, totossint1 INT64, minossint2 INT64, maxossint2 INT64, totossint2 INT64, minossint3 INT64, maxossint3 INT64, totossint3 INT64, minossint4 INT64, maxossint4 INT64, totossint4 INT64, totextdbs INT64, rectype INT64, uid INT64, gid INT64);"
 #define TREESUMMARY_CREATE                          \
     DROP_TABLE(TREESUMMARY)                         \
     TREESUMMARY_SCHEMA(TREESUMMARY, "")

scripts/CMakeLists.txt

@@ -62,6 +62,7 @@
 
 # gufi_* scripts
 set(TOOLS
+  du
   find
   getfattr
   ls