Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,15 @@ private static class SubCommandOpts {
public boolean help = false;
}

@Parameters(commandDescription = "Compaction Temp Files Utility")
static class FindCompactionTmpFilesCommand {
@Parameter(names = {"-t", "--tables"}, description = "comma separated list of table names")
String tables;

@Parameter(names = {"-d", "--delete"}, description = "if true, will delete tmp files")
boolean delete = false;
}

@Parameters(
commandDescription = "Stop the servers at the given addresses allowing them to complete current task but not start new task. When no port is specified uses ports from tserver.port.client property.")
static class StopCommand extends SubCommandOpts {
Expand Down Expand Up @@ -476,6 +485,9 @@ public void execute(final String[] args) {
VolumesCommand volumesCommand = new VolumesCommand();
cl.addCommand("volumes", volumesCommand);

FindCompactionTmpFilesCommand filesCommand = new FindCompactionTmpFilesCommand();
cl.addCommand("compactionTempFiles", filesCommand);

cl.parse(args);

if (cl.getParsedCommand() == null) {
Expand Down Expand Up @@ -530,6 +542,8 @@ public void execute(final String[] args) {
tServerLocksOpts.delete);
} else if (cl.getParsedCommand().equals("fate")) {
executeFateOpsCommand(context, fateOpsCommand);
} else if (cl.getParsedCommand().equals("compactionTempFiles")) {
FindCompactionTmpFiles.execute(context, filesCommand.tables, filesCommand.delete);
} else if (cl.getParsedCommand().equals("serviceStatus")) {
ServiceStatusCmd ssc = new ServiceStatusCmd();
ssc.execute(context, serviceStatusCommandOpts.json, serviceStatusCommandOpts.showHosts);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,36 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set<Path> files
return stats;
}

public static void execute(ServerContext context, final String tablesToSearch,
final boolean delete) throws Exception {
LOG.info("Looking for compaction tmp files over tables: {}, deleting: {}", tablesToSearch,
delete);

String[] tables = tablesToSearch.split(",");
for (String table : tables) {

table = table.trim();
String tableId = context.tableOperations().tableIdMap().get(table);
if (tableId == null || tableId.isEmpty()) {
LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table);
continue;
}

final Set<Path> matches = findTempFiles(context, tableId);
LOG.info("Found the following compaction tmp files for table {}:", table);
matches.forEach(p -> LOG.info("{}", p));

if (delete) {
LOG.info("Deleting compaction tmp files for table {}...", table);
DeleteStats stats = deleteTempFiles(context, matches);
LOG.info(
"Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}",
table, stats.success, stats.failure, stats.error);
}

}
}

public static void main(String[] args) throws Exception {
Opts opts = new Opts();
opts.parseArgs(FindCompactionTmpFiles.class.getName(), args);
Expand All @@ -186,31 +216,7 @@ public static void main(String[] args) throws Exception {
try (Scope scope = span.makeCurrent()) {

ServerContext context = opts.getServerContext();
String[] tables = opts.tables.split(",");

final var stringStringMap = context.tableOperations().tableIdMap();
for (String table : tables) {

table = table.trim();
String tableId = stringStringMap.get(table);
if (tableId == null || tableId.isEmpty()) {
LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table);
continue;
}

final Set<Path> matches = findTempFiles(context, tableId);
LOG.info("Found the following compaction tmp files for table {}:", table);
matches.forEach(p -> LOG.info("{}", p));

if (opts.delete) {
LOG.info("Deleting compaction tmp files for table {}...", table);
DeleteStats stats = deleteTempFiles(context, matches);
LOG.info(
"Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}",
table, stats.success, stats.failure, stats.error);
}

}
execute(context, opts.tables, opts.delete);

} finally {
span.end();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,11 @@ public void testStopOpts() {
Admin.StopCommand opts = new Admin.StopCommand();
assertFalse(opts.force);
}

@Test
public void testFindCompactionTmpFilesCommand() {
Admin.FindCompactionTmpFilesCommand filesCommand = new Admin.FindCompactionTmpFilesCommand();
assertNull(filesCommand.tables);
assertFalse(filesCommand.delete);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.HashSet;
Expand All @@ -36,6 +37,7 @@
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.tablets.TabletNameGenerator;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.server.util.FindCompactionTmpFiles;
import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats;
import org.apache.hadoop.fs.FileSystem;
Expand Down Expand Up @@ -216,4 +218,57 @@ public void testFindCompactionTmpFilesMainWithDelete() throws Exception {
}
}

@Test
public void testFindCompactionTmpFilesAdminCommand() throws Exception {

try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {

String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
ReadWriteIT.ingest(c, 100, 1, 1, 0, tableName);
c.tableOperations().flush(tableName);

String tableId = c.tableOperations().tableIdMap().get(tableName);
TableId tid = TableId.of(tableId);

ServerContext ctx = getCluster().getServerContext();
FileSystem fs = getCluster().getFileSystem();

Set<String> tablesDirs = ctx.getTablesDirs();
assertEquals(1, tablesDirs.size());

String tdir = tablesDirs.iterator().next() + "/" + tid.canonical() + "/default_tablet";
Path defaultTabletPath = new Path(tdir);
assertTrue(fs.exists(defaultTabletPath));

Set<Path> generatedPaths = generateTmpFilePaths(ctx, tid, defaultTabletPath, 100);
for (Path p : generatedPaths) {
assertFalse(fs.exists(p));
assertTrue(fs.createNewFile(p));
assertTrue(fs.exists(p));
}

Set<Path> foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(100, foundPaths.size());
assertEquals(foundPaths, generatedPaths);

System.setProperty("accumulo.properties",
"file://" + getCluster().getAccumuloPropertiesPath());
assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName)
.getProcess().waitFor());

foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(100, foundPaths.size());
assertEquals(foundPaths, generatedPaths);

assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName, "-d")
.getProcess().waitFor());

foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(0, foundPaths.size());
assertNotSame(foundPaths, generatedPaths);

}
}

}