Skip to content

Commit

Permalink
Use MFile instead of Path to get radar server to work for S3
Browse files Browse the repository at this point in the history
  • Loading branch information
@tdrwenski
tdrwenski committed Oct 1, 2024
1 parent f0d7388 commit 27ac865
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 132 deletions.
146 changes: 80 additions & 66 deletions tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@

package thredds.server.radarServer2;

import thredds.inventory.CollectionConfig;
import thredds.inventory.MController;
import thredds.inventory.MControllers;
import thredds.inventory.MFile;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.dt.RadialDatasetSweep;
import ucar.nc2.ft.FeatureDatasetFactoryManager;
Expand Down Expand Up @@ -33,7 +37,7 @@ public enum DirType {

private static final long updateIntervalMsec = 600000;
private EnumMap<DirType, Set<String>> items;
private Path collectionDir;
private MFile collectionDir;
private DirectoryStructure structure;
private String fileTimeFmt, dataFormat;
private java.util.regex.Pattern fileTimeRegex;
Expand All @@ -46,7 +50,7 @@ public enum DirType {
private DateRange timeCoverage;
private RadarServerConfig.RadarConfigEntry.GeoInfo geoCoverage;

public RadarDataInventory(Path datasetRoot, int numCrawl) {
public RadarDataInventory(MFile datasetRoot, int numCrawl) {
items = new EnumMap<>(DirType.class);
collectionDir = datasetRoot;
structure = new DirectoryStructure(collectionDir);
Expand All @@ -56,7 +60,7 @@ public RadarDataInventory(Path datasetRoot, int numCrawl) {
nearestWindow = CalendarPeriod.of(1, CalendarPeriod.Field.Hour);
}

public Path getCollectionDir() {
public MFile getCollectionDir() {
return collectionDir;
}

Expand Down Expand Up @@ -159,11 +163,11 @@ public SimpleDateFormat getFormat() {
return sdf;
}

public Date getDate(Path path) {
Path relPath = base.relativize(path);
public Date getDate(MFile mFile) {
String relPath = base.relativize(mFile);
StringBuilder sb = new StringBuilder("");
for (Integer l : levels) {
sb.append(relPath.getName(l));
sb.append(Paths.get(relPath).getName(l));
}
try {
SimpleDateFormat fmt = getFormat();
Expand All @@ -174,12 +178,12 @@ public Date getDate(Path path) {
}
}

private Path base;
private MFile base;

private List<DirEntry> order;
private List<Integer> keyIndices;

public DirectoryStructure(Path dir) {
public DirectoryStructure(MFile dir) {
base = dir;
order = new ArrayList<>();
keyIndices = new ArrayList<>();
Expand All @@ -194,11 +198,12 @@ public void addSubDir(DirType type, String fmt) {
}

// Get a key for a path based on station/var
public String getKey(Path path) {
Path relPath = base.relativize(path);
public String getKey(MFile mFile) {
String relPath = base.relativize(mFile);
StringBuilder sb = new StringBuilder("");

for (int ind : keyIndices) {
sb.append(relPath.getName(ind));
sb.append(Paths.get(relPath).getName(ind));
}
return sb.toString();
}
Expand Down Expand Up @@ -251,7 +256,7 @@ public void addFileTime(String regex, String fmt) {
fileTimeFmt = fmt;
}

private void findItems(Path start, int level) {
private void findItems(MFile start, int level) {
// Add each entry from this level to the appropriate item box
// and recurse
if (level >= structure.order.size() || level >= structure.maxCrawlDepth)
Expand All @@ -268,23 +273,17 @@ private void findItems(Path start, int level) {
}

int crawled = 0;
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(start)) {
for (Path p : dirStream) {
if (Files.isDirectory(p)) {
String item = p.getFileName().toString();
values.add(item);
// Try to grab station info from some file
// TODO: Fix or remove
// if (entry.type == DirType.Station)
// updateStations(item, p);
if (crawled < maxCrawlItems) {
findItems(p, level + 1);
++crawled;
}
}
for (MFile subDir : getSubDirs(start)) {
String item = subDir.getName();
values.add(item);
// Try to grab station info from some file
// TODO: Fix or remove
// if (entry.type == DirType.Station)
// updateStations(item, p);
if (crawled < maxCrawlItems) {
findItems(subDir, level + 1);
++crawled;
}
} catch (IOException e) {
System.out.println("findItems(): Error reading directory: " + start.toString());
}
}

Expand Down Expand Up @@ -354,12 +353,12 @@ public Query newQuery() {

public class Query {
public class QueryResultItem {
private QueryResultItem(Path f, CalendarDate cd) {
private QueryResultItem(MFile f, CalendarDate cd) {
file = f;
time = cd;
}

public Path file;
public MFile file;
public CalendarDate time;
}

Expand Down Expand Up @@ -412,8 +411,8 @@ private boolean checkDate(CalendarDateRange range, CalendarDate d) {
return range == null || range.includes(d);
}

public Collection<QueryResultItem> results() {
List<Path> results = new ArrayList<>();
public Collection<QueryResultItem> results() throws IOException {
List<MFile> results = new ArrayList<>();
DirectoryStructure.DirectoryDateMatcher matcher = structure.matcher();
results.add(structure.base);

Expand All @@ -432,7 +431,7 @@ public Collection<QueryResultItem> results() {
// exists. For dates, add the items that are within the filter
for (int i = 0; i < structure.order.size(); ++i) {
DirectoryStructure.DirEntry entry = structure.order.get(i);
List<Path> newResults = new ArrayList<>();
List<MFile> newResults = new ArrayList<>();
List<Object> queryItem = q.get(entry.type);
switch (entry.type) {
// Loop over results and add subdirs that are within the
Expand All @@ -443,15 +442,11 @@ public Collection<QueryResultItem> results() {
SimpleDateFormat fmt = matcher.getFormat();
CalendarDateRange dirRange = rangeFromFormat(fmt, range);

for (Path p : results)
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(p)) {
for (Path sub : dirStream) {
Date d = matcher.getDate(sub);
if (d != null && checkDate(dirRange, CalendarDate.of(d)))
newResults.add(sub);
}
} catch (IOException e) {
System.out.println("results(): Error reading dir: " + p.toString());
for (MFile mFile : results)
for (MFile sub : getSubDirs(mFile)) {
Date d = matcher.getDate(sub);
if (d != null && checkDate(dirRange, CalendarDate.of(d)))
newResults.add(sub);
}
break;

Expand All @@ -461,10 +456,11 @@ public Collection<QueryResultItem> results() {
case Variable:
default:
for (Object next : queryItem) {
for (Path p : results) {
Path nextPath = p.resolve(next.toString());
if (Files.exists(nextPath))
newResults.add(nextPath);
for (MFile mFile : results) {
MFile nextMFile = mFile.getChild(next.toString() + "/");
if (nextMFile != null && nextMFile.exists()) {
newResults.add(nextMFile);
}
}
}
}
Expand All @@ -473,28 +469,24 @@ public Collection<QueryResultItem> results() {

// Now get the contents of the remaining directories
Collection<QueryResultItem> filteredFiles = new ArrayList<>();
for (Path p : results) {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(p)) {
for (Path f : dirStream) {
java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(f.toString());
if (!regexMatcher.find())
continue;

try {
SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt);
fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
Date d = fmt.parse(regexMatcher.group());
if (d != null) {
CalendarDate cd = CalendarDate.of(d);
if (checkDate(range, cd))
filteredFiles.add(new QueryResultItem(f, cd));
}
} catch (ParseException e) {
// Ignore file
for (MFile dir : results) {
for (MFile mFile : getFiles(dir)) {
java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(mFile.getName());
if (!regexMatcher.find())
continue;

try {
SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt);
fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
Date d = fmt.parse(regexMatcher.group());
if (d != null) {
CalendarDate cd = CalendarDate.of(d);
if (checkDate(range, cd))
filteredFiles.add(new QueryResultItem(mFile, cd));
}
} catch (ParseException e) {
// Ignore file
}
} catch (IOException e) {
System.out.println("results(): Error getting files for: " + p.toString());
}
}

Expand Down Expand Up @@ -524,4 +516,26 @@ public Collection<QueryResultItem> results() {
return filteredFiles;
}
}

private List<MFile> getSubDirs(MFile directory) {
final MController mController = MControllers.create(directory.getPath());
final List<MFile> mFiles = new ArrayList<>();

final CollectionConfig dirs = new CollectionConfig("dirs", directory.getPath(), false, null, null);
final Iterator<MFile> dirIterator = mController.getSubdirs(dirs, true);
dirIterator.forEachRemaining(mFiles::add);

return mFiles;
}

private List<MFile> getFiles(MFile directory) throws IOException {
final MController mController = MControllers.create(directory.getPath());
final List<MFile> mFiles = new ArrayList<>();

final CollectionConfig files = new CollectionConfig("files", directory.getPath(), false, null, null);
final Iterator<MFile> fileIterator = mController.getInventoryTop(files, true);
fileIterator.forEachRemaining(mFiles::add);

return mFiles;
}
}
67 changes: 4 additions & 63 deletions tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import org.jdom2.Namespace;
import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;
import thredds.inventory.MFile;
import thredds.inventory.MFiles;
import ucar.nc2.constants.CDM;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
Expand All @@ -18,20 +20,13 @@

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.*;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

/**
* Handle configuration for the Radar Server
*/
public class RadarServerConfig {
static ConcurrentHashMap<String, FileSystemProvider> fsproviders = new ConcurrentHashMap<>();

static public List<RadarConfigEntry> readXML(String filename) {
List<RadarConfigEntry> configs = new ArrayList<>();

Expand Down Expand Up @@ -61,7 +56,7 @@ static public List<RadarConfigEntry> readXML(String filename) {
Element meta = dataset.getChild("metadata", catNS);
conf.name = dataset.getAttributeValue("name");
conf.urlPath = dataset.getAttributeValue("path");
conf.dataPath = getPath(AliasTranslator.translateAlias(dataset.getAttributeValue("location")));
conf.mFile = MFiles.create(AliasTranslator.translateAlias(dataset.getAttributeValue("location")));
conf.dataFormat = meta.getChild("dataFormat", catNS).getValue();
conf.stationFile = meta.getChild("stationFile", catNS).getAttributeValue("path");
conf.doc = meta.getChild("documentation", catNS).getValue();
Expand All @@ -88,60 +83,6 @@ static public List<RadarConfigEntry> readXML(String filename) {
return configs;
}

private static Path getPath(String location) throws IOException {
FileSystem fs;

// If we're given an actual URI, use that to find the file system.
// Otherwise, use the default.
if (location.contains(":")) {
URI uri = URI.create(location);

// Fix parsing of s3:// (note two '/') style paths
if (uri.getPath().isEmpty()) {
uri = URI.create(location.replace("//", "///"));
}

location = uri.getPath();
fs = getFS(uri);
} else {
fs = FileSystems.getDefault();
}
return fs.getPath(location);
}

private static FileSystem getFS(URI uri) throws IOException {
FileSystem fs;

try {
fs = getProvider(uri).getFileSystem(uri);
} catch (ProviderNotFoundException e) {
System.out.println("Provider not found: " + e.getMessage());
System.out.println("Using default file system.");
fs = FileSystems.getDefault();
}
return fs;
}

// This is to work around the fact that when we *get* a filesystem, we
// cannot pass in the class loader. This results in custom providers (say
// S3) not being found. However, the filesystem already exists, so the
// filesystem can't be re-created either.
private static FileSystemProvider getProvider(URI uri) throws IOException {
if (fsproviders.containsKey(uri.getScheme())) {
return fsproviders.get(uri.getScheme());
} else {
FileSystem fs;
try {
fs = FileSystems.newFileSystem(uri, new HashMap<String, Object>(),
Thread.currentThread().getContextClassLoader());
} catch (FileSystemAlreadyExistsException e) {
fs = FileSystems.getFileSystem(uri);
}
fsproviders.put(uri.getScheme(), fs.provider());
return fs.provider();
}
}

protected static RadarConfigEntry.GeoInfo readGeospatialCoverage(Element gcElem) {
if (gcElem == null)
return null;
Expand Down Expand Up @@ -232,7 +173,7 @@ protected static TimeDuration readDuration(Element elem) {
}

static public class RadarConfigEntry {
public Path dataPath;
public MFile mFile;
public String name, urlPath, dataFormat, stationFile, doc;
public String dateParseRegex, dateFmt, layout;
public DateRange timeCoverage;
Expand Down
Loading

0 comments on commit 27ac865

Please sign in to comment.