diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java b/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java index 83e7e84087..84c38e23ec 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java @@ -5,6 +5,10 @@ package thredds.server.radarServer2; +import thredds.inventory.CollectionConfig; +import thredds.inventory.MController; +import thredds.inventory.MControllers; +import thredds.inventory.MFile; import ucar.nc2.constants.FeatureType; import ucar.nc2.dt.RadialDatasetSweep; import ucar.nc2.ft.FeatureDatasetFactoryManager; @@ -33,7 +37,7 @@ public enum DirType { private static final long updateIntervalMsec = 600000; private EnumMap> items; - private Path collectionDir; + private MFile collectionDir; private DirectoryStructure structure; private String fileTimeFmt, dataFormat; private java.util.regex.Pattern fileTimeRegex; @@ -46,7 +50,7 @@ public enum DirType { private DateRange timeCoverage; private RadarServerConfig.RadarConfigEntry.GeoInfo geoCoverage; - public RadarDataInventory(Path datasetRoot, int numCrawl) { + public RadarDataInventory(MFile datasetRoot, int numCrawl) { items = new EnumMap<>(DirType.class); collectionDir = datasetRoot; structure = new DirectoryStructure(collectionDir); @@ -56,7 +60,7 @@ public RadarDataInventory(Path datasetRoot, int numCrawl) { nearestWindow = CalendarPeriod.of(1, CalendarPeriod.Field.Hour); } - public Path getCollectionDir() { + public MFile getCollectionDir() { return collectionDir; } @@ -159,11 +163,11 @@ public SimpleDateFormat getFormat() { return sdf; } - public Date getDate(Path path) { - Path relPath = base.relativize(path); + public Date getDate(MFile mFile) { + String relPath = base.relativize(mFile); StringBuilder sb = new StringBuilder(""); for (Integer l : levels) { - sb.append(relPath.getName(l)); + sb.append(Paths.get(relPath).getName(l)); } try { SimpleDateFormat fmt = getFormat(); @@ -174,12 +178,12 @@ public Date getDate(Path path) { } } - private Path base; + private MFile base; private List order; private List keyIndices; - public DirectoryStructure(Path dir) { + public DirectoryStructure(MFile dir) { base = dir; order = new ArrayList<>(); keyIndices = new ArrayList<>(); @@ -194,11 +198,12 @@ public void addSubDir(DirType type, String fmt) { } // Get a key for a path based on station/var - public String getKey(Path path) { - Path relPath = base.relativize(path); + public String getKey(MFile mFile) { + String relPath = base.relativize(mFile); StringBuilder sb = new StringBuilder(""); + for (int ind : keyIndices) { - sb.append(relPath.getName(ind)); + sb.append(Paths.get(relPath).getName(ind)); } return sb.toString(); } @@ -251,7 +256,7 @@ public void addFileTime(String regex, String fmt) { fileTimeFmt = fmt; } - private void findItems(Path start, int level) { + private void findItems(MFile start, int level) { // Add each entry from this level to the appropriate item box // and recurse if (level >= structure.order.size() || level >= structure.maxCrawlDepth) @@ -268,23 +273,17 @@ private void findItems(Path start, int level) { } int crawled = 0; - try (DirectoryStream dirStream = Files.newDirectoryStream(start)) { - for (Path p : dirStream) { - if (Files.isDirectory(p)) { - String item = p.getFileName().toString(); - values.add(item); - // Try to grab station info from some file - // TODO: Fix or remove - // if (entry.type == DirType.Station) - // updateStations(item, p); - if (crawled < maxCrawlItems) { - findItems(p, level + 1); - ++crawled; - } - } + for (MFile subDir : getSubDirs(start)) { + String item = subDir.getName(); + values.add(item); + // Try to grab station info from some file + // TODO: Fix or remove + // if (entry.type == DirType.Station) + // updateStations(item, p); + if (crawled < maxCrawlItems) { + findItems(subDir, level + 1); + ++crawled; } - } catch (IOException e) { - System.out.println("findItems(): Error reading directory: " + start.toString()); } } @@ -354,12 +353,12 @@ public Query newQuery() { public class Query { public class QueryResultItem { - private QueryResultItem(Path f, CalendarDate cd) { + private QueryResultItem(MFile f, CalendarDate cd) { file = f; time = cd; } - public Path file; + public MFile file; public CalendarDate time; } @@ -412,8 +411,8 @@ private boolean checkDate(CalendarDateRange range, CalendarDate d) { return range == null || range.includes(d); } - public Collection results() { - List results = new ArrayList<>(); + public Collection results() throws IOException { + List results = new ArrayList<>(); DirectoryStructure.DirectoryDateMatcher matcher = structure.matcher(); results.add(structure.base); @@ -432,7 +431,7 @@ public Collection results() { // exists. For dates, add the items that are within the filter for (int i = 0; i < structure.order.size(); ++i) { DirectoryStructure.DirEntry entry = structure.order.get(i); - List newResults = new ArrayList<>(); + List newResults = new ArrayList<>(); List queryItem = q.get(entry.type); switch (entry.type) { // Loop over results and add subdirs that are within the @@ -443,15 +442,11 @@ public Collection results() { SimpleDateFormat fmt = matcher.getFormat(); CalendarDateRange dirRange = rangeFromFormat(fmt, range); - for (Path p : results) - try (DirectoryStream dirStream = Files.newDirectoryStream(p)) { - for (Path sub : dirStream) { - Date d = matcher.getDate(sub); - if (d != null && checkDate(dirRange, CalendarDate.of(d))) - newResults.add(sub); - } - } catch (IOException e) { - System.out.println("results(): Error reading dir: " + p.toString()); + for (MFile mFile : results) + for (MFile sub : getSubDirs(mFile)) { + Date d = matcher.getDate(sub); + if (d != null && checkDate(dirRange, CalendarDate.of(d))) + newResults.add(sub); } break; @@ -461,10 +456,11 @@ public Collection results() { case Variable: default: for (Object next : queryItem) { - for (Path p : results) { - Path nextPath = p.resolve(next.toString()); - if (Files.exists(nextPath)) - newResults.add(nextPath); + for (MFile mFile : results) { + MFile nextMFile = mFile.getChild(next.toString() + "/"); + if (nextMFile != null && nextMFile.exists()) { + newResults.add(nextMFile); + } } } } @@ -473,28 +469,24 @@ public Collection results() { // Now get the contents of the remaining directories Collection filteredFiles = new ArrayList<>(); - for (Path p : results) { - try (DirectoryStream dirStream = Files.newDirectoryStream(p)) { - for (Path f : dirStream) { - java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(f.toString()); - if (!regexMatcher.find()) - continue; - - try { - SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt); - fmt.setTimeZone(TimeZone.getTimeZone("UTC")); - Date d = fmt.parse(regexMatcher.group()); - if (d != null) { - CalendarDate cd = CalendarDate.of(d); - if (checkDate(range, cd)) - filteredFiles.add(new QueryResultItem(f, cd)); - } - } catch (ParseException e) { - // Ignore file + for (MFile dir : results) { + for (MFile mFile : getFiles(dir)) { + java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(mFile.getName()); + if (!regexMatcher.find()) + continue; + + try { + SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt); + fmt.setTimeZone(TimeZone.getTimeZone("UTC")); + Date d = fmt.parse(regexMatcher.group()); + if (d != null) { + CalendarDate cd = CalendarDate.of(d); + if (checkDate(range, cd)) + filteredFiles.add(new QueryResultItem(mFile, cd)); } + } catch (ParseException e) { + // Ignore file } - } catch (IOException e) { - System.out.println("results(): Error getting files for: " + p.toString()); } } @@ -524,4 +516,26 @@ public Collection results() { return filteredFiles; } } + + private List getSubDirs(MFile directory) { + final MController mController = MControllers.create(directory.getPath()); + final List mFiles = new ArrayList<>(); + + final CollectionConfig dirs = new CollectionConfig("dirs", directory.getPath(), false, null, null); + final Iterator dirIterator = mController.getSubdirs(dirs, true); + dirIterator.forEachRemaining(mFiles::add); + + return mFiles; + } + + private List getFiles(MFile directory) throws IOException { + final MController mController = MControllers.create(directory.getPath()); + final List mFiles = new ArrayList<>(); + + final CollectionConfig files = new CollectionConfig("files", directory.getPath(), false, null, null); + final Iterator fileIterator = mController.getInventoryTop(files, true); + fileIterator.forEachRemaining(mFiles::add); + + return mFiles; + } } diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java b/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java index cd5559f8be..8ddb6bb273 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java @@ -10,6 +10,8 @@ import org.jdom2.Namespace; import org.jdom2.JDOMException; import org.jdom2.input.SAXBuilder; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.constants.CDM; import ucar.nc2.units.DateRange; import ucar.nc2.units.DateType; @@ -18,20 +20,13 @@ import java.io.File; import java.io.IOException; -import java.net.URI; -import java.nio.file.*; -import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; /** * Handle configuration for the Radar Server */ public class RadarServerConfig { - static ConcurrentHashMap fsproviders = new ConcurrentHashMap<>(); - static public List readXML(String filename) { List configs = new ArrayList<>(); @@ -61,7 +56,7 @@ static public List readXML(String filename) { Element meta = dataset.getChild("metadata", catNS); conf.name = dataset.getAttributeValue("name"); conf.urlPath = dataset.getAttributeValue("path"); - conf.dataPath = getPath(AliasTranslator.translateAlias(dataset.getAttributeValue("location"))); + conf.mFile = MFiles.create(AliasTranslator.translateAlias(dataset.getAttributeValue("location"))); conf.dataFormat = meta.getChild("dataFormat", catNS).getValue(); conf.stationFile = meta.getChild("stationFile", catNS).getAttributeValue("path"); conf.doc = meta.getChild("documentation", catNS).getValue(); @@ -88,60 +83,6 @@ static public List readXML(String filename) { return configs; } - private static Path getPath(String location) throws IOException { - FileSystem fs; - - // If we're given an actual URI, use that to find the file system. - // Otherwise, use the default. - if (location.contains(":")) { - URI uri = URI.create(location); - - // Fix parsing of s3:// (note two '/') style paths - if (uri.getPath().isEmpty()) { - uri = URI.create(location.replace("//", "///")); - } - - location = uri.getPath(); - fs = getFS(uri); - } else { - fs = FileSystems.getDefault(); - } - return fs.getPath(location); - } - - private static FileSystem getFS(URI uri) throws IOException { - FileSystem fs; - - try { - fs = getProvider(uri).getFileSystem(uri); - } catch (ProviderNotFoundException e) { - System.out.println("Provider not found: " + e.getMessage()); - System.out.println("Using default file system."); - fs = FileSystems.getDefault(); - } - return fs; - } - - // This is to work around the fact that when we *get* a filesystem, we - // cannot pass in the class loader. This results in custom providers (say - // S3) not being found. However, the filesystem already exists, so the - // filesystem can't be re-created either. - private static FileSystemProvider getProvider(URI uri) throws IOException { - if (fsproviders.containsKey(uri.getScheme())) { - return fsproviders.get(uri.getScheme()); - } else { - FileSystem fs; - try { - fs = FileSystems.newFileSystem(uri, new HashMap(), - Thread.currentThread().getContextClassLoader()); - } catch (FileSystemAlreadyExistsException e) { - fs = FileSystems.getFileSystem(uri); - } - fsproviders.put(uri.getScheme(), fs.provider()); - return fs.provider(); - } - } - protected static RadarConfigEntry.GeoInfo readGeospatialCoverage(Element gcElem) { if (gcElem == null) return null; @@ -232,7 +173,7 @@ protected static TimeDuration readDuration(Element elem) { } static public class RadarConfigEntry { - public Path dataPath; + public MFile mFile; public String name, urlPath, dataFormat, stationFile, doc; public String dateParseRegex, dateFmt, layout; public DateRange timeCoverage; diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java b/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java index 5cc1dc197f..1f42e4e573 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java @@ -148,7 +148,7 @@ public void init(ContextRefreshedEvent event) { List configs = RadarServerConfig.readXML(contentPath + "/radar/radarCollections.xml"); for (RadarServerConfig.RadarConfigEntry conf : configs) { - RadarDataInventory di = new RadarDataInventory(conf.dataPath, conf.crawlItems); + RadarDataInventory di = new RadarDataInventory(conf.mFile, conf.crawlItems); di.setName(conf.name); di.setDescription(conf.doc); @@ -495,14 +495,14 @@ private HttpEntity makeCatalog(String dataset, RadarDataInventory inv, R for (RadarDataInventory.Query.QueryResultItem i : res) { DatasetBuilder fileDB = new DatasetBuilder(mainDB); - fileDB.setName(i.file.getFileName().toString()); + fileDB.setName(i.file.getName()); fileDB.put(Dataset.Id, String.valueOf(i.file.hashCode())); fileDB.put(Dataset.Dates, new DateType(i.time.toString(), null, "start of ob", i.time.getCalendar())); // TODO: Does this need to be converted from the on-disk path // to a mapped url path? - fileDB.put(Dataset.UrlPath, inv.getCollectionDir().relativize(i.file).toString()); + fileDB.put(Dataset.UrlPath, inv.getCollectionDir().relativize(i.file)); mainDB.addDataset(fileDB); }