using plugin in path that is set after instantiation of a data mode fails. (#875)

Ensure that Python storages are reloaded if the Python storage plugin path has changed.

---------

Co-authored-by: Jesper Friis <[email protected]>

Author: francescalb

examples/storage_plugin/main.py

@@ -5,7 +5,12 @@
 
 # Set search path to our user-defined storage plugin
 thisdir = Path(__file__).resolve().parent
+#dlite.python_storage_plugin_path.append(thisdir  / "plugins")
+
+dlite.storage_path.append(thisdir / "entities" / "TempProfile.json")
+DataModel = dlite.get_instance("http://onto-ns.com/meta/0.1/TempProfile")
 dlite.python_storage_plugin_path.append(thisdir  / "plugins")
+#DataModel = dlite.get_instance("http://onto-ns.com/meta/0.1/TempProfile")
 
 # Create instance from dataset
 inst = dlite.Instance.from_location("tempprofile", thisdir / "dataset.txt",

src/dlite-storage-plugins.c

@@ -31,7 +31,7 @@ struct _DLiteStoragePluginIter {
 /* Global variables for dlite-storage-plugins */
 typedef struct {
   PluginInfo *storage_plugin_info;  /* reference to storage plugin info */
-  unsigned char storage_plugin_path_hash[32];  /* Sha256 hash of plugin paths */
+  unsigned char storage_plugin_path_hash[32];  /* Sha3 hash of plugin paths */
 } Globals;
 
 
@@ -119,7 +119,7 @@ const DLiteStoragePlugin *dlite_storage_plugin_get(const char *name)
 
   /* ...otherwise, if any plugin path has changed, reload all plugins
      and try again */
-  if (pathshash(hash, sizeof(hash), &info->paths) == 0) {
+  if (pathshash(hash, sizeof(hash), &info->paths, DSL_EXT) == 0) {
 
     if (memcmp(g->storage_plugin_path_hash, hash, sizeof(hash)) != 0) {
       plugin_load_all(info);

src/pathshash.c

@@ -6,24 +6,30 @@
 #include "pathshash.h"
 
 /*
-  Calculate sha3 hash of `paths` and stores it in `hash`.
+  Calculate sha3 hash of all files in `paths` and stores it in `hash`.
   `hashsize` is the size of `hash` in bytes. Should be 32, 48 or 64.
+
+  If `pattern` is given, it should be a glob pattern for selecting what
+  files to include.
+
   Returns non-zero on error.
 */
-int pathshash(unsigned char *hash, int hashsize, const FUPaths *paths)
+int pathshash(unsigned char *hash, int hashsize, const FUPaths *paths,
+              const char *pattern)
 {
   sha3_context c;
   FUIter *iter;
   unsigned bitsize = hashsize * 8;
   const unsigned char *buf;
   const char *path;
-  if (!(iter = fu_pathsiter_init(paths, NULL))) return 1;
+  if (!(iter = fu_startmatch(pattern, paths)))
+    return err(1, "cannot initiate paths iterator (%d)", (paths) ? (int)paths->n : -1);
   if (sha3_Init(&c, bitsize))
     return err(1, "invalid hash size: %d bytes", hashsize);
-  while ((path = fu_pathsiter_next(iter)))
+  while ((path = fu_nextmatch(iter)))
     sha3_Update(&c, path, strlen(path));
   buf = sha3_Finalize(&c);
-  fu_pathsiter_deinit(iter);
+  fu_endmatch(iter);
   memcpy(hash, buf, hashsize);
   return 0;
 }

src/pathshash.h

@@ -4,11 +4,16 @@
 #include "utils/fileutils.h"
 
 /**
-  Calculate sha3 hash of `paths` and stores it in `hash`.
+  Calculate sha3 hash of all files in `paths` and stores it in `hash`.
   `hashsize` is the size of `hash` in bytes. Should be 32, 48 or 64.
+
+  If `pattern` is given, it should be a glob pattern for selecting what
+  files to include.
+
   Returns non-zero on error.
 */
-int pathshash(unsigned char *hash, int hashsize, const FUPaths *paths);
+int pathshash(unsigned char *hash, int hashsize, const FUPaths *paths,
+              const char *pattern);
 
 
 #endif  /* _PATHSHASH_H */

src/pyembed/dlite-python-storage.c

@@ -11,6 +11,7 @@
 #include "config-paths.h"
 
 #include "utils/strutils.h"
+#include "pathshash.h"
 #include "dlite.h"
 #include "dlite-macros.h"
 #include "dlite-misc.h"
@@ -28,13 +29,13 @@ typedef PyObject *(*InstanceConverter)(DLiteInstance *inst);
 
 /* Global state for this module */
 typedef struct {
-  FUPaths paths;              /* Python storage paths */
-  int initialised;            /* Whether `paths` is initiated */
-  int modified;               /* Whether `paths` is modified */
-  PyObject *loaded_storages;  /* Cache with all loaded python storage plugins */
-  char **failed_paths;        /* NULL-terminated array of paths to storages
-                                 that fail to load. */
-  size_t failed_len;          /* Allocated length of `failed_paths`. */
+  FUPaths paths;                 /* Python storage paths */
+  int initialised;               /* Whether `paths` is initiated */
+  unsigned char paths_hash[32];  /* Sha3 hash of plugin paths */
+  PyObject *loaded_storages;     /* Cache with all loaded python storage plugins */
+  char **failed_paths;           /* NULL-terminated array of paths to storages
+                                    that fail to load. */
+  size_t failed_len;             /* Allocated length of `failed_paths`. */
 } PythonStorageGlobals;
 
 
@@ -102,7 +103,6 @@ FUPaths *dlite_python_storage_paths(void)
     if (s < 0) return dlite_err(1, "error initialising dlite python storage "
                                 "plugin dirs"), NULL;
     g->initialised = 1;
-    g->modified = 0;
 
     /* Make sure that dlite DLLs are added to the library search path */
     dlite_add_dll_path();
@@ -119,7 +119,6 @@ void dlite_python_storage_paths_clear(void)
   if (g->initialised) {
     fu_paths_deinit(&g->paths);
     g->initialised = 0;
-    g->modified = 0;
   }
 }
 
@@ -131,14 +130,9 @@ void dlite_python_storage_paths_clear(void)
 */
 int dlite_python_storage_paths_insert(const char *path, int n)
 {
-  int stat;
   const FUPaths *paths;
   if (!(paths = dlite_python_storage_paths())) return -1;
-  if ((stat = fu_paths_insert((FUPaths *)paths, path, n))) {
-    PythonStorageGlobals *g = get_globals();
-    g->modified = 1;
-  }
-  return stat;
+  return fu_paths_insert((FUPaths *)paths, path, n);
 }
 
 /*
@@ -147,14 +141,9 @@ int dlite_python_storage_paths_insert(const char *path, int n)
 */
 int dlite_python_storage_paths_append(const char *path)
 {
-  int stat;
   const FUPaths *paths;
   if (!(paths = dlite_python_storage_paths())) return -1;
-  if ((stat = fu_paths_append((FUPaths *)paths, path))) {
-    PythonStorageGlobals *g = get_globals();
-    g->modified = 1;
-  }
-  return stat;
+  return fu_paths_append((FUPaths *)paths, path);
 }
 
 /*
@@ -163,14 +152,9 @@ int dlite_python_storage_paths_append(const char *path)
 */
 int dlite_python_storage_paths_remove_index(int index)
 {
-  int stat;
   const FUPaths *paths;
   if (!(paths = dlite_python_storage_paths())) return -1;
-  if ((stat = fu_paths_remove_index((FUPaths *)paths, index))) {
-    PythonStorageGlobals *g = get_globals();
-    g->modified = 1;
-  }
-  return stat;
+  return fu_paths_remove_index((FUPaths *)paths, index);
 }
 
 /*
@@ -194,15 +178,17 @@ const char **dlite_python_storage_paths_get(void)
 void *dlite_python_storage_load(void)
 {
   PyObject *storagebase;
+  unsigned char hash[32];
+  const FUPaths *paths;
   PythonStorageGlobals *g = get_globals();
 
   if (!(storagebase = dlite_python_storage_base())) return NULL;
+  if (!(paths = dlite_python_storage_paths())) return NULL;
+  if (pathshash(hash, sizeof(hash), paths, "*.py")) return NULL;
 
-  if (!g->loaded_storages || g->modified) {
-    const FUPaths *paths;
+  if (!g->loaded_storages || memcmp(g->paths_hash, hash, sizeof(hash)) != 0) {
+    memcpy(g->paths_hash, hash, sizeof(hash));
     if (g->loaded_storages) dlite_python_storage_unload();
-    if (!(paths = dlite_python_storage_paths())) return NULL;
-
     g->loaded_storages = dlite_pyembed_load_plugins((FUPaths *)paths,
                                                     storagebase,
                                                     &g->failed_paths,