UCT/GAUDI: Free the fds when called md_close (openucx#11017)

*UCT/GAUDI: Free the fds when called md_close

Author: yihuaxu

src/uct/gaudi/base/gaudi_base.c

@@ -20,16 +20,37 @@
 #include <hlthunk.h>
 #include <synapse_api.h>
 
-int uct_gaudi_base_get_fd(int device_id) {
+int uct_gaudi_base_get_fd(int device_id, bool *fd_created) {
     synDeviceInfo deviceInfo;
 
     if (synDeviceGetInfo(-1, &deviceInfo) != synSuccess) {
-       return hlthunk_open_by_module_id(device_id);
+        int fd = hlthunk_open_by_module_id(device_id);
+        if (fd <0) {
+            ucs_info("Failed to get device fd via hlthunk_open_by_module_id, id %d", device_id);
+	         fd = hlthunk_open(HLTHUNK_DEVICE_DONT_CARE, NULL);
+        }
+
+        if (fd >=0 && fd_created != NULL) {
+            *fd_created = true;
+        }
+        return fd;
     }
 
     return deviceInfo.fd;
 }
 
+void uct_gaudi_base_close_fd(int fd, bool fd_created) {
+    if (fd_created && fd >= 0) {
+        hlthunk_close(fd);
+    }
+}
+
+void uct_gaudi_base_close_dmabuf_fd(int fd) {
+    if (fd >= 0) {
+        close(fd);
+    }
+}
+
 ucs_status_t uct_gaudi_base_get_sysdev(int fd, ucs_sys_device_t* sys_dev) {
     ucs_status_t status;
     char pci_bus_id[13];

src/uct/gaudi/base/gaudi_base.h

@@ -6,11 +6,14 @@
 #ifndef GAUDI_BASE_H_
 #define GAUDI_BASE_H_
 
+#include <stdbool.h>
 #include <uct/base/uct_iface.h>
 #include <uct/base/uct_md.h>
 #include "scal.h"
 
-int uct_gaudi_base_get_fd(int device_id);
+int uct_gaudi_base_get_fd(int device_id, bool *fd_created);
+void uct_gaudi_base_close_fd(int fd, bool fd_created);
+void uct_gaudi_base_close_dmabuf_fd(int fd);
 ucs_status_t uct_gaudi_base_get_sysdev(int fd, ucs_sys_device_t* sys_dev);
 ucs_status_t uct_gaudi_base_get_info(int fd, uint64_t *device_base_allocated_address, uint64_t *device_base_address,
                                 uint64_t *totalSize, int *dmabuf_fd);

src/uct/gaudi/gaudi_gdr/gaudi_gdr_md.c

@@ -42,12 +42,8 @@ static ucs_status_t uct_gaudi_md_query(uct_md_h md, uct_md_attr_v2_t *attr)
 static void uct_gaudi_md_close(uct_md_h uct_md)
 {
     uct_gaudi_md_t *md = ucs_derived_of(uct_md, uct_gaudi_md_t);
-    if (md->dmabuf_fd >= 0) {
-        close(md->dmabuf_fd);
-    }
-    if (md->fd >= 0) {
-        close(md->fd);
-    }        
+    uct_gaudi_base_close_dmabuf_fd(md->dmabuf_fd);
+    uct_gaudi_base_close_fd(md->fd, md->fd_created);
     ucs_free(md);
 }
 
@@ -156,6 +152,7 @@ uct_gaudi_md_open(uct_component_h component, const char *md_name,
                                                      uct_gaudi_md_config_t);
     uct_gaudi_md_t *md;
     ucs_status_t status;
+    bool fd_created = false;
     int fd;
 
     md = ucs_malloc(sizeof(uct_gaudi_md_t), "uct_gaudi_md_t");
@@ -164,7 +161,7 @@ uct_gaudi_md_open(uct_component_h component, const char *md_name,
         return UCS_ERR_NO_MEMORY;
     }
 
-    fd = uct_gaudi_base_get_fd(config->device_id);
+    fd = uct_gaudi_base_get_fd(config->device_id, &fd_created);
     if (fd <0) {
         ucs_error("Failed to get device fd");
 	    status = UCS_ERR_NO_DEVICE;
@@ -187,7 +184,8 @@ uct_gaudi_md_open(uct_component_h component, const char *md_name,
         goto err_close_dmabuf;
     }
 
-    md->fd              = dup(fd);
+    md->fd              = fd;
+    md->fd_created      = fd_created;
     md->super.ops       = &md_ops;
     md->super.component = &uct_gaudi_gdr_component;
 

src/uct/gaudi/gaudi_gdr/gaudi_gdr_md.h

@@ -6,6 +6,7 @@
 #ifndef GAUDI_MD_H
 #define GAUDI_MD_H
 
+#include <stdbool.h>
 #include <uct/base/uct_md.h>
 #include <ucs/config/types.h>
 
@@ -14,6 +15,7 @@ extern uct_component_t uct_gaudi_gdr_component;
 typedef struct uct_gaudi_md {
     uct_md_t            super;
     int                 fd;
+    bool                fd_created;
     uint64_t            device_base_allocated_address;
     uint64_t            device_base_address;
     uint64_t            totalSize;