src/hmem_ze: Move creation of ze command queues out of initialization and into the copy function.

In the case of ULLS (ultra low latency submission) where ze gets initialized and never used;
creating a command queue spawns a thread that repeatedly submits 0 byte copies to the command
queue so that it never gets swapped out. If ze is initialized and never used it creates a
performance impact because it is never used. Move the initialization to the copy function to
remove this overhead.

Signed-off-by: zdworkin <[email protected]>

Author: zachdworkin

src/hmem_ze.c

@@ -754,15 +754,6 @@ int ze_hmem_init(void)
 		if (ze_ret)
 			goto err;
 
-		cq_desc.ordinal = ordinals[num_devices];
-		cq_desc.index = indices[num_devices];
-		ze_ret = ofi_zeCommandQueueCreate(context,
-						  devices[num_devices],
-						  &cq_desc,
-						  &cmd_queue[num_devices]);
-		if (ze_ret)
-			goto err;
-
 		for (i = 0; i < count; i++) {
 			if (ofi_zeDeviceCanAccessPeer(devices[num_devices],
 					devices[i], &access) || !access)
@@ -793,6 +784,17 @@ int ze_hmem_copy(uint64_t device, void *dst, const void *src, size_t size)
 		return 0;
 	}
 
+	if (!cmd_queue[device]) {
+		cq_desc.ordinal = ordinals[device];
+		cq_desc.index = indices[device];
+		ze_ret = ofi_zeCommandQueueCreate(context,
+						  devices[device],
+						  &cq_desc,
+						  &cmd_queue[device]);
+		if (ze_ret)
+			goto err;
+	}
+
 	cl_desc.commandQueueGroupOrdinal = ordinals[dev_id];
 	ze_ret = ofi_zeCommandListCreate(context, devices[dev_id], &cl_desc,
 					 &cmd_list);