Skip to content

Commit 3a89a04

Browse files
rhc54rhc54
committed
Provide access to resource usage for processes and nodes
Operating systems typically maintain a running measure of resource utilization by active processes. This includes metrics on CPU utilization, disk accesses, memory size, and network activity. Define a set of attributes by which these these metrics can be requested and returned. Attributes are used as a means of providing for later extension to include a broader range of metrics. Signed-off-by: Ralph Castain <[email protected]>
1 parent 46c63a5 commit 3a89a04

File tree

4 files changed

+366
-15
lines changed

4 files changed

+366
-15
lines changed

‎Chap_API_Job_Mgmt.tex‎

Lines changed: 323 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -605,7 +605,6 @@ \subsection{Job control attributes}
605605
When recursively cleaning subdirectories, do not remove the top-level directory (the one given in the cleanup request).
606606
}
607607

608-
609608
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
610609
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
611610
\section{Process and Job Monitoring}
@@ -668,6 +667,7 @@ \subsection{\code{PMIx_Process_monitor}}
668667
\pasteAttributeItem{PMIX_MONITOR_FILE_CHECK_TIME}
669668
\pasteAttributeItem{PMIX_MONITOR_FILE_DROPS}
670669
\pasteAttributeItem{PMIX_SEND_HEARTBEAT}
670+
\pasteAttributeItem{PMIX_MONITOR_RESOURCE_USAGE}
671671

672672
\optattrend
673673

@@ -746,7 +746,7 @@ \subsection{\code{PMIx_Process_monitor_nb}}
746746
\pasteAttributeItem{PMIX_MONITOR_FILE_CHECK_TIME}
747747
\pasteAttributeItem{PMIX_MONITOR_FILE_DROPS}
748748
\pasteAttributeItem{PMIX_SEND_HEARTBEAT}
749-
749+
\pasteAttributeItem{PMIX_MONITOR_RESOURCE_USAGE}
750750
\optattrend
751751

752752
%%%%
@@ -850,6 +850,327 @@ \subsection{Monitoring attributes}
850850
\declareAttribute{PMIX_MONITOR_FILE_DROPS}{"pmix.monitor.fdrop"}{uint32_t}{
851851
Number of file checks that can be missed before generating the event.
852852
}
853+
%
854+
\declareAttributeProvisional{PMIX_MONITOR_RESOURCE_RATE}{pmix.monitor.resrate}{uint64_t}{
855+
Monitor resource usage every N seconds, where N is the value provided by the attribute.
856+
}
857+
%
858+
\declareAttributeProvisional{PMIX_MONITOR_RESOURCE_USAGE}{"pmix.monitor.resuse"}{pmix_data_array_t*}{
859+
Monitor the resources specified in the provided \refstruct{pmix_data_array_t}. Resource types may
860+
include any of the following:
861+
862+
\begin{itemize}
863+
\item \refattr{PMIX_MONITOR_RESOURCE_RATE}. If not provided, then the request will be treated as a one-shot
864+
sampling of resource usage.
865+
\item \refattr{PMIX_PROC_RESOURCE_USAGE}. If the \refstruct{pmix_data_array_t} is empty, then
866+
all process resource usage values shall be returned for all processes in the session.
867+
Optionally, the array of \refstruct{pmix_info_t} can specify the processes to be monitored, and/or the particular attributes to be included. Note that the values in the provided structures will be
868+
ignored (i.e., only the attribute keys are relevant) except where noted, and that the
869+
\refattr{PMIX_PROC_SAMPLE_TIME} will always be included in the returned data (there is no
870+
need to include it in the request). Optional attributes include:
871+
\begin{itemize}
872+
\item \refattr{PMIX_PROCID}. Optionally specify the process to be monitored. Can include a
873+
\refconst{PMIX_RANK_WILDCARD} to indicate that all processes
874+
from a given namespace are to be included. If omitted, then
875+
all processes in the session will be monitored. May be included
876+
multiple times to fully specify all processes to be included.
877+
\item \refattr{PMIX_HOSTNAME}. Include the hostname where the process is located.
878+
\item \refattr{PMIX_PROC_PID} Optionally specify the process to be monitored.
879+
\item \refattr{PMIX_PROC_OS_STATE}
880+
\item \refattr{PMIX_PROC_TIME}
881+
\item \refattr{PMIX_PROC_PERCENT_CPU}
882+
\item \refattr{PMIX_PROC_PRIORITY}
883+
\item \refattr{PMIX_PROC_NUM_THREADS}
884+
\item \refattr{PMIX_PROC_PSS}
885+
\item \refattr{PMIX_PROC_VSIZE}
886+
\item \refattr{PMIX_PROC_RSS}
887+
\item \refattr{PMIX_PROC_PEAK_VSIZE}
888+
\item \refattr{PMIX_PROC_CPU}
889+
\item \refattr{PMIX_PROC_SAMPLE_TIME}
890+
\end{itemize}
891+
\item \refattr{PMIX_NODE_RESOURCE_USAGE}. If the \refstruct{pmix_data_array_t} is empty, then
892+
all node resource usage values shall be returned for all nodes in the session.
893+
Optionally, the array of \refstruct{pmix_info_t} can specify the nodes to be monitored (using the \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID} attributes), and/or the particular attributes to be included. Note that the values in the provided structures will be
894+
ignored (i.e., only the attribute keys are relevant) except where noted, and that the
895+
\refattr{PMIX_NODE_SAMPLE_TIME} will always be included in the returned data (there is no
896+
need to include it in the request). Optional
897+
attributes include:
898+
\begin{itemize}
899+
\item \refattr{PMIX_HOSTNAME}. Optionally specify the node to be monitored. May be included multiple
900+
times to fully specify all nodes to be included. Only
901+
hostname or node ID need be included (not both). If omitted, then all nodes in the session
902+
shall be monitored.
903+
\item \refattr{PMIX_NODEID}. Optionally specify the process to be monitored. May be included multiple
904+
times to fully specify all nodes to be included. Only
905+
hostname or node ID need be included (not both). If omitted, then all nodes in the session
906+
shall be monitored.
907+
\item \refattr{PMIX_NODE_LOAD_AVG}
908+
\item \refattr{PMIX_NODE_LOAD_AVG5}
909+
\item \refattr{PMIX_NODE_LOAD_AVG15}
910+
\item \refattr{PMIX_NODE_MEM_TOTAL}
911+
\item \refattr{PMIX_NODE_MEM_FREE}
912+
\item \refattr{PMIX_NODE_MEM_BUFFERS}
913+
\item \refattr{PMIX_NODE_MEM_CACHED}
914+
\item \refattr{PMIX_NODE_MEM_SWAP_CACHED}
915+
\item \refattr{PMIX_NODE_MEM_SWAP_TOTAL}
916+
\item \refattr{PMIX_NODE_MEM_SWAP_FREE}
917+
\item \refattr{PMIX_NODE_MEM_MAPPED}
918+
\item \refattr{PMIX_DISK_RESOURCE_USAGE}. If the \refstruct{pmix_data_array_t} is empty, then
919+
all disk resource usage values shall be returned for all disks attached to the node.
920+
Optionally, the array of \refstruct{pmix_info_t} can specify the disks to be monitored (using the \refattr{PMIX_DISK_ID} attribute), and/or the particular attributes to be included. Note that the values in the provided structures will be
921+
ignored (i.e., only the attribute keys are relevant) except where noted, and that the
922+
\refattr{PMIX_DISK_SAMPLE_TIME} will always be included in the returned data (there is no
923+
need to include it in the request). Optional
924+
attributes include:
925+
\begin{itemize}
926+
\item \refattr{PMIX_DISK_ID}. Optionally specify the disk to be monitored. If omitted, then all disks
927+
attached to the node will be monitored.
928+
\item \refattr{PMIX_DISK_READ_COMPLETED}
929+
\item \refattr{PMIX_DISK_READ_MERGED}
930+
\item \refattr{PMIX_DISK_READ_SECTORS}
931+
\item \refattr{PMIX_DISK_READ_MILLISEC}
932+
\item \refattr{PMIX_DISK_WRITE_COMPLETED}
933+
\item \refattr{PMIX_DISK_WRITE_MERGED}
934+
\item \refattr{PMIX_DISK_WRITE_SECTORS}
935+
\item \refattr{PMIX_DISK_WRITE_MILLISEC}
936+
\item \refattr{PMIX_DISK_IO_IN_PROGRESS}
937+
\item \refattr{PMIX_DISK_IO_MILLISEC}
938+
\item \refattr{PMIX_DISK_IO_WEIGHTED}
939+
\end{itemize}
940+
\item \refattr{PMIX_NETWORK_RESOURCE_USAGE}. If the \refstruct{pmix_data_array_t} is empty, then
941+
all network resource usage values shall be returned for all interfaces on the node.
942+
Optionally, the array of \refstruct{pmix_info_t} can specify the networks to be monitored (using the \refattr{PMIX_NETWORK_ID} attribute), and/or the particular attributes to be included. Note that the values in the provided structures will be
943+
ignored (i.e., only the attribute keys are relevant) except where noted, and that the
944+
\refattr{PMIX_NET_SAMPLE_TIME} will always be included in the returned data (there is no
945+
need to include it in the request). Optional
946+
attributes include:
947+
\begin{itemize}
948+
\item \refattr{PMIX_NETWORK_ID}. Optionally specify the interface to be monitored. If omitted, then all
949+
interfaces on the node will be monitored.
950+
\item \refattr{PMIX_NET_RECVD_BYTES}
951+
\item \refattr{PMIX_NET_RECVD_PCKTS}
952+
\item \refattr{PMIX_NET_RECVD_ERRS}
953+
\item \refattr{PMIX_NET_SENT_BYTES}
954+
\item \refattr{PMIX_NET_SENT_PCKTS}
955+
\item \refattr{PMIX_NET_SENT_ERRS}
956+
\end{itemize}
957+
\end{itemize}
958+
\end{itemize}
959+
}
960+
961+
%%%%%%%%%%%
962+
\versionMarkerProvisional{6.0}
963+
\subsection{Resource usage attributes}
964+
\label{api:struct:attributes:resusage}
965+
966+
Operating systems typically maintain a running measure of resource utilization by active processes,
967+
attached disks, and local network interfaces.
968+
Though the precise values being tracked can vary by \ac{OS} flavor and local configuration, the following
969+
attributes are defined to provide a means for requesting and returning the available metrics.
970+
971+
\subsubsection{Process resource usage}
972+
973+
\declareAttributeProvisional{PMIX_PROC_RESOURCE_USAGE}{"pmix.proc.res"}{pmix_data_array_t*}{
974+
An array of \refstruct{pmix_info_t} describing the resource usage of the specified process, with
975+
the first element containing the ID of the process (marked by either the \refattr{PMIX_PROCID}
976+
or \refattr{PMIX_PROC_PID} key)
977+
whose usage is reported in the array. The list of included information may vary across
978+
implementations and \acp{OS}, depending upon availability and access restrictions. Except for
979+
the process ID as the first element, ordering of information in the array is arbitrary.
980+
}
981+
982+
Optional information that may be included (see \href{https://www.kernel.org/doc/html/latest/filesystems/proc.html}{PROCSTATS} for a detailed description of the following fields):
983+
\begin{itemize}
984+
\item \refattr{PMIX_HOSTNAME}. Either the hostname or \refattr{PMIX_NODEID} may be provided.
985+
\item \refattr{PMIX_PROC_PID}
986+
\item \refattr{PMIX_CMD_LINE}. Typically limited solely to the argv[0] for the process
987+
\item \declareAttributeProvisional{PMIX_PROC_OS_STATE}{"pmix.proc.osstate"}{char*}{
988+
The state of the process as reported by the \ac{OS} - for Linux, this is expressed as a single character.
989+
}
990+
\item \declareAttributeProvisional{PMIX_PROC_TIME}{"pmix.proc.time"}{struct timeval}{
991+
Cumulative CPU time
992+
}
993+
\item \declareAttributeProvisional{PMIX_PROC_PERCENT_CPU}{"pmix.proc.pcpu"}{float}{
994+
Percent cpu utilization by the process. Often, it is the CPU time used divided by the time the process has
995+
been running (cputime/realtime ratio), expressed as a percentage.
996+
}
997+
\item \declareAttributeProvisional{PMIX_PROC_PRIORITY}{"pmix.proc.pri"}{int32_t}{
998+
Priority of the process. Higher number means higher priority.
999+
}
1000+
\item \declareAttributeProvisional{PMIX_PROC_NUM_THREADS}{"pmix.proc.nthr"}{uint16_t}{
1001+
Number of threads operating in the process
1002+
}
1003+
\item \declareAttributeProvisional{PMIX_PROC_PSS}{"pmix.proc.pss"}{float}{
1004+
Proportional share size, the non-swapped physical memory, with shared memory
1005+
proportionally accounted to all tasks mapping it (MBytes)
1006+
}
1007+
\item \declareAttributeProvisional{PMIX_PROC_VSIZE}{"pmix.proc.vsize"}{float}{
1008+
Virtual memory size of the process (MBytes)
1009+
}
1010+
\item \declareAttributeProvisional{PMIX_PROC_RSS}{"pmix.proc.rss"}{float}{
1011+
Resident set size, the non-swapped physical memory that a task has used (MBytes)
1012+
}
1013+
\item \declareAttributeProvisional{PMIX_PROC_PEAK_VSIZE}{"pmix.proc.pkvsize"}{float}{
1014+
Peak virtual memory size of the process (MBytes)
1015+
}
1016+
\item \declareAttributeProvisional{PMIX_PROC_CPU}{"pmix.proc.cpu"}{uint16_t}{
1017+
Processor that process last executed on
1018+
}
1019+
\item \declareAttributeProvisional{PMIX_PROC_SAMPLE_TIME}{"pmix.proc.samptime"}{struct timeval}{
1020+
Time when sample was taken
1021+
}
1022+
\end{itemize}
1023+
1024+
1025+
\subsubsection{Disk resource usage}
1026+
1027+
\declareAttributeProvisional{PMIX_DISK_ID}{"pmix.disk.id"}{char*}{
1028+
String identifier of a disk
1029+
}
1030+
1031+
\declareAttributeProvisional{PMIX_DISK_RESOURCE_USAGE}{"pmix.disk.res"}{pmix_data_array_t*}{
1032+
An array of \refstruct{pmix_info_t} describing the resource usage of the specified disk, with
1033+
the first element containing the string name of the disk (marked by the \refattr{PMIX_DISK_ID} key)
1034+
whose usage is reported in the array. The list of included information may vary across
1035+
implementations and \acp{OS}, depending upon availability and access restrictions. Except for
1036+
the disk ID as the first element, ordering of information in the array is arbitrary.
1037+
}
1038+
1039+
Optional information that may be included (see \href{https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)}{IOSTATS} for a detailed description of the following fields):
1040+
\begin{itemize}
1041+
\item \declareAttributeProvisional{PMIX_DISK_READ_COMPLETED}{"pmix.disk.rdscomp"}{uint64_t}{
1042+
Number of completed read operations
1043+
}
1044+
\item \declareAttributeProvisional{PMIX_DISK_READ_MERGED}{"pmix.disk.rdsmrgd"}{uint64_t}{
1045+
Number of merged reads
1046+
}
1047+
\item \declareAttributeProvisional{PMIX_DISK_READ_SECTORS}{"pmix.disk.rdsct"}{uint64_t}{
1048+
Number of sectors read
1049+
}
1050+
\item \declareAttributeProvisional{PMIX_DISK_READ_MILLISEC}{"pmix.disk.rdms"}{uint64_t}{
1051+
Number of milliseconds spent reading the disk
1052+
}
1053+
\item \declareAttributeProvisional{PMIX_DISK_WRITE_COMPLETED}{"pmix.disk.wtscomp"}{uint64_t}{
1054+
Number of completed write operations
1055+
}
1056+
\item \declareAttributeProvisional{PMIX_DISK_WRITE_MERGED}{"pmix.disk.wtsmrgd"}{uint64_t}{
1057+
Number of merged write operations
1058+
}
1059+
\item \declareAttributeProvisional{PMIX_DISK_WRITE_SECTORS}{"pmix.disk.wtsct"}{uint64_t}{
1060+
Number of sectors written
1061+
}
1062+
\item \declareAttributeProvisional{PMIX_DISK_WRITE_MILLISEC}{"pmix.disk.wtms"}{uint64_t}{
1063+
Number of milliseconds spent writing to the disk
1064+
}
1065+
\item \declareAttributeProvisional{PMIX_DISK_IO_IN_PROGRESS}{"pmix.disk.ios"}{uint64_t}{
1066+
Number of disk IO operations in progress
1067+
}
1068+
\item \declareAttributeProvisional{PMIX_DISK_IO_MILLISEC}{"pmix.disk.ioms"}{uint64_t}{
1069+
Number of milliseconds spent in IO operations
1070+
}
1071+
\item \declareAttributeProvisional{PMIX_DISK_IO_WEIGHTED}{"pmix.disk.iowght"}{uint64_t}{
1072+
Number of IOs in progress times the number of milliseconds spent doing IO since
1073+
last update of the field - indicator of backlog that may be accumulating
1074+
}
1075+
\item \declareAttributeProvisional{PMIX_DISK_SAMPLE_TIME}{"pmix.disk.samptime"}{struct timeval}{
1076+
Time when sample was taken
1077+
}
1078+
\end{itemize}
1079+
1080+
\subsubsection{Network resource usage}
1081+
1082+
\declareAttributeProvisional{PMIX_NETWORK_ID}{"pmix.net.id"}{char*}{
1083+
String identifier of a network interface
1084+
}
1085+
1086+
\declareAttributeProvisional{PMIX_NETWORK_RESOURCE_USAGE}{"pmix.net.res"}{pmix_data_array_t*}{
1087+
An array of \refstruct{pmix_info_t} describing the resource usage of the specified network, with
1088+
the first element containing the string name of the interface (marked by the \refattr{PMIX_NETWORK_ID} key)
1089+
whose usage is reported in the array. The list of included information may vary across
1090+
implementations and \acp{OS}, depending upon availability and access restrictions. Except for
1091+
the network ID as the first element, ordering of information in the array is arbitrary.
1092+
}
1093+
1094+
Optional information that may be included (see \href{https://www.kernel.org/doc/html/latest/networking/statistics.html}{NETSTATS} for a detailed description of the following fields):
1095+
\begin{itemize}
1096+
\item \declareAttributeProvisional{PMIX_NET_RECVD_BYTES}{"pmix.net.rcb"}{uint64_t}{
1097+
Number of bytes received
1098+
}
1099+
\item \declareAttributeProvisional{PMIX_NET_RECVD_PCKTS}{"pmix.net.rcp"}{uint64_t}{
1100+
Number of packets received
1101+
}
1102+
\item \declareAttributeProvisional{PMIX_NET_RECVD_ERRS}{"pmix.net.rcerr"}{uint64_t}{
1103+
Number of receive errors
1104+
}
1105+
\item \declareAttributeProvisional{PMIX_NET_SENT_BYTES}{"pmix.net.sntb"}{uint64_t}{
1106+
Number of bytes sent
1107+
}
1108+
\item \declareAttributeProvisional{PMIX_NET_SENT_PCKTS}{"pmix.net.sntp"}{uint64_t}{
1109+
Number of packets sent
1110+
}
1111+
\item \declareAttributeProvisional{PMIX_NET_SENT_ERRS}{"pmix.net.snterr"}{uint64_t}{
1112+
Number of send errors
1113+
}
1114+
\item \declareAttributeProvisional{PMIX_NET_SAMPLE_TIME}{"pmix.net.samptime"}{struct timeval}{
1115+
Time when sample was taken
1116+
}
1117+
\end{itemize}
1118+
1119+
1120+
\subsubsection{Node resource usage}
1121+
1122+
\declareAttributeProvisional{PMIX_NODE_RESOURCE_USAGE}{"pmix.node.res"}{pmix_data_array_t*}{
1123+
An array of \refstruct{pmix_info_t} describing the overall resource usage on the specified node,
1124+
with the first element containing
1125+
the ID of the node (marked by the \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID} key) whose usage
1126+
is reported in the array. The list of included information may vary across
1127+
implementations and \acp{OS}, depending upon availability and access restrictions. Except for
1128+
the node ID as the first element, ordering of information in the array is arbitrary.
1129+
}
1130+
1131+
Optional information that may be included (see \href{https://www.kernel.org/doc/html/latest/filesystems/proc.html#kernel-data}{KERNEL} and \href{https://www.kernel.org/doc/html/latest/filesystems/proc.html#meminfo}{MEMINFO} for a detailed description of the following fields):
1132+
\begin{itemize}
1133+
\item \declareAttributeProvisional{PMIX_NODE_LOAD_AVG}{"pmix.node.la"}{float}{
1134+
Load average of last minute
1135+
}
1136+
\item \declareAttributeProvisional{PMIX_NODE_LOAD_AVG5}{"pmix.node.la5"}{float}{
1137+
Load average of last five minutes
1138+
}
1139+
\item \declareAttributeProvisional{PMIX_NODE_LOAD_AVG15}{"pmix.node.la15"}{float}{
1140+
Load average of last fifteen minutes
1141+
}
1142+
\item \declareAttributeProvisional{PMIX_NODE_MEM_TOTAL}{"pmix.node.mtot"}{float}{
1143+
Total usable RAM (i.e., physical RAM minus reserved bits and kernel binary code). In MBytes
1144+
}
1145+
\item \declareAttributeProvisional{PMIX_NODE_MEM_FREE}{"pmix.node.mfree"}{float}{
1146+
Total free RAM. In MBytes
1147+
}
1148+
\item \declareAttributeProvisional{PMIX_NODE_MEM_BUFFERS}{"pmix.node.mbuf"}{float}{
1149+
Temporary storage for raw disk blocks. In MBytes
1150+
}
1151+
\item \declareAttributeProvisional{PMIX_NODE_MEM_CACHED}{"pmix.node.mcache"}{float}{
1152+
In-memory cache for files read from the disk (the pagecache) as well as tmpfs and shmem. In MBytes.
1153+
Doesn't include \refattr{PMIX_NODE_MEM_SWAP_CACHED}.
1154+
}
1155+
\item \declareAttributeProvisional{PMIX_NODE_MEM_SWAP_CACHED}{"pmix.node.mswpc"}{float}{
1156+
Memory that once was swapped out, is swapped back in but still also is in the swapfile. In MBytes
1157+
}
1158+
\item \declareAttributeProvisional{PMIX_NODE_MEM_SWAP_TOTAL}{"pmix.node.mswpt"}{float}{
1159+
Total amount of swap space available. In MBytes
1160+
}
1161+
\item \declareAttributeProvisional{PMIX_NODE_MEM_SWAP_FREE}{"pmix.node.mswpfree"}{float}{
1162+
Memory which has been evicted from RAM, and is temporarily on the disk. In MBytes
1163+
}
1164+
\item \declareAttributeProvisional{PMIX_NODE_MEM_MAPPED}{"pmix.node.mmap"}{float}{
1165+
files which have been mmapped, such as libraries. Note that some kernel configurations might consider all pages part of a larger allocation (e.g., THP) as “mapped”, as soon as a single page is mapped. In MBytes
1166+
}
1167+
\item \refattr{PMIX_DISK_RESOURCE_USAGE} One for each disk attached to the node.
1168+
\item \refattr{PMIX_NETWORK_RESOURCE_USAGE} One for each network interface on the node.
1169+
\item \declareAttributeProvisional{PMIX_NODE_SAMPLE_TIME}{"pmix.node.samptime"}{struct timeval}{
1170+
Time when sample was taken
1171+
}
1172+
\end{itemize}
1173+
8531174

8541175
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
8551176
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

0 commit comments

Comments
 (0)