Slurm PS

TTIC SLURM Cluster: Process Statistics
Mon May 23 06:54:11 CDT 2022 [Docs] | [Job Process Info] | [Job Queue] | [Cluster Usage & Load]

Node	User	Command	CPU %	MEM %	MEM (kb)
cpu1	NCPU: 64 load average: 0.00, 0.01, 0.05

cpu2	NCPU: 8 load average: 35.50, 35.72, 35.65
	bshi	python	124	6.6	3298264
	bshi	python	123	4.2	2114016
	bshi	python	122	3.4	1702248
	bshi	python	114	1.5	754332
	bshi	python	104	3.1	1528512
	bshi	python	101	3.7	1853924
	bshi	ffmpeg	71.0	0.3	178572
	bshi	ffmpeg	52.4	0.6	316876
	bshi	ffmpeg	51.4	0.6	317892
	bshi	ffmpeg	41.8	0.6	318092
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	680
	bshi	python	0.0	0.0	8
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4724
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688

cpu3	NCPU: 8 load average: 47.11, 46.74, 47.21
	bshi	python	100	4.3	2150944
	bshi	python	95.0	6.4	3163176
	bshi	python	92.8	5.4	2665236
	bshi	python	90.0	2.5	1248648
	bshi	python	87.9	3.9	1959832
	bshi	python	87.6	3.1	1545428
	bshi	python	85.6	2.6	1327736
	bshi	python	83.8	4.5	2249640
	bshi	ffmpeg	24.3	0.6	311484
	bshi	ffmpeg	22.4	0.5	293712
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4724
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684

cpu4	NCPU: 8 load average: 46.71, 45.33, 45.33
	bshi	python	97.4	1.1	550748
	bshi	python	93.9	3.2	1597332
	bshi	python	90.2	4.3	2163772
	bshi	python	89.9	5.8	2870740
	bshi	python	89.2	4.7	2322884
	bshi	python	86.9	3.6	1818764
	bshi	python	84.4	6.6	3280168
	bshi	python	78.7	2.0	987548
	bshi	ffmpeg	24.9	0.6	315848
	bshi	ffmpeg	22.3	0.6	316840
	bshi	ffmpeg	22.3	0.2	99116
	bshi	slurm_script	0.0	0.0	1432
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	688
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4724
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4724
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680

cpu5	NCPU: 8 load average: 45.89, 47.49, 47.47
	bshi	python	97.2	4.6	2271332
	bshi	python	97.0	7.4	3648900
	bshi	python	95.2	2.2	1105576
	bshi	python	89.3	3.8	1913976
	bshi	python	87.9	3.1	1538484
	bshi	python	85.9	1.6	801972
	bshi	python	82.8	4.0	2005456
	bshi	python	70.7	3.7	1823024
	bshi	ffmpeg	38.5	0.6	309124
	bshi	ffmpeg	31.6	0.6	332164
	bshi	ffmpeg	29.9	0.6	321452
	bshi	ffmpeg	28.1	0.6	321640
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4724
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684

cpu6	NCPU: 8 load average: 0.06, 0.06, 0.06

cpu7	NCPU: 8 load average: 0.01, 0.02, 0.05

cpu8	NCPU: 8 load average: 0.00, 0.01, 0.05

cpu9	NCPU: 8 load average: 0.00, 0.01, 0.05

cpu10	NCPU: 12 load average: 68.49, 68.78, 69.46
	bshi	python	134	3.3	2192872
	bshi	python	127	1.1	772776
	bshi	python	122	3.2	2134480
	bshi	python	119	1.7	1164772
	bshi	python	119	2.8	1877736
	bshi	python	109	4.3	2880568
	bshi	python	106	2.5	1706712
	bshi	python	106	2.3	1515200
	bshi	python	104	4.6	3030648
	bshi	python	94.4	3.4	2297512
	bshi	ffmpeg	51.0	0.3	253924
	bshi	ffmpeg	50.0	0.5	379796
	bshi	ffmpeg	48.6	0.5	381940
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	684

cpu11	NCPU: 12 load average: 0.00, 0.01, 0.05

cpu12	NCPU: 12 load average: 0.04, 0.03, 0.05

cpu13	NCPU: 12 load average: 0.01, 0.04, 0.05

cpu14	NCPU: 12 load average: 0.00, 0.02, 0.05

cpu15	NCPU: 12 load average: 0.00, 0.01, 0.05

cpu16	NCPU: 12 load average: 0.01, 0.02, 0.05

cpu17	NCPU: 12 load average: 0.05, 0.04, 0.05

cpu18	NCPU: 20 load average: 171.57, 173.17, 171.17
	bshi	python	102	1.2	1613908
	bshi	python	102	1.0	1438316
	bshi	python	100	0.9	1241744
	bshi	python	100	1.0	1355604
	bshi	python	99.2	1.8	2462212
	bshi	python	97.9	1.8	2437188
	bshi	python	95.4	2.2	2910872
	bshi	python	93.9	1.2	1591276
	bshi	python	93.9	1.9	2607916
	bshi	python	93.5	1.7	2255380
	bshi	python	91.7	0.7	955756
	bshi	python	90.0	1.0	1342604
	bshi	python	89.5	1.4	1925460
	bshi	python	87.7	1.0	1404796
	bshi	python	87.3	2.2	2994044
	bshi	python	86.4	1.5	2049896
	bshi	python	85.7	1.3	1723900
	bshi	python	84.1	1.5	2064356
	bshi	python	83.7	0.8	1090204
	bshi	python	83.7	1.4	1875372
	bshi	ffmpeg	39.8	0.3	415292
	bshi	ffmpeg	38.5	0.3	424780
	bshi	ffmpeg	33.5	0.3	426564
	bshi	ffmpeg	30.5	0.3	409040
	bshi	ffmpeg	26.6	0.3	416348
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4720
	bshi	srun	0.0	0.0	688

cpu19	NCPU: 8 load average: 0.02, 0.03, 0.05

cpu20	NCPU: 8 load average: 0.00, 0.01, 0.05

cpu21	NCPU: 8 load average: 0.01, 0.02, 0.05

cpu22	NCPU: 8 load average: 0.05, 0.07, 0.06

cpu23	NCPU: 64 load average: 0.00, 0.02, 0.05

gpu1	NCPU: 20 load average: 9.12, 8.71, 8.33
	hzhao	python	102	1.7	4688856
	hzhao	python	102	1.7	4680700
	hzhao	python	102	1.7	4680912
	hzhao	python	102	1.7	4683536
	hzhao	python	102	1.7	4695048
	hzhao	python	99.3	1.7	4698120
	hzhao	python	98.4	1.6	4310240
	hzhao	python	98.2	1.7	4698372
	hzhao	python	97.5	1.6	4262508
	hzhao	python	97.3	1.6	4286644
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1560
	hzhao	slurm_script	0.0	0.0	1552
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556

gpu2	NCPU: 16 load average: 0.05, 0.08, 0.08

gpu3	NCPU: 28 load average: 0.00, 0.01, 0.05

gpu4	NCPU: 20 load average: 4.37, 3.34, 2.97
	hzhao	python	105	2.3	4681080
	hzhao	python	105	2.3	4680616
	hzhao	python	105	2.3	4680712
	hzhao	python	104	2.3	4680776
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1552
	hzhao	slurm_script	0.0	0.0	1556

gpu5	NCPU: 48 load average: 0.00, 0.01, 0.05

gpu6	NCPU: 20 load average: 0.01, 0.60, 7.82
	jiahao	bash	0.0	0.0	1304
	jiahao	bash	0.0	0.0	1444

gpu10	NCPU: 6 load average: 0.00, 0.02, 0.05

gpu11	NCPU: 20 load average: 0.00, 0.01, 0.05

gpu12	NCPU: 16 load average: 0.00, 0.01, 0.05

gpu13	NCPU: 16 load average: 53.34, 59.59, 61.63
	bshi	python	194	0.7	1880576
	bshi	python	193	0.6	1786700
	bshi	python	192	0.4	1184372
	bshi	python	183	0.6	1790672
	bshi	python	174	0.4	1195520
	bshi	python	174	0.7	2085372
	bshi	python	170	0.3	803516
	bshi	python	161	0.2	636252
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	684

gpu14	NCPU: 16 load average: 58.39, 62.44, 65.43
	bshi	python	172	0.6	1715176
	bshi	python	169	0.9	2611548
	bshi	python	155	0.6	1671744
	bshi	python	155	1.3	3570936
	bshi	python	148	0.7	2110388
	bshi	python	145	0.8	2170088
	bshi	python	145	0.9	2506388
	bshi	python	143	0.6	1609004
	bshi	ffmpeg	70.6	0.1	428468
	hzhao	python	53.4	1.7	4679872
	hzhao	python	53.4	1.7	4679776
	hzhao	python	53.1	1.7	4679920
	hzhao	python	53.1	1.7	4683124
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	676
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1552
	hzhao	slurm_script	0.0	0.0	1556
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	680
	hzhao	slurm_script	0.0	0.0	1552
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684

gpu15	NCPU: 16 load average: 64.84, 62.54, 62.52
	bshi	python	194	0.9	2523344
	bshi	python	190	0.4	1249560
	bshi	python	186	0.4	1235036
	bshi	python	184	0.8	2222004
	bshi	python	183	0.9	2566176
	bshi	python	173	0.7	1966676
	bshi	python	172	0.7	2042244
	bshi	python	168	0.8	2327056
	bshi	ffmpeg	112	0.1	437140
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688

gpu16	NCPU: 16 load average: 67.41, 71.11, 69.95
	bshi	python	178	0.9	2402016
	bshi	python	175	0.7	2056796
	bshi	python	166	1.0	2720808
	bshi	python	165	0.8	2178496
	bshi	python	160	0.9	2438304
	bshi	python	156	0.8	2181380
	bshi	python	154	1.2	3166388
	bshi	python	146	0.8	2134544
	bshi	ffmpeg	75.0	0.1	430108
	hzhao	python	50.4	1.7	4677368
	hzhao	python	50.2	1.7	4677084
	hzhao	python	49.7	1.6	4285076
	hzhao	python	48.2	1.6	4290548
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4700
	bshi	srun	0.0	0.0	676
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4704
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1552
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	hzhao	slurm_script	0.0	0.0	1560
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	680
	hzhao	slurm_script	0.0	0.0	1556

gpu17	NCPU: 16 load average: 68.26, 64.05, 61.49
	bshi	python	191	0.7	1894936
	bshi	python	190	1.0	2658036
	bshi	python	185	0.6	1689256
	bshi	python	183	0.4	1263368
	bshi	python	181	1.1	2949844
	bshi	python	178	0.7	1898488
	bshi	python	176	1.1	2963616
	bshi	python	172	1.4	3912176
	bshi	ffmpeg	95.6	0.1	429100
	bshi	ffmpeg	75.1	0.1	426068
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4696
	bshi	srun	0.0	0.0	680
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1436
	bshi	srun	0.0	0.0	4708
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4712
	bshi	srun	0.0	0.0	684
	bshi	slurm_script	0.0	0.0	1432
	bshi	srun	0.0	0.0	4716
	bshi	srun	0.0	0.0	688

gpu19	NCPU: 8 load average: 0.04, 0.04, 0.05
	xdu	bash	0.0	0.0	2428
	xdu	bash	0.0	0.0	2308

gpu20	NCPU: 4 load average: 0.00, 0.01, 0.05

gpu21	NCPU: 4 load average: 0.02, 0.03, 0.05

gpu22	NCPU: 4 load average: 0.00, 0.01, 0.05

gpu23	NCPU: 4 load average: 11.92, 12.07, 12.12
	jiahao	python3.9	49.9	0.9	632784
	jiahao	python3.9	49.7	0.8	550496
	jiahao	python3.9	49.5	0.8	542272
	jiahao	python3.9	49.2	1.0	683808
	jiahao	python3.9	48.7	0.9	609588
	jiahao	python3.9	48.7	0.8	547924
	jiahao	python3.9	48.7	0.8	553660
	jiahao	python3.9	48.5	0.8	555828
	jiahao	python3.9	1.4	0.5	349840
	jiahao	bash	0.0	0.0	136
	jiahao	img2dataset	0.0	0.8	575860
	jiahao	python3.9	0.0	0.0	9296
	jiahao	jupyter-noteboo	0.0	0.0	152
	jiahao	bash	0.0	0.0	1560
	jiahao	bash	0.0	0.0	1824

gpu24	NCPU: 8 load average: 0.00, 0.01, 0.05

gpu25	NCPU: 4 load average: 0.01, 0.02, 0.05

gpu27	NCPU: 16 load average: 0.00, 0.01, 0.05
	simin8	bash	0.0	0.0	2264

gpu28	NCPU: 20 load average: 29.66, 30.02, 30.34
	jiahao	python	76.5	6.1	24409680
	jiahao	python	76.3	6.1	24427180
	jiahao	python	76.3	6.1	24404572
	jiahao	python	76.3	6.1	24409584
	jiahao	python	74.3	6.1	24414216
	jiahao	python	74.2	6.1	24410380
	jiahao	python	74.1	6.1	24420708
	jiahao	python	74.1	6.1	24416916
	jiahao	python	57.1	0.9	3938784
	jiahao	python	56.8	0.9	3928392
	jiahao	python	56.5	0.9	3934296
	jiahao	python	56.4	0.9	3941692
	jiahao	python	55.9	1.0	4079336
	jiahao	python	55.8	0.9	3926588
	jiahao	python	55.8	0.9	3936032
	jiahao	python	55.7	0.9	3925192
	jiahao	python	55.6	0.9	3943672
	jiahao	python	55.6	1.0	3948900
	jiahao	python	55.6	1.0	4097464
	jiahao	python	55.5	0.9	3920916
	jiahao	python	55.4	1.0	4084544
	jiahao	python	55.3	0.9	3943668
	jiahao	python	55.3	0.9	3926636
	jiahao	python	55.3	0.9	3942672
	jiahao	python	55.2	1.0	3954216
	jiahao	python	55.2	1.0	4082328
	jiahao	python	55.1	1.0	3953500
	jiahao	python	54.9	0.9	3928120
	jiahao	python	54.8	1.0	4080076
	jiahao	python	54.7	1.0	3947084
	jiahao	python	54.7	0.9	3934072
	jiahao	python	54.6	1.0	4080052
	jiahao	python	1.2	3.3	13262804
	jiahao	python	0.8	3.4	13647300
	jiahao	python	0.8	3.3	13371556
	jiahao	python	0.8	3.3	13384740
	jiahao	python	0.7	3.4	13647648
	jiahao	python	0.7	3.3	13340352
	jiahao	python	0.7	3.3	13324888
	jiahao	python	0.7	3.4	13659848
	jiahao	python	0.7	3.3	13328604
	jiahao	python	0.7	3.4	13651276
	jiahao	python	0.7	3.3	13321460
	jiahao	python	0.7	3.4	13650400
	jiahao	python	0.7	3.4	13636728
	jiahao	python	0.7	3.3	13353168
	jiahao	python	0.7	3.4	13601408
	jiahao	python	0.7	3.4	13626416
	jiahao	python	0.7	3.4	13626000
	jiahao	python	0.7	3.4	13643880
	jiahao	python	0.7	3.3	13332532
	jiahao	python	0.7	3.3	13372316
	jiahao	python	0.7	3.3	13371740
	jiahao	python	0.7	3.4	13497924
	jiahao	python	0.6	3.4	13617412
	jiahao	python	0.5	3.3	13330176
	jiahao	python	0.3	3.3	13281488
	jiahao	python	0.3	3.3	13282200
	jiahao	python	0.3	3.3	13275908
	jiahao	python	0.3	3.3	13272740
	jiahao	python	0.3	3.3	13273864
	jiahao	python	0.3	3.3	13282948
	jiahao	python	0.3	3.3	13274992
	jiahao	python	0.3	3.3	13279792
	jiahao	bash	0.0	0.0	1508
	jiahao	bash	0.0	0.0	1100
	jiahao	bash	0.0	0.0	1224
	jiahao	python	0.0	0.0	215960

gpu29	NCPU: 20 load average: 0.08, 0.07, 0.06

gpu30	NCPU: 20 load average: 0.05, 0.05, 0.05

gpu31	NCPU: 24 load average: 0.09, 0.07, 0.05

gpu32	NCPU: 24 load average: 0.05, 0.03, 0.05

gpu33	NCPU: 20 load average: 0.00, 0.01, 0.05

gpu34	NCPU: 8 load average: 0.00, 0.01, 0.05

gpu35	NCPU: 16 load average: 0.00, 0.01, 0.05

gpu36	NCPU: 20 load average: 0.04, 0.03, 0.05

gpu37	NCPU: 20 load average: 0.00, 0.01, 0.05

gpu38	NCPU: 20 load average: 7.30, 7.10, 6.94
	hzhao	python	107	2.3	4677732
	hzhao	python	107	2.3	4677760
	hzhao	python	107	2.3	4677700
	hzhao	python	107	2.3	4677840
	hzhao	python	107	2.3	4677556
	hzhao	python	107	2.3	4677720
	hzhao	python	106	2.3	4677768
	hzhao	python	98.2	2.2	4350480
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1560
	hzhao	slurm_script	0.0	0.0	1560
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1560
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1556
	hzhao	slurm_script	0.0	0.0	1560

gpu39	NCPU: 8 load average: 0.00, 0.01, 0.05

gpu40	NCPU: 8 load average: 0.00, 0.01, 0.05

TTIC SLURM Cluster: GPU Statistics
Mon May 23 06:54:11 CDT 2022

Node	Fan Speed	Temp	Perf State	Power	Max	Memory	Max	GPU Usage	User
gpu1	85%	52C	P2	86W	140W	13245MiB	16117MiB	30%	hzhao
gpu1	85%	50C	P2	69W	140W	13245MiB	16117MiB	44%	hzhao
gpu1	85%	32C	P8	13W	140W	3283MiB	16117MiB	0%	hzhao
gpu1	85%	31C	P8	17W	140W	2965MiB	16117MiB	0%	hzhao
gpu1	85%	47C	P2	51W	140W	13245MiB	16117MiB	35%	hzhao
gpu1	85%	48C	P2	64W	140W	13245MiB	16117MiB	33%	hzhao
gpu1	85%	33C	P8	13W	140W	3283MiB	16117MiB	0%	hzhao
gpu1	85%	49C	P2	55W	140W	13245MiB	16117MiB	19%	hzhao
gpu1	85%	32C	P8	15W	140W	2965MiB	16117MiB	0%	hzhao
gpu1	85%	34C	P8	14W	140W	2974MiB	16117MiB	0%	hzhao

gpu2	85%	27C	P8	19W	250W	11MiB	11178MiB	0%
gpu2	85%	23C	P8	12W	250W	11MiB	11178MiB	0%
gpu2	85%	27C	P8	12W	250W	11MiB	11178MiB	0%
gpu2	85%	27C	P8	12W	250W	11MiB	11178MiB	0%

gpu3	85%	30C	P8	3W	250W	10MiB	11019MiB	0%
gpu3	85%	28C	P8	13W	250W	10MiB	11019MiB	0%
gpu3	85%	29C	P8	21W	250W	10MiB	11019MiB	0%
gpu3	85%	28C	P8	8W	250W	10MiB	11019MiB	0%

gpu4	85%	68C	P2	94W	140W	13244MiB	16117MiB	26%	hzhao
gpu4	85%	59C	P2	59W	140W	13244MiB	16117MiB	28%	hzhao
gpu4	85%	53C	P2	62W	140W	13244MiB	16117MiB	30%	hzhao
gpu4	85%	50C	P2	89W	140W	13221MiB	16117MiB	23%	hzhao

gpu5	86%	34C	P8	30W	300W	19MiB	48685MiB	0%
gpu5	85%	32C	P8	28W	300W	19MiB	48685MiB	0%
gpu5	85%	33C	P8	29W	300W	19MiB	48685MiB	0%
gpu5	85%	29C	P8	31W	300W	19MiB	48685MiB	0%
gpu5	85%	30C	P8	23W	300W	19MiB	48685MiB	0%
gpu5	85%	28C	P8	31W	300W	19MiB	48685MiB	0%
gpu5	85%	30C	P8	22W	300W	19MiB	48685MiB	0%
gpu5	85%	28C	P8	23W	300W	28MiB	48685MiB	0%

gpu6	85%	35C	P8	20W	300W	19MiB	48685MiB	0%
gpu6	85%	32C	P8	18W	300W	19MiB	48685MiB	0%
gpu6	85%	35C	P8	19W	300W	19MiB	48685MiB	0%
gpu6	85%	34C	P8	21W	300W	19MiB	48685MiB	0%
gpu6	85%	32C	P8	21W	300W	19MiB	48685MiB	0%
gpu6	85%	35C	P8	21W	300W	19MiB	48685MiB	0%
gpu6	85%	34C	P8	20W	300W	19MiB	48685MiB	0%
gpu6	85%	36C	P8	20W	300W	28MiB	48685MiB	0%

gpu10	85%	35C	P8	20W	250W	11MiB	12195MiB	0%
gpu10	85%	32C	P8	13W	250W	11MiB	12196MiB	0%

gpu11	85%	31C	P8	9W	250W	10MiB	11019MiB	0%
gpu11	85%	31C	P8	1W	250W	10MiB	11019MiB	0%
gpu11	85%	30C	P8	6W	250W	10MiB	11019MiB	0%
gpu11	85%	29C	P8	4W	250W	10MiB	11019MiB	0%

gpu12	85%	28C	P8	9W	140W	19MiB	16117MiB	0%
gpu12	85%	28C	P8	6W	140W	19MiB	16117MiB	0%
gpu12	85%	28C	P8	8W	140W	19MiB	16117MiB	0%
gpu12	85%	28C	P8	11W	140W	27MiB	16117MiB	0%

gpu13	85%	36C	P8	15W	250W	10MiB	11019MiB	0%
gpu13	85%	32C	P8	25W	250W	10MiB	11019MiB	0%
gpu13	85%	35C	P8	38W	250W	10MiB	11019MiB	0%
gpu13	85%	34C	P8	34W	250W	10MiB	11019MiB	0%

gpu14	85%	44C	P2	54W	140W	13540MiB	16117MiB	32%	hzhao
gpu14	85%	48C	P2	49W	140W	13540MiB	16117MiB	14%	hzhao
gpu14	85%	45C	P2	52W	140W	13540MiB	16117MiB	12%	hzhao
gpu14	85%	45C	P2	49W	140W	13537MiB	16117MiB	9%	hzhao

gpu15	85%	34C	P8	26W	250W	10MiB	11019MiB	0%
gpu15	85%	33C	P8	18W	250W	10MiB	11019MiB	0%
gpu15	85%	32C	P8	17W	250W	10MiB	11019MiB	0%
gpu15	85%	32C	P8	24W	250W	10MiB	11019MiB	0%

gpu16	85%	50C	P2	109W	300W	22446MiB	48685MiB	11%	hzhao
gpu16	85%	56C	P2	105W	300W	22446MiB	48685MiB	21%	hzhao
gpu16	85%	33C	P8	28W	300W	3092MiB	48685MiB	0%	hzhao
gpu16	85%	36C	P8	36W	300W	3101MiB	48685MiB	0%	hzhao

gpu17	85%	36C	P8	28W	250W	10MiB	12066MiB	0%
gpu17	85%	37C	P8	25W	250W	10MiB	12066MiB	0%
gpu17	85%	36C	P8	27W	250W	10MiB	12066MiB	0%
gpu17	85%	37C	P8	27W	250W	10MiB	12066MiB	0%

gpu19	85%	30C	P8	24W	260W	18MiB	24219MiB	0%
gpu19	85%	27C	P8	16W	260W	27MiB	24220MiB	0%

gpu20	85%	26C	P8	20W	250W	11MiB	11178MiB	0%
gpu20	85%	27C	P8	13W	250W	11MiB	11178MiB	0%

gpu21	85%	32C	P8	12W	260W	18MiB	48600MiB	0%
gpu21	85%	30C	P8	10W	260W	27MiB	48601MiB	0%

gpu22	23%	32C	P8	16W	250W	1MiB	11178MiB	0%
gpu22	27%	30C	P8	8W	250W	1MiB	11178MiB	0%

gpu23	85%	29C	P8	16W	260W	18MiB	24219MiB	0%
gpu23	85%	27C	P8	15W	260W	27MiB	24220MiB	0%

gpu24	85%	25C	P8	20W	250W	11MiB	11178MiB	0%
gpu24	85%	23C	P8	11W	250W	11MiB	11178MiB	0%

gpu25	23%	32C	P8	16W	250W	1MiB	11178MiB	0%
gpu25	20%	28C	P8	7W	250W	1MiB	11178MiB	0%

gpu27	85%	33C	P8	20W	250W	10MiB	11019MiB	0%
gpu27	85%	32C	P8	21W	250W	10MiB	11019MiB	0%
gpu27	85%	31C	P8	20W	250W	10MiB	11019MiB	0%
gpu27	85%	31C	P8	20W	250W	10MiB	11019MiB	0%

gpu28	85%	75C	P2	203W	300W	45890MiB	48685MiB	99%	jiahao
gpu28	85%	61C	P2	189W	300W	45878MiB	48685MiB	99%	jiahao
gpu28	85%	83C	P2	203W	300W	45890MiB	48685MiB	94%	jiahao
gpu28	85%	81C	P2	206W	300W	45912MiB	48685MiB	99%	jiahao
gpu28	85%	66C	P2	184W	300W	45895MiB	48685MiB	99%	jiahao
gpu28	85%	81C	P2	179W	300W	45918MiB	48685MiB	92%	jiahao
gpu28	85%	67C	P2	183W	300W	45902MiB	48685MiB	97%	jiahao
gpu28	85%	84C	P2	203W	300W	45876MiB	48685MiB	92%	jiahao

gpu29	85%	32C	P8	16W	280W	10MiB	24220MiB	0%
gpu29	85%	31C	P8	8W	280W	10MiB	24220MiB	0%
gpu29	85%	30C	P8	16W	280W	10MiB	24220MiB	0%
gpu29	85%	31C	P8	7W	280W	10MiB	24220MiB	0%

gpu30	85%	33C	P8	35W	250W	10MiB	11019MiB	0%
gpu30	85%	28C	P8	30W	250W	10MiB	11019MiB	0%
gpu30	85%	31C	P8	41W	250W	10MiB	11019MiB	0%
gpu30	85%	31C	P8	27W	250W	10MiB	11019MiB	0%
gpu30	85%	30C	P8	22W	250W	10MiB	11019MiB	0%
gpu30	85%	30C	P8	32W	250W	10MiB	11019MiB	0%
gpu30	85%	32C	P8	28W	250W	10MiB	11019MiB	0%
gpu30	85%	30C	P8	33W	250W	10MiB	11019MiB	0%

gpu31	85%	34C	P8	32W	250W	10MiB	11019MiB	0%
gpu31	85%	34C	P8	46W	250W	10MiB	11019MiB	0%
gpu31	85%	32C	P8	34W	250W	10MiB	11019MiB	0%
gpu31	85%	31C	P8	42W	250W	10MiB	11019MiB	0%

gpu32	85%	31C	P8	33W	250W	10MiB	11019MiB	0%
gpu32	85%	32C	P8	27W	250W	10MiB	11019MiB	0%
gpu32	85%	32C	P8	21W	250W	10MiB	11019MiB	0%
gpu32	85%	31C	P8	34W	250W	10MiB	11019MiB	0%

gpu33	85%	33C	P8	28W	250W	10MiB	11019MiB	0%
gpu33	85%	29C	P8	29W	250W	10MiB	11019MiB	0%
gpu33	85%	34C	P8	27W	250W	10MiB	11019MiB	0%
gpu33	85%	33C	P8	14W	250W	10MiB	11019MiB	0%
gpu33	85%	29C	P8	25W	250W	10MiB	11019MiB	0%
gpu33	85%	30C	P8	19W	250W	10MiB	11019MiB	0%
gpu33	85%	34C	P8	27W	250W	10MiB	11019MiB	0%
gpu33	85%	31C	P8	38W	250W	10MiB	11019MiB	0%

gpu34	85%	30C	P8	20W	250W	10MiB	11019MiB	0%
gpu34	85%	27C	P8	14W	250W	10MiB	11019MiB	0%

gpu35	85%	37C	P8	23W	250W	10MiB	11019MiB	0%
gpu35	85%	36C	P8	36W	250W	10MiB	11019MiB	0%
gpu35	85%	33C	P8	23W	250W	10MiB	11019MiB	0%
gpu35	85%	34C	P8	26W	250W	10MiB	11019MiB	0%

gpu36	85%	30C	P8	21W	250W	10MiB	11019MiB	0%
gpu36	85%	27C	P8	16W	250W	10MiB	11019MiB	0%
gpu36	85%	31C	P8	22W	250W	10MiB	11019MiB	0%
gpu36	85%	29C	P8	7W	250W	10MiB	11019MiB	0%
gpu36	85%	27C	P8	7W	250W	10MiB	11019MiB	0%
gpu36	85%	29C	P8	19W	250W	10MiB	11019MiB	0%
gpu36	85%	29C	P8	21W	250W	10MiB	11019MiB	0%
gpu36	85%	29C	P8	16W	250W	10MiB	11019MiB	0%

gpu37	85%	33C	P8	27W	250W	10MiB	11019MiB	0%
gpu37	85%	31C	P8	25W	250W	10MiB	11019MiB	0%
gpu37	85%	39C	P8	25W	260W	10MiB	11019MiB	0%
gpu37	85%	32C	P8	33W	250W	10MiB	11019MiB	0%

gpu38	85%	54C	P2	141W	300W	38564MiB	48685MiB	22%	hzhao
gpu38	85%	29C	P8	25W	300W	3092MiB	48685MiB	0%	hzhao
gpu38	85%	60C	P2	129W	300W	38564MiB	48685MiB	39%	hzhao
gpu38	85%	57C	P2	117W	300W	38564MiB	48685MiB	27%	hzhao
gpu38	85%	50C	P2	146W	300W	38564MiB	48685MiB	33%	hzhao
gpu38	85%	61C	P2	148W	300W	38564MiB	48685MiB	37%	hzhao
gpu38	85%	49C	P2	137W	300W	38564MiB	48685MiB	34%	hzhao
gpu38	85%	55C	P2	124W	300W	38573MiB	48685MiB	12%	hzhao

gpu39	85%	28C	P8	30W	250W	10MiB	11019MiB	0%
gpu39	85%	26C	P8	25W	250W	10MiB	11019MiB	0%

gpu40	85%	30C	P8	10W	140W	19MiB	16117MiB	0%
gpu40	85%	30C	P8	8W	140W	19MiB	16117MiB	0%
gpu40	85%	31C	P8	10W	140W	19MiB	16117MiB	0%
gpu40	85%	32C	P8	8W	140W	27MiB	16116MiB	0%