# Master Node执行for i in `seq 1 2`; do scp /etc/munge/munge.key root@c$i:/etc/munge/ ; done# 计算节点执行chown munge: /etc/munge/munge.keychmod 400 /etc/munge/munge.key
检查账户是否存在
#master node执行for i in `seq 1 2`;do ssh c$i id munge;done#uid=1108(munge) gid=1108(munge) groups=1108(munge)#uid=1108(munge) gid=1108(munge) groups=1108(munge)
mysql -u root -p#slurm_acct_db数据库的配置grant all on slurm_acct_db.* to 'slurm'@'172.16.183.%' identified by '!QAZ2wsx3edc' with grant option;grant all on slurm_acct_db.* to 'slurm'@'localhost' identified by '!QAZ2wsx3edc' with grant option;SHOW VARIABLES LIKE 'have_innodb';create database slurm_acct_db;quit;
vim /etc/slurm/slurmdbd.confLogFile=/var/log/slurm/slurmdbd.logDbdHost=localhostDbdPort=6819slurmUser=slurmStorageHost=localhostStoragePass=!QAZ2wsx3edcStorageLoc=slurm_acct_db
vim /etc/slurm/slurm.conf#在AccountingStorageType=accounting_storage/mysql后添加AccountingStorageHost=localhostAccountingStoragePort=3306AccountingStoragePass=!QAZ2wsx3edcAccountingStorageUser=slurm
启动SlurmDBD服务
systemctl enable slurmdbdsystemctl start slurmdbdsystemctl status slurmdbd
搭建slurmdbd-docker方法
MariaDB搭建(docker)
初始化宿主机目录
mkdir -p /docker_data/mariadb/{log,data,conf}
修改数据库必备参数
vim /docker_data/mariadb/conf/my.cnf[mysqld] innodb_buffer_pool_size=1024M innodb_log_file_size=64M innodb_lock_wait_timeout=900 character-set-server=utf8mb4[client-server]# Port or socket location where to connect# port = 3306#socket = /run/mysqld/mysqld.socksocket = /var/lib/mysql/mysql.sock
mysql -h m1 -P 3306 -u slurm slurm_acct_db -pSHOW GLOBAL VARIABLES like 'innodb_buffer_pool_size';
设置数据库权限
mysql -h m1 -P 3306 -u root slurm_acct_db -p# 生成slurm用户，以便该用户操作slurm_acct_db数据库，其密码是123create user 'slurm'@'localhost' identified by '123';# 生成账户数据库slurm_acct_dbCREATE DATABASE IF NOT EXISTS slurm_acct_db;# 赋予slurm从本机localhost采用密码SomePassWD登录具备操作slurm_acct_db数据下所有表的全部权限grant all on slurm_acct_db.* TO 'slurm'@'localhost' identified by '123' with grant option;# 赋予slurm从system0采用密码123登录具备操作slurm_acct_db数据下所有表的全部权限grant all on slurm_acct_db.* TO 'slurm'@'system0' identified by '123' with grant option;#grant all on slurm_acct_db.* TO 'slurm'@'%' identified by '123' with grant option;# 生成作业信息数据库slurm_jobcomp_dbcreate database slurm_jobcomp_db;# 赋予slurm从本机localhost采用密码SomePassWD登录具备操作slurm_jobcomp_db数据下所有表的全部权限grant all on slurm_jobcomp_db.* TO 'slurm'@'localhost' identified by '123' with grant option;# 赋予slurm从m1采用密码SomePassWD登录具备操作slurm_jobcomp_db数据下所有表的全部权限grant all on slurm_jobcomp_db.* TO 'slurm'@'m1' identified by '123' with grant option;#grant all on slurm_jobcomp_db.* TO 'slurm'@'%' identified by '123' with grant option;
## Example slurmdbd.conf file.## See the slurmdbd.conf man page for more information.## Archive info#ArchiveJobs=yes#ArchiveDir="/tmp"#ArchiveSteps=yes#ArchiveScript=#JobPurge=12#StepPurge=1## Authentication infoAuthType=auth/mungeAuthInfo=/var/run/munge/munge.socket.2## slurmDBD info#DbdAddr=localhost# 控制节点IPDbdAddr=172.16.183.70#DbdHost=localhostDbdHost=m1#DbdPort=7031SlurmUser=slurm#MessageTimeout=300DebugLevel=verbose#DefaultQOS=normal,standbyDefaultQOS=normalLogFile=/var/log/slurm/slurmdbd.logPidFile=/var/run/slurmdbd.pid#PluginDir=/usr/lib/slurm#PrivateData=accounts,users,usage,jobs#TrackWCKey=yes## Database infoStorageType=accounting_storage/mysql #StorageType=accounting_storage/slurmdbd is invalid in slurmdbd.conf# 存储数据库地址StorageHost=172.16.183.70#StorageHost=localhostStoragePort=3306#StoragePort=1234StoragePass=123StorageUser=slurmStorageLoc=slurm_acct_db
sudo vim /etc/netplan/00-installer-config.yaml# This is the network config written by 'subiquity'network: ethernets: ens33: addresses: - 172.16.183.130/24 nameservers: addresses: - 8.8.8.8 - 114.114.114.114 routes: - to: default via: 172.16.183.2 version: 2
#master节点上tee /etc/slurm/slurm.conf << 'EOF'# slurm.conf file generated by configurator.html.# Put this file on all nodes of your cluster.# See the slurm.conf man page for more information.#ClusterName=ubuntutestclusterSlurmctldHost=um1#SlurmctldHost=##DisableRootJobs=NO#EnforcePartLimits=NO#Epilog=#EpilogSlurmctld=#FirstJobId=1#MaxJobId=67043328#GresTypes=#GroupUpdateForce=0#GroupUpdateTime=600#JobFileAppend=0#JobRequeue=1#JobSubmitPlugins=lua#KillOnBadExit=0#LaunchType=launch/slurm#Licenses=foo*4,bar#MailProg=/bin/mail#MaxJobCount=10000#MaxStepCount=40000#MaxTasksPerNode=512#MpiDefault=#MpiParams=ports=#-##PluginDir=#PlugStackConfig=#PrivateData=jobsProctrackType=proctrack/cgroup#Prolog=#PrologFlags=#PrologSlurmctld=#PropagatePrioProcess=0#PropagateResourceLimits=#PropagateResourceLimitsExcept=#RebootProgram=ReturnToService=1SlurmctldPidFile=/var/run/slurmctld.pidSlurmctldPort=6817SlurmdPidFile=/var/run/slurmd.pidSlurmdPort=6818SlurmdSpoolDir=/var/spool/slurm/slurmdSlurmUser=slurm#SlurmdUser=root#SrunEpilog=#SrunProlog=StateSaveLocation=/var/spool/slurm/slurmctld#SwitchType=#TaskEpilog=TaskPlugin=task/affinity,task/cgroup#TaskProlog=#TopologyPlugin=topology/tree#TmpFS=/tmp#TrackWCKey=no#TreeWidth=#UnkillableStepProgram=#UsePAM=0### TIMERS#BatchStartTimeout=10#CompleteWait=0#EpilogMsgTime=2000#GetEnvTimeout=2#HealthCheckInterval=0#HealthCheckProgram=InactiveLimit=0KillWait=30#MessageTimeout=10#ResvOverRun=0MinJobAge=300#OverTimeLimit=0SlurmctldTimeout=120SlurmdTimeout=300#UnkillableStepTimeout=60#VSizeFactor=0Waittime=0### SCHEDULING#DefMemPerCPU=0#MaxMemPerCPU=0#SchedulerTimeSlice=30SchedulerType=sched/backfillSelectType=select/cons_tres### JOB PRIORITY#PriorityFlags=#PriorityType=priority/multifactor#PriorityDecayHalfLife=#PriorityCalcPeriod=#PriorityFavorSmall=#PriorityMaxAge=#PriorityUsageResetPeriod=#PriorityWeightAge=#PriorityWeightFairshare=#PriorityWeightJobSize=#PriorityWeightPartition=#PriorityWeightQOS=### LOGGING AND ACCOUNTING#AccountingStorageEnforce=0#AccountingStorageHost=#AccountingStoragePass=#AccountingStoragePort=#AccountingStorageType=#AccountingStorageUser=#AccountingStoreFlags=#JobCompHost=#JobCompLoc=#JobCompParams=#JobCompPass=#JobCompPort=JobCompType=jobcomp/none#JobCompUser=#JobContainerType=JobAcctGatherFrequency=30#JobAcctGatherType=SlurmctldDebug=infoSlurmctldLogFile=/var/log/slurm/slurmctld.logSlurmdDebug=infoSlurmdLogFile=/var/log/slurm/slurmd.log#SlurmSchedLogFile=#SlurmSchedLogLevel=#DebugFlags=### POWER SAVE SUPPORT FOR IDLE NODES (optional)#SuspendProgram=#ResumeProgram=#SuspendTimeout=#ResumeTimeout=#ResumeRate=#SuspendExcNodes=#SuspendExcParts=#SuspendRate=#SuspendTime=### COMPUTE NODESNodeName=uc1 NodeAddr=172.16.183.131 CPUs=2 RealMemory=3969468 Sockets=2 CoresPerSocket=1 ThreadsPerCore=1 State=UNKNOWNPartitionName=debug Nodes=ALL Default=YES MaxTime=1 State=UPEOF
配置同步/权限修改
# 复制配置文件（Master node执行）for i in `seq 1`;do ssh uc$i mkdir -p /etc/slurm ;donefor i in `seq 1`;do scp -p /etc/slurm/*.conf root@uc$i:/etc/slurm/;done# 设置文件权限，所有节点执行#chmod 0755 /var/spool#chown -R slurm:slurm /var/spoolmkdir -p /var/spool/slurmchown slurm: /var/spool/slurmmkdir -p /var/log/slurmchown slurm: /var/log/slurmfor i in `seq 1`;do ssh uc$i mkdir -p /var/spool/slurm;donefor i in `seq 1`;do ssh uc$i chown slurm: /var/spool/slurm;donefor i in `seq 1`;do ssh uc$i mkdir -p /var/log/slurm;donefor i in `seq 1`;do ssh uc$i chown slurm: /var/log/slurm;done
启动服务
#um1上systemctl enable slurmctldsystemctl start slurmctldsystemctl status slurmctld#uc1上systemctl enable slurmdsystemctl start slurmdsystemctl status slurmd
作业调度测试
查看集群状态
# 查看集群sinfoscontrol show partitionscontrol show node# 提交作业 srun -N1 hostnamescontrol show jobs# 查看作业squeue -a#计算节点查看slurmd报错sudo slurmd -cDvvvvv#重启控制节点组件systemctl daemon-reload && systemctl restart slurmctld && systemctl restart slurmdbdsystemctl status slurmctld && systemctl status slurmdbd#重启计算节点组件systemctl daemon-reload && systemctl restart slurmd && systemctl status slurmd#恢复计算节点状态#如果Compute Nodes的State=DOWN，则如下执行，将状态变成恢复scontrol update nodename=c1 state=resume
#申请partition compute上申请一个核的资源salloc -p compute -N1 -n1 -q low -t 2:00:00#查看分配到的nodesqueue JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) 70 compute interact root R 3:59 1 c1 71 compute interact root R 0:10 1 c2# 登录c2调试作业ssh c2# 取消作业scancel 71# 查看作业是否还在执行squeue -j 71
常见命令
scontrol show nodes #显示所有计算节点#如果Compute Nodes的State=DOWN，则如下执行，将状态变成恢复scontrol update nodename=uc1 state=resume# Why is a node shown in state DOWN when the node has registered for service?# https://slurm.schedmd.com/faq.html#return_to_service#The configuration parameter ReturnToService in slurm.conf controls how DOWN nodes are handled. Set its value to one in order for DOWN nodes to automatically be returned to service once the slurmd daemon registers with a valid node configuration. A value of zero is the default and results in a node staying DOWN until an administrator explicitly returns it to service using the command "scontrol update NodeName=whatever State=RESUME". See "man slurm.conf" and "man scontrol" for more details.sacctmgr add cluster cluster-testsqueue #检查队列状况scancel #结合作业ID，终止作业#信息查看scontrol show jobs #显示作业数量scontrol show job JOBID #查看作业的详细信息scontrol show node #查看所有节点详细信息scontrol show node node-name #查看指定节点详细信息scontrol show node | grep CPU #查看各节点cpu状态scontrol show node node-name | grep CPU #查看指定节点cpu状态
Slurm用户账户管理
#在Slurm中，账户通常用于跟踪和控制用户对集群资源的使用。分区则定义了一组节点和作业在这些节点上的运行参数。#查看所有的账户sacctmgr list assoc#查看账户和分区的关联sacctmgr show associations
