Ticket #47: PBSQuery_torque5_ranges.patch
File PBSQuery_torque5_ranges.patch, 17.4 KB (added by stijn.deweirdt@…, 9 years ago) |
---|
-
PBSQuery.py
old new 1 1 # 2 # Authors: Roy Dragseth (roy.dragseth@cc.uit.no) 2 # Authors: Roy Dragseth (roy.dragseth@cc.uit.no) 3 3 # Bas van der Vlies (basv@sara.nl) 4 4 # 5 5 # SVN INFO: … … 13 13 with id as key and batch object as value 14 14 15 15 There are four batch objects: 16 - server 16 - server 17 17 - queue 18 18 - job 19 19 - node 20 20 21 21 Each object can be handled as an dictionary and has several member 22 functions. The second parameter is an python list and can be used if you 22 functions. The second parameter is an python list and can be used if you 23 23 are only interested in certain resources, see example 24 24 25 25 There are the following functions for PBSQuery: 26 job - 26 job - 27 27 getjob(job_id, attributes=<default is all>) 28 28 getjobs(attributes=<default is all>) 29 29 30 30 node - 31 31 getnode(node_id, attributes=<default is all>) 32 32 getnodes(attributes=<default is all>) … … 52 52 for name,node in nodes.items(): 53 53 print node, node['state'] 54 54 55 The parameter 'attributes' is an python list of resources that 55 The parameter 'attributes' is an python list of resources that 56 56 you are interested in, eg: only show state of nodes 57 57 l = list() 58 58 l.append('state') … … 65 65 import re 66 66 import types 67 67 68 69 REG_SUBRANGE = re.compile(r'^\d+(-\d+)?$') 70 """ 71 >>> b='rectime=1424696750,macaddr=40:a8:f0:2f:17:f4,cpuclock=Fixed,varattr=,jobs=419[1].master(cput=236745,energy_used=0,mem=6562224kb,vmem=7391872kb,walltime=22647,session_id=15941) 446[1].master(cput=7385,energy_used=0,mem=202936kb,vmem=368184kb,walltime=7391,session_id=30940) 446[2].master(cput=7385,energy_used=0,mem=204016kb,vmem=368184kb,walltime=7387,session_id=31168) 446[3].master(cput=7384,energy_used=0,mem=202380kb,vmem=368184kb,walltime=7387,session_id=31344) 446[4].master(cput=7384,energy_used=0,mem=204108kb,vmem=368184kb,walltime=7386,session_id=31536) 446[5].master(cput=7384,energy_used=0,mem=203940kb,vmem=368184kb,walltime=7386,session_id=31718) 446[6].master(cput=7383,energy_used=0,mem=188720kb,vmem=352608kb,walltime=7386,session_id=31904),state=free,size=456341748kb:459945088kb,netload=587288451179,gres=,loadave=18.07,ncpus=24,physmem=65850220kb,availmem=77961112kb,totmem=86821736kb,idletime=13933,nusers=1,nsessions=7,sessions=31904 31718 31536 31344 31168 30940 15941,uname=Linux node24 3.10.0 #1 SMP Wed Feb 4 08:16:54 CET 2015 x86_64,opsys=linux' 72 >>> c=','.join([x[1] for x in re.compile(r'((:?[^,(]+(?:\(.*?\))?))(?:,|$)').findall(b)]) 73 >>> c == b 74 True 75 76 """ 77 REG_SPLIT_COMMA_BRACE = re.compile(r'((:?[^,(]+(?:\(.*?\))?))(?:,|$)') 78 """ 79 >>> d='jobs=419[1].master(cput=236745,energy_used=0,mem=6562224kb,vmem=7391872kb,walltime=22647,session_id=15941) 446[1].master(cput=7385,energy_used=0,mem=202936kb,vmem=368184kb,walltime=7391,session_id=30940) 446[2].master(cput=7385,energy_used=0,mem=204016kb,vmem=368184kb,walltime=7387,session_id=31168) 446[3].master(cput=7384,energy_used=0,mem=202380kb,vmem=368184kb,walltime=7387,session_id=31344) 446[4].master(cput=7384,energy_used=0,mem=204108kb,vmem=368184kb,walltime=7386,session_id=31536) 446[5].master(cput=7384,energy_used=0,mem=203940kb,vmem=368184kb,walltime=7386,session_id=31718) 446[6].master(cput=7383,energy_used=0,mem=188720kb,vmem=352608kb,walltime=7386,session_id=31904)' 80 >>> e='='.join([x[1] for x in re.compile(r'((:?[^=(]+(?:\(.*?\))?))(?:=|$)').findall(d)]) 81 >>> d == e 82 True 83 """ 84 REG_SPLIT_EQUAL_BRACE = re.compile(r'((:?[^=(]+(?:\(.*?\))?))(?:=|$)') 85 86 JOB_RE = re.compile('(?:^|,)(?:((?:[\d,-]+)?\d+)/)?(.+)') 87 88 89 def convert_range(rangetxt): 90 """ 91 Convert range string into list of id strings: eg.g '3,5,7-9' -> ['3','5','7','8','9'] 92 """ 93 ids=[] 94 for subrange in [r.split('-') for r in rangetxt.split(',')]: 95 start = int(subrange[0]) 96 if(len(subrange) == 2 ): 97 end = int(subrange[1]) + 1 98 else: 99 end = start + 1 100 ids.extend([str(i) for i in range(start,end)]) 101 return ids 102 103 68 104 class PBSError(Exception): 69 105 def __init__(self, msg=''): 70 106 self.msg = msg 71 107 Exception.__init__(self, msg) 72 108 73 109 def __repr__(self): 74 110 return self.msg 75 111 … … 92 128 ## this is needed for getjob a jobid is made off: 93 129 # sequence_number.server (is not self.server) 94 130 # 95 self.job_server_id = list(self.get_serverinfo())[0] 131 self.job_server_id = list(self.get_serverinfo())[0] 96 132 self._disconnect() 97 133 98 134 … … 111 147 def _list_2_attrib(self, list): 112 148 """Convert a python list to an attrib list suitable for pbs""" 113 149 self.attribs = pbs.new_attrl( len(list) ) 114 i = 0 150 i = 0 115 151 for attrib in list: 116 152 # So we can user Resource 117 153 attrib = attrib.split('.') … … 126 162 127 163 def _list_2_dict(self, l, class_func): 128 164 """ 129 Convert a pbsstat function list to a class dictionary, The 165 Convert a pbsstat function list to a class dictionary, The 130 166 data structure depends on the function new_data_structure(). 131 167 132 168 Default data structure is: 133 169 class[key] = value, Where key and value are of type string 134 170 … … 138 174 2. a dictionary with as list of values of type string. If 139 175 values contain a '=' character 140 176 141 eg: 177 eg: 142 178 print node['np'] 143 179 >> [ '2' ] 144 180 … … 149 185 for item in l: 150 186 new = class_func() 151 187 152 self.d[item.name] = new 153 188 self.d[item.name] = new 189 154 190 new.name = item.name 155 191 156 192 for a in item.attribs: … … 165 201 new[key] = a.value 166 202 167 203 else: 168 values = string.split(a.value, ',') 169 sub_dict = string.split(a.value, '=') 170 204 # Don't split , between () 205 values = [x[1] for x in REG_SPLIT_COMMA_BRACE.findall(a.value)] 171 206 172 # We must creat sub dicts, only for specified 207 # We must creat sub dicts, only for specified 173 208 # key values 174 209 # 175 210 if a.name in ['status', 'Variable_List']: 176 211 177 212 for v in values: 178 213 179 tmp_l = v.split('=') 214 # Don't split between () 215 tmp_l = [x[1] for x in REG_SPLIT_EQUAL_BRACE.findall(v)] 180 216 181 ## Support for multiple EVENT mesages in format [key=value:]+ 217 ## Support for multiple EVENT mesages in format [key=value:]+ 182 218 # format eg: message=EVENT:sample.time=1288864220.003,EVENT:kernel=upgrade,cputotals.user=0 183 219 # message=ERROR <text> 184 220 # … … 200 236 # 201 237 new['error'] = tmp_l [1:] 202 238 203 elif tmp_l[0].startswith('EVENT:'): 239 elif tmp_l[0].startswith('EVENT:'): 204 240 205 241 message_list = v.split(':') 206 242 for event_type in message_list[1:]: … … 212 248 # 213 249 if new.has_key(a.name): 214 250 215 new[a.name][ tmp_l[0] ] = tmp_l[1:] 251 new[a.name][ tmp_l[0] ] = tmp_l[1:] 216 252 217 253 else: 218 254 219 255 tmp_d = dict() 220 256 tmp_d[ tmp_l[0] ] = tmp_l[1:] 221 new[a.name] = class_func(tmp_d) 257 new[a.name] = class_func(tmp_d) 222 258 223 else: 259 else: 224 260 225 261 ## Check if it is a resource type variable, eg: 226 262 # - Resource_List.(nodes, walltime, ..) … … 233 269 else: 234 270 tmp_d = dict() 235 271 tmp_d[a.resource] = values 236 new[a.name] = class_func(tmp_d) 272 new[a.name] = class_func(tmp_d) 237 273 else: 238 274 # Simple value 239 275 # 240 276 new[a.name] = values 241 277 242 278 self._free(l) 243 279 244 280 def _free(self, memory): 245 281 """ 246 282 freeing up used memmory … … 253 289 if attrib_list: 254 290 self._list_2_attrib(attrib_list) 255 291 else: 256 self.attribs = 'NULL' 257 292 self.attribs = 'NULL' 293 258 294 self._connect() 259 295 serverinfo = pbs.pbs_statserver(self.con, self.attribs, 'NULL') 260 self._disconnect() 261 296 self._disconnect() 297 262 298 self._list_2_dict(serverinfo, server) 263 299 264 300 def get_serverinfo(self, attrib_list=None): … … 270 306 if attrib_list: 271 307 self._list_2_attrib(attrib_list) 272 308 else: 273 self.attribs = 'NULL' 274 309 self.attribs = 'NULL' 310 275 311 self._connect() 276 312 queues = pbs.pbs_statque(self.con, queue_name, self.attribs, 'NULL') 277 self._disconnect() 278 313 self._disconnect() 314 279 315 self._list_2_dict(queues, queue) 280 316 281 317 def getqueue(self, name, attrib_list=None): … … 284 320 return self.d[name] 285 321 except KeyError, detail: 286 322 return self.d 287 323 288 324 def getqueues(self, attrib_list=None): 289 325 self._statqueue('', attrib_list) 290 326 return self.d … … 294 330 if attrib_list: 295 331 self._list_2_attrib(attrib_list) 296 332 else: 297 self.attribs = 'NULL' 298 333 self.attribs = 'NULL' 334 299 335 if property: 300 336 select = ':%s' %(property) 301 337 302 338 self._connect() 303 339 nodes = pbs.pbs_statnode(self.con, select, self.attribs, 'NULL') 304 self._disconnect() 305 340 self._disconnect() 341 306 342 self._list_2_dict(nodes, node) 307 343 308 344 def getnode(self, name, attrib_list=None): … … 311 347 return self.d[name] 312 348 except KeyError, detail: 313 349 return self.d 314 350 315 351 def getnodes(self, attrib_list=None): 316 352 self._statnode('', attrib_list) 317 353 return self.d … … 325 361 if attrib_list: 326 362 self._list_2_attrib(attrib_list) 327 363 else: 328 self.attribs = 'NULL' 329 364 self.attribs = 'NULL' 365 330 366 self._connect() 331 367 jobs = pbs.pbs_statjob(self.con, job_name, self.attribs, 'NULL') 332 self._disconnect() 333 368 self._disconnect() 369 334 370 self._list_2_dict(jobs, job) 335 371 336 372 def getjob(self, name, attrib_list=None): … … 345 381 return self.d[name] 346 382 except KeyError, detail: 347 383 return self.d 348 384 349 385 def getjobs(self, attrib_list=None): 350 386 self._statjob('', attrib_list) 351 387 return self.d … … 353 389 def get_server_name(self): 354 390 return self.server 355 391 356 def new_data_structure(self): 392 def new_data_structure(self): 357 393 """ 358 394 Use the new data structure. Is now the default 359 395 """ 360 396 self.OLD_DATA_STRUCTURE = False 361 397 362 def old_data_structure(self): 398 def old_data_structure(self): 363 399 """ 364 400 Use the old data structure. This function is obselete and 365 401 will be removed in a future release … … 403 439 error = 'Attribute key error: %s' %(name) 404 440 raise PBSError(error) 405 441 406 ## Disabled for this moment, BvdV 16 July 2010407 #408 #def __setattr__(self, name, value):409 # """410 # override the class attribute set method only when the UserDict411 # has set its class attribute412 # """413 # if self.__dict__.has_key('data'):414 # self.data[name] = value415 # else:416 # self.__dict__[name] = value417 418 442 def __iter__(self): 419 443 return iter(self.data.keys()) 420 444 … … 436 460 if isinstance(self[key], types.ListType): 437 461 return self[key][0] 438 462 else: 439 return self[key] 463 return self[key] 440 464 441 465 class job(_PBSobject): 442 """PBS job class""" 466 """PBS job class""" 443 467 def is_running(self): 444 468 445 469 value = self.return_value('job_state') 446 470 if value == 'Q': 447 return self.TRUE 471 return self.TRUE 448 472 else: 449 473 return self.FALSE 450 474 … … 456 480 * split on '+' and if uniq is set split on '/' 457 481 """ 458 482 nodes = self.get_value('exec_host') 459 483 if not nodes: 484 return list() 485 460 486 if isinstance(nodes, str): 461 if nodes: 462 nodelist = string.split(nodes,'+') 463 if not unique: 464 return nodelist 487 nodelist = string.split(nodes,'+') 488 else: 489 nodelist = [] 490 for n in nodes: 491 if REG_SUBRANGE.search(n): 492 # This is a range split by _list_2_dict in a list 493 # E.g. exec_host node1/4,5,8-9 is after _list_dict ['node1/4', '5', '8-9'] 494 # Re-join them with the last node 495 nodelist[-1] += ',%s' % n 465 496 else: 466 l = list() 467 468 for n in nodelist: 469 t = string.split(n,'/') 470 if t[0] not in l: 471 l.append(t[0]) 497 nodelist.extend(n.split('+')) 472 498 473 return l 499 res=[] 500 for n in nodelist: 501 t = string.split(n,'/') 474 502 503 if not unique: 504 res.extend(["%s/%s" % (t[0],i) for i in convert_range(t[1])]) 475 505 else: 476 return list() 477 else: 478 l = list() 479 for n in nodes: 480 481 nlist = string.split(n,'+') 506 if t[0] not in res: 507 res.append(t[0]) 482 508 483 if unique: 484 for entry in nlist: 485 486 t = string.split(entry,'/') 487 if t[0] not in l: 488 l.append(t[0]) 489 else: 490 l += nlist 509 return res 491 510 492 return l493 494 511 495 512 class node(_PBSobject): 496 513 """PBS node class""" 497 514 498 515 def is_free(self): 499 516 """Check if node is free""" 500 517 501 518 value = self.return_value('state') 502 519 if value == 'free': 503 520 return self.TRUE 504 else: 505 return self.FALSE 521 else: 522 return self.FALSE 506 523 507 524 def has_job(self): 508 525 """Does the node run a job""" … … 511 528 return self.TRUE 512 529 except KeyError, detail: 513 530 return self.FALSE 514 531 515 532 def get_jobs(self, unique=None): 516 533 """Returns a list of the currently running job-id('s) on the node""" 517 534 518 535 jobs = self.get_value('jobs') 519 if jobs: 520 if isinstance(jobs, str): 521 jlist = re.compile('[^\\ /]\\d+[^/.]').findall( jobs ) 522 523 if not unique: 524 return jlist 525 else: 526 return self.uniq(jlist) 536 if not jobs: 537 return list() 527 538 539 if isinstance(jobs, str): 540 jlist = re.compile('[^\\ /]\\d+[^/.]').findall( jobs ) 541 542 if not unique: 543 return jlist 528 544 else: 529 job_re = re.compile('^(?:\d+/)?(.+)') 530 l = list() 545 return self.uniq(jlist) 531 546 547 else: 548 # Support id ranges before job id 549 joblist = [] 550 # Jobs might be splitted in ranges, but here _list_2_dict does 551 # 1,3,7-9/jobid -> ['1','3','7-9/jobid'] 552 # Process in reverse order 553 for j in jobs[::-1]: 554 if REG_SUBRANGE.search(j): 555 joblist[-1] = '%s,%s' % (j, joblist[-1]) 556 else: 557 joblist.append(j) 558 559 # extend with nodes 560 l = [] 561 562 for j in joblist[::-1]: 563 r=JOB_RE.search(j) 564 # 1st match is range, second part is jobid 565 jobstr=r.groups()[1] 532 566 if unique: 533 for j in jobs: 534 jobstr = job_re.findall(j.strip())[0] 535 if jobstr not in l: 536 l.append(jobstr) 537 538 return l 567 if jobstr not in l: 568 l.append(jobstr) 539 569 else: 540 return jobs541 542 return list()570 l.extend(["%s/%s"%(i,jobstr) for i in convert_range(r.groups()[0])]) 571 572 return l 543 573 544 574 545 575 class queue(_PBSobject): … … 548 578 549 579 value = self.return_value('enabled') 550 580 if value == 'True': 551 return self.TRUE 581 return self.TRUE 552 582 else: 553 583 return self.FALSE 554 584 … … 556 586 557 587 value = self.return_value('queue_type') 558 588 if value == 'Execution': 559 return self.TRUE 589 return self.TRUE 560 590 else: 561 591 return self.FALSE 562 592