Changeset 339
- Timestamp:
- 03/10/15 17:55:29 (9 years ago)
- Location:
- trunk
- Files:
-
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/CHANGES
r336 r339 2 2 * Added support for torque version 5.x. Note: the rm interface does not work for me on debian wheezy. 3 3 Author: Bas van der Vlies 4 5 * PBSQuery had been update to new cpuid range for torque 5 6 Author: ? 7 Applied by : Bas van der Vlies 4 8 5 9 =========== 4.5.0 -
trunk/README
r336 r339 15 15 The Python wrapper class is tested on: 16 16 17 - LISA cluster debian wheezy, torque 2.5. 717 - LISA cluster debian wheezy, torque 2.5.13 18 18 - LTC test cluster debian wheezy, torque 5.0.1 19 19 - LTC test cluster centos 6, torque 5.0.1 -
trunk/debian/changelog
r336 r339 1 1 pbs-python (4.6.0-1) UNRELEASED; urgency=low 2 2 3 * Added support torque 5.X3 * See Changes 4 4 5 5 -- Bas van der Vlies <bas.vandervlies@surfsara.nl> Tue, 10 Mar 2015 12:03:25 +0100 -
trunk/examples/new_interface.py
r286 r339 32 32 33 33 l = ['np', 'state' ] 34 node = p.getnode("gb-r5n1", l) 34 node = p.getnode("r2n2", l) 35 36 print node.name 35 37 print node.name, node['np'] 36 38 -
trunk/src/PBSQuery.py
r289 r339 1 1 # 2 # Authors: Roy Dragseth (roy.dragseth@cc.uit.no) 2 # Authors: Roy Dragseth (roy.dragseth@cc.uit.no) 3 3 # Bas van der Vlies (basv@sara.nl) 4 4 # … … 14 14 15 15 There are four batch objects: 16 - server 16 - server 17 17 - queue 18 18 - job … … 20 20 21 21 Each object can be handled as an dictionary and has several member 22 functions. The second parameter is an python list and can be used if you 22 functions. The second parameter is an python list and can be used if you 23 23 are only interested in certain resources, see example 24 24 25 25 There are the following functions for PBSQuery: 26 job - 26 job - 27 27 getjob(job_id, attributes=<default is all>) 28 28 getjobs(attributes=<default is all>) 29 29 30 30 node - 31 31 getnode(node_id, attributes=<default is all>) … … 53 53 print node, node['state'] 54 54 55 The parameter 'attributes' is an python list of resources that 55 The parameter 'attributes' is an python list of resources that 56 56 you are interested in, eg: only show state of nodes 57 57 l = list() … … 66 66 import types 67 67 68 69 REG_SUBRANGE = re.compile(r'^\d+(-\d+)?$') 70 """ 71 >>> b='rectime=1424696750,macaddr=40:a8:f0:2f:17:f4,cpuclock=Fixed,varattr=,jobs=419[1].master(cput=236745,energy_used=0,mem=6562224kb,vmem=7391872kb,walltime=22647,session_id=15941) 446[1].master(cput=7385,energy_used=0,mem=202936kb,vmem=368184kb,walltime=7391,session_id=30940) 446[2].master(cput=7385,energy_used=0,mem=204016kb,vmem=368184kb,walltime=7387,session_id=31168) 446[3].master(cput=7384,energy_used=0,mem=202380kb,vmem=368184kb,walltime=7387,session_id=31344) 446[4].master(cput=7384,energy_used=0,mem=204108kb,vmem=368184kb,walltime=7386,session_id=31536) 446[5].master(cput=7384,energy_used=0,mem=203940kb,vmem=368184kb,walltime=7386,session_id=31718) 446[6].master(cput=7383,energy_used=0,mem=188720kb,vmem=352608kb,walltime=7386,session_id=31904),state=free,size=456341748kb:459945088kb,netload=587288451179,gres=,loadave=18.07,ncpus=24,physmem=65850220kb,availmem=77961112kb,totmem=86821736kb,idletime=13933,nusers=1,nsessions=7,sessions=31904 31718 31536 31344 31168 30940 15941,uname=Linux node24 3.10.0 #1 SMP Wed Feb 4 08:16:54 CET 2015 x86_64,opsys=linux' 72 >>> c=','.join([x[1] for x in re.compile(r'((:?[^,(]+(?:\(.*?\))?))(?:,|$)').findall(b)]) 73 >>> c == b 74 True 75 76 """ 77 REG_SPLIT_COMMA_BRACE = re.compile(r'((:?[^,(]+(?:\(.*?\))?))(?:,|$)') 78 """ 79 >>> d='jobs=419[1].master(cput=236745,energy_used=0,mem=6562224kb,vmem=7391872kb,walltime=22647,session_id=15941) 446[1].master(cput=7385,energy_used=0,mem=202936kb,vmem=368184kb,walltime=7391,session_id=30940) 446[2].master(cput=7385,energy_used=0,mem=204016kb,vmem=368184kb,walltime=7387,session_id=31168) 446[3].master(cput=7384,energy_used=0,mem=202380kb,vmem=368184kb,walltime=7387,session_id=31344) 446[4].master(cput=7384,energy_used=0,mem=204108kb,vmem=368184kb,walltime=7386,session_id=31536) 446[5].master(cput=7384,energy_used=0,mem=203940kb,vmem=368184kb,walltime=7386,session_id=31718) 446[6].master(cput=7383,energy_used=0,mem=188720kb,vmem=352608kb,walltime=7386,session_id=31904)' 80 >>> e='='.join([x[1] for x in re.compile(r'((:?[^=(]+(?:\(.*?\))?))(?:=|$)').findall(d)]) 81 >>> d == e 82 True 83 """ 84 REG_SPLIT_EQUAL_BRACE = re.compile(r'((:?[^=(]+(?:\(.*?\))?))(?:=|$)') 85 86 JOB_RE = re.compile('(?:^|,)(?:((?:[\d,-]+)?\d+)/)?(.+)') 87 88 89 def convert_range(rangetxt): 90 """ 91 Convert range string into list of id strings: eg.g '3,5,7-9' -> ['3','5','7','8','9'] 92 """ 93 ids=[] 94 for subrange in [r.split('-') for r in rangetxt.split(',')]: 95 start = int(subrange[0]) 96 if(len(subrange) == 2 ): 97 end = int(subrange[1]) + 1 98 else: 99 end = start + 1 100 ids.extend([str(i) for i in range(start,end)]) 101 return ids 102 103 68 104 class PBSError(Exception): 69 105 def __init__(self, msg=''): 70 106 self.msg = msg 71 107 Exception.__init__(self, msg) 72 108 73 109 def __repr__(self): 74 110 return self.msg … … 93 129 # sequence_number.server (is not self.server) 94 130 # 95 self.job_server_id = list(self.get_serverinfo())[0] 131 self.job_server_id = list(self.get_serverinfo())[0] 96 132 self._disconnect() 97 133 … … 112 148 """Convert a python list to an attrib list suitable for pbs""" 113 149 self.attribs = pbs.new_attrl( len(list) ) 114 i = 0 150 i = 0 115 151 for attrib in list: 116 152 # So we can user Resource … … 127 163 def _list_2_dict(self, l, class_func): 128 164 """ 129 Convert a pbsstat function list to a class dictionary, The 165 Convert a pbsstat function list to a class dictionary, The 130 166 data structure depends on the function new_data_structure(). 131 167 132 168 Default data structure is: 133 169 class[key] = value, Where key and value are of type string … … 139 175 values contain a '=' character 140 176 141 eg: 177 eg: 142 178 print node['np'] 143 179 >> [ '2' ] … … 150 186 new = class_func() 151 187 152 self.d[item.name] = new 153 188 self.d[item.name] = new 189 154 190 new.name = item.name 155 191 … … 166 202 167 203 else: 168 values = string.split(a.value, ',') 169 sub_dict = string.split(a.value, '=') 170 171 172 # We must creat sub dicts, only for specified 204 # Don't split , between () 205 values = [x[1] for x in REG_SPLIT_COMMA_BRACE.findall(a.value)] 206 207 # We must creat sub dicts, only for specified 173 208 # key values 174 209 # … … 177 212 for v in values: 178 213 179 tmp_l = v.split('=') 180 181 ## Support for multiple EVENT mesages in format [key=value:]+ 214 # Don't split between () 215 tmp_l = [x[1] for x in REG_SPLIT_EQUAL_BRACE.findall(v)] 216 217 ## Support for multiple EVENT mesages in format [key=value:]+ 182 218 # format eg: message=EVENT:sample.time=1288864220.003,EVENT:kernel=upgrade,cputotals.user=0 183 219 # message=ERROR <text> … … 201 237 new['error'] = tmp_l [1:] 202 238 203 elif tmp_l[0].startswith('EVENT:'): 239 elif tmp_l[0].startswith('EVENT:'): 204 240 205 241 message_list = v.split(':') … … 213 249 if new.has_key(a.name): 214 250 215 new[a.name][ tmp_l[0] ] = tmp_l[1:] 251 new[a.name][ tmp_l[0] ] = tmp_l[1:] 216 252 217 253 else: … … 219 255 tmp_d = dict() 220 256 tmp_d[ tmp_l[0] ] = tmp_l[1:] 221 new[a.name] = class_func(tmp_d) 222 223 else: 257 new[a.name] = class_func(tmp_d) 258 259 else: 224 260 225 261 ## Check if it is a resource type variable, eg: … … 234 270 tmp_d = dict() 235 271 tmp_d[a.resource] = values 236 new[a.name] = class_func(tmp_d) 272 new[a.name] = class_func(tmp_d) 237 273 else: 238 274 # Simple value … … 241 277 242 278 self._free(l) 243 279 244 280 def _free(self, memory): 245 281 """ … … 254 290 self._list_2_attrib(attrib_list) 255 291 else: 256 self.attribs = 'NULL' 257 292 self.attribs = 'NULL' 293 258 294 self._connect() 259 295 serverinfo = pbs.pbs_statserver(self.con, self.attribs, 'NULL') 260 self._disconnect() 261 296 self._disconnect() 297 262 298 self._list_2_dict(serverinfo, server) 263 299 … … 271 307 self._list_2_attrib(attrib_list) 272 308 else: 273 self.attribs = 'NULL' 274 309 self.attribs = 'NULL' 310 275 311 self._connect() 276 312 queues = pbs.pbs_statque(self.con, queue_name, self.attribs, 'NULL') 277 self._disconnect() 278 313 self._disconnect() 314 279 315 self._list_2_dict(queues, queue) 280 316 … … 285 321 except KeyError, detail: 286 322 return self.d 287 323 288 324 def getqueues(self, attrib_list=None): 289 325 self._statqueue('', attrib_list) … … 295 331 self._list_2_attrib(attrib_list) 296 332 else: 297 self.attribs = 'NULL' 298 333 self.attribs = 'NULL' 334 299 335 if property: 300 336 select = ':%s' %(property) … … 302 338 self._connect() 303 339 nodes = pbs.pbs_statnode(self.con, select, self.attribs, 'NULL') 304 self._disconnect() 305 340 self._disconnect() 341 306 342 self._list_2_dict(nodes, node) 307 343 … … 312 348 except KeyError, detail: 313 349 return self.d 314 350 315 351 def getnodes(self, attrib_list=None): 316 352 self._statnode('', attrib_list) … … 326 362 self._list_2_attrib(attrib_list) 327 363 else: 328 self.attribs = 'NULL' 329 364 self.attribs = 'NULL' 365 330 366 self._connect() 331 367 jobs = pbs.pbs_statjob(self.con, job_name, self.attribs, 'NULL') 332 self._disconnect() 333 368 self._disconnect() 369 334 370 self._list_2_dict(jobs, job) 335 371 … … 346 382 except KeyError, detail: 347 383 return self.d 348 384 349 385 def getjobs(self, attrib_list=None): 350 386 self._statjob('', attrib_list) … … 354 390 return self.server 355 391 356 def new_data_structure(self): 392 def new_data_structure(self): 357 393 """ 358 394 Use the new data structure. Is now the default … … 360 396 self.OLD_DATA_STRUCTURE = False 361 397 362 def old_data_structure(self): 398 def old_data_structure(self): 363 399 """ 364 400 Use the old data structure. This function is obselete and … … 403 439 error = 'Attribute key error: %s' %(name) 404 440 raise PBSError(error) 405 406 ## Disabled for this moment, BvdV 16 July 2010407 #408 #def __setattr__(self, name, value):409 # """410 # override the class attribute set method only when the UserDict411 # has set its class attribute412 # """413 # if self.__dict__.has_key('data'):414 # self.data[name] = value415 # else:416 # self.__dict__[name] = value417 441 418 442 def __iter__(self): … … 437 461 return self[key][0] 438 462 else: 439 return self[key] 463 return self[key] 440 464 441 465 class job(_PBSobject): 442 """PBS job class""" 466 """PBS job class""" 443 467 def is_running(self): 444 468 445 469 value = self.return_value('job_state') 446 470 if value == 'Q': 447 return self.TRUE 471 return self.TRUE 448 472 else: 449 473 return self.FALSE … … 457 481 """ 458 482 nodes = self.get_value('exec_host') 459 483 if not nodes: 484 return list() 485 460 486 if isinstance(nodes, str): 461 if nodes: 462 nodelist = string.split(nodes,'+') 463 if not unique: 464 return nodelist 487 nodelist = string.split(nodes,'+') 488 else: 489 nodelist = [] 490 for n in nodes: 491 if REG_SUBRANGE.search(n): 492 # This is a range split by _list_2_dict in a list 493 # E.g. exec_host node1/4,5,8-9 is after _list_dict ['node1/4', '5', '8-9'] 494 # Re-join them with the last node 495 nodelist[-1] += ',%s' % n 465 496 else: 466 l = list() 467 468 for n in nodelist: 469 t = string.split(n,'/') 470 if t[0] not in l: 471 l.append(t[0]) 472 473 return l 474 497 nodelist.extend(n.split('+')) 498 499 res=[] 500 for n in nodelist: 501 t = string.split(n,'/') 502 503 if not unique: 504 res.extend(["%s/%s" % (t[0],i) for i in convert_range(t[1])]) 475 505 else: 476 return list() 477 else: 478 l = list() 479 for n in nodes: 480 481 nlist = string.split(n,'+') 482 483 if unique: 484 for entry in nlist: 485 486 t = string.split(entry,'/') 487 if t[0] not in l: 488 l.append(t[0]) 489 else: 490 l += nlist 491 492 return l 493 506 if t[0] not in res: 507 res.append(t[0]) 508 509 return res 510 494 511 495 512 class node(_PBSobject): 496 513 """PBS node class""" 497 514 498 515 def is_free(self): 499 516 """Check if node is free""" … … 502 519 if value == 'free': 503 520 return self.TRUE 504 else: 505 return self.FALSE 521 else: 522 return self.FALSE 506 523 507 524 def has_job(self): … … 512 529 except KeyError, detail: 513 530 return self.FALSE 514 531 515 532 def get_jobs(self, unique=None): 516 533 """Returns a list of the currently running job-id('s) on the node""" 517 534 518 535 jobs = self.get_value('jobs') 519 if jobs: 520 if isinstance(jobs, str): 521 jlist = re.compile('[^\\ /]\\d+[^/.]').findall( jobs ) 536 if not jobs: 537 return list() 538 539 if isinstance(jobs, str): 540 jlist = re.compile('[^\\ /]\\d+[^/.]').findall( jobs ) 541 542 if not unique: 543 return jlist 544 else: 545 return self.uniq(jlist) 546 547 else: 548 # Support id ranges before job id 549 joblist = [] 550 # Jobs might be splitted in ranges, but here _list_2_dict does 551 # 1,3,7-9/jobid -> ['1','3','7-9/jobid'] 552 # Process in reverse order 553 for j in jobs[::-1]: 554 if REG_SUBRANGE.search(j): 555 joblist[-1] = '%s,%s' % (j, joblist[-1]) 556 else: 557 joblist.append(j) 522 558 523 if not unique: 524 return jlist 559 # extend with nodes 560 l = [] 561 562 for j in joblist[::-1]: 563 r=JOB_RE.search(j) 564 # 1st match is range, second part is jobid 565 jobstr=r.groups()[1] 566 if unique: 567 if jobstr not in l: 568 l.append(jobstr) 525 569 else: 526 return self.uniq(jlist) 527 528 else: 529 job_re = re.compile('^(?:\d+/)?(.+)') 530 l = list() 531 532 if unique: 533 for j in jobs: 534 jobstr = job_re.findall(j.strip())[0] 535 if jobstr not in l: 536 l.append(jobstr) 537 538 return l 539 else: 540 return jobs 541 542 return list() 570 l.extend(["%s/%s"%(i,jobstr) for i in convert_range(r.groups()[0])]) 571 572 return l 543 573 544 574 … … 549 579 value = self.return_value('enabled') 550 580 if value == 'True': 551 return self.TRUE 581 return self.TRUE 552 582 else: 553 583 return self.FALSE … … 557 587 value = self.return_value('queue_type') 558 588 if value == 'Execution': 559 return self.TRUE 589 return self.TRUE 560 590 else: 561 591 return self.FALSE
Note: See TracChangeset
for help on using the changeset viewer.