Context Navigation

source: trunk/daemon/togad.py @ 36

Last change on this file since 36 was 36, checked in by bastiaans, 19 years ago

daemon/togad.py:

Rewrote multithreaded
Error occurs when trying to update a RRD with multiple values

File size: 16.3 KB

Rev	Line
[3]	1	#!/usr/bin/env python
	2
[5]	3	from xml.sax import make_parser
	4	from xml.sax.handler import ContentHandler
	5	import socket
	6	import sys
[9]	7	import rrdtool
	8	import string
[12]	9	import os
	10	import os.path
[17]	11	import time
	12	import re
[36]	13	import threading
	14	import mutex
[3]	15
[8]	16	# Specify debugging level here;
	17	#
[30]	18	# <=11 = metric XML
	19	# <=10 = host,cluster,grid,ganglia XML
	20	# <=9 = RRD activity,gmetad config parsing
	21	# <=8 = host processing
[31]	22	# <=7 = daemon threading - NOTE: Daemon will 'halt on all errors' from this level
[8]	23	#
[34]	24	DEBUG_LEVEL = 10
[6]	25
[9]	26	# Where is the gmetad.conf located
	27	#
	28	GMETAD_CONF = '/etc/gmetad.conf'
	29
	30	# Where to store the archived rrd's
	31	#
	32	ARCHIVE_PATH = '/data/toga/rrds'
	33
	34	# List of data_source names to archive for
	35	#
	36	ARCHIVE_SOURCES = [ "LISA Cluster" ]
	37
[13]	38	# Amount of hours to store in one single archived .rrd
[9]	39	#
[22]	40	ARCHIVE_HOURS_PER_RRD = 12
[13]	41
[22]	42	# Wether or not to run as a daemon in background
	43	#
	44	DAEMONIZE = 0
	45
[17]	46	######################
	47	# #
	48	# Configuration ends #
	49	# #
	50	######################
[13]	51
[17]	52	# What XML data types not to store
[13]	53	#
[17]	54	UNSUPPORTED_ARCHIVE_TYPES = [ 'string' ]
[9]	55
[8]	56	"""
	57	This is TOrque-GAnglia's data Daemon
	58	"""
	59
[6]	60	class GangliaXMLHandler( ContentHandler ):
[8]	61	"Parse Ganglia's XML"
[3]	62
[33]	63	def __init__( self, config ):
	64	self.config = config
[35]	65	self.clusters = { }
[33]	66
[6]	67	def startElement( self, name, attrs ):
[8]	68	"Store appropriate data from xml start tags"
[3]	69
[7]	70	if name == 'GANGLIA_XML':
[32]	71
	72	self.XMLSource = attrs.get( 'SOURCE', "" )
	73	self.gangliaVersion = attrs.get( 'VERSION', "" )
	74
[12]	75	debug_msg( 10, 'Found XML data: source %s version %s' %( self.XMLSource, self.gangliaVersion ) )
[6]	76
[7]	77	elif name == 'GRID':
[32]	78
	79	self.gridName = attrs.get( 'NAME', "" )
	80	self.time = attrs.get( 'LOCALTIME', "" )
	81
[12]	82	debug_msg( 10, '`-Grid found: %s' %( self.gridName ) )
[6]	83
[7]	84	elif name == 'CLUSTER':
[32]	85
	86	self.clusterName = attrs.get( 'NAME', "" )
	87	self.time = attrs.get( 'LOCALTIME', "" )
	88
[35]	89	if not self.clusters.has_key( self.clusterName ) and self.clusterName in ARCHIVE_SOURCES:
[32]	90
[34]	91	self.clusters[ self.clusterName ] = RRDHandler( self.config, self.clusterName )
[33]	92
[35]	93	debug_msg( 10, ' \|-Cluster found: %s' %( self.clusterName ) )
[6]	94
[14]	95	elif name == 'HOST' and self.clusterName in ARCHIVE_SOURCES:
[32]	96
	97	self.hostName = attrs.get( 'NAME', "" )
	98	self.hostIp = attrs.get( 'IP', "" )
	99	self.hostReported = attrs.get( 'REPORTED', "" )
	100
[12]	101	debug_msg( 10, ' \| \|-Host found: %s - ip %s reported %s' %( self.hostName, self.hostIp, self.hostReported ) )
[6]	102
[14]	103	elif name == 'METRIC' and self.clusterName in ARCHIVE_SOURCES:
[6]	104
[32]	105	type = attrs.get( 'TYPE', "" )
[6]	106
[32]	107	if type not in UNSUPPORTED_ARCHIVE_TYPES:
[3]	108
[32]	109	myMetric = { }
	110	myMetric['name'] = attrs.get( 'NAME', "" )
	111	myMetric['val'] = attrs.get( 'VAL', "" )
	112	myMetric['time'] = self.hostReported
[3]	113
[34]	114	self.clusters[ self.clusterName ].memMetric( self.hostName, myMetric )
[3]	115
[34]	116	debug_msg( 11, ' \| \| \|-metric: %s:%s' %( myMetric['name'], myMetric['val'] ) )
[6]	117
[34]	118	def storeMetrics( self ):
[9]	119
[34]	120	for clustername, rrdh in self.clusters.items():
[16]	121
[34]	122	print 'storing metrics of cluster ' + clustername
	123	rrdh.storeMetrics()
[9]	124
[5]	125	class GangliaXMLGatherer:
[8]	126	"Setup a connection and file object to Ganglia's XML"
[3]	127
[8]	128	s = None
	129
	130	def __init__( self, host, port ):
	131	"Store host and port for connection"
	132
[5]	133	self.host = host
	134	self.port = port
[33]	135	self.connect()
[3]	136
[33]	137	def connect( self ):
	138	"Setup connection to XML source"
[8]	139
	140	for res in socket.getaddrinfo( self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM ):
[32]	141
[5]	142	af, socktype, proto, canonname, sa = res
[32]	143
[5]	144	try:
[32]	145
[8]	146	self.s = socket.socket( af, socktype, proto )
[32]	147
[5]	148	except socket.error, msg:
[32]	149
[8]	150	self.s = None
[5]	151	continue
[32]	152
[5]	153	try:
[32]	154
[8]	155	self.s.connect( sa )
[32]	156
[5]	157	except socket.error, msg:
[32]	158
[8]	159	self.s.close()
	160	self.s = None
[5]	161	continue
[32]	162
[5]	163	break
[3]	164
[8]	165	if self.s is None:
[32]	166
[33]	167	debug_msg( 0, 'Could not open socket' )
	168	sys.exit( 1 )
[5]	169
[33]	170	def disconnect( self ):
	171	"Close socket"
	172
	173	if self.s:
	174	self.s.close()
	175	self.s = None
	176
	177	def __del__( self ):
	178	"Kill the socket before we leave"
	179
	180	self.disconnect()
	181
	182	def getFileObject( self ):
	183	"Connect, and return a file object"
	184
[36]	185	self.disconnect()
	186	self.connect()
[33]	187
[8]	188	return self.s.makefile( 'r' )
[5]	189
[8]	190	class GangliaXMLProcessor:
[5]	191
[33]	192	def __init__( self ):
	193	"Setup initial XML connection and handlers"
	194
	195	self.config = GangliaConfigParser( GMETAD_CONF )
	196
	197	self.myXMLGatherer = GangliaXMLGatherer( 'localhost', 8651 )
	198	self.myParser = make_parser()
	199	self.myHandler = GangliaXMLHandler( self.config )
	200	self.myParser.setContentHandler( self.myHandler )
	201
[9]	202	def daemon( self ):
[8]	203	"Run as daemon forever"
[5]	204
[8]	205	self.DAEMON = 1
[5]	206
[8]	207	# Fork the first child
	208	#
	209	pid = os.fork()
[32]	210
[8]	211	if pid > 0:
[7]	212
[32]	213	sys.exit(0) # end parent
	214
[8]	215	# creates a session and sets the process group ID
	216	#
	217	os.setsid()
[7]	218
[8]	219	# Fork the second child
	220	#
	221	pid = os.fork()
[32]	222
[8]	223	if pid > 0:
[5]	224
[32]	225	sys.exit(0) # end parent
	226
[8]	227	# Go to the root directory and set the umask
	228	#
	229	os.chdir('/')
	230	os.umask(0)
	231
	232	sys.stdin.close()
	233	sys.stdout.close()
[36]	234	#sys.stderr.close()
[8]	235
	236	os.open('/dev/null', 0)
	237	os.dup(0)
	238	os.dup(0)
	239
	240	self.run()
	241
[22]	242	def printTime( self ):
[33]	243	"Print current time in human readable format"
[22]	244
[33]	245	return time.strftime("%a %d %b %Y %H:%M:%S")
[22]	246
[33]	247	def grabXML( self ):
	248
	249	debug_msg( 7, self.printTime() + ' - mainthread() - xmlthread() started' )
	250	pid = os.fork()
	251
	252	if pid == 0:
	253	# Child - XML Thread
	254	#
	255	# Process XML and exit
	256
	257	debug_msg( 7, self.printTime() + ' - xmlthread() - Start XML processing..' )
	258	self.processXML()
	259	debug_msg( 7, self.printTime() + ' - xmlthread() - Done processing; exiting.' )
	260	sys.exit( 0 )
	261
	262	elif pid > 0:
	263	# Parent - Time/sleep Thread
	264	#
	265	# Make sure XML is processed on time and at regular intervals
	266
[34]	267	debug_msg( 7, self.printTime() + ' - mainthread() - Sleep '+ str( self.config.getLowestInterval() ) +'s: zzzzz..' )
	268	time.sleep( float( self.config.getLowestInterval() ) )
[33]	269	debug_msg( 7, self.printTime() + ' - mainthread() - Awoken: waiting for XML thread..' )
	270
	271	r = os.wait()
	272	ret = r[1]
	273	if ret != 0:
	274	debug_msg( 7, self.printTime() + ' - mainthread() - Done waiting: ERROR! xmlthread() exited with status %d' %(ret) )
	275	if DEBUG_LEVEL>=7: sys.exit( 1 )
	276	else:
	277
	278	debug_msg( 7, self.printTime() + ' - mainthread() - Done waiting: xmlthread() finished succesfully' )
	279
[9]	280	def run( self ):
[8]	281	"Main thread"
	282
[36]	283	#self.processXML()
	284	#self.storeMetrics()
[22]	285
[36]	286	#time.sleep( 20 )
[22]	287
[36]	288	#self.processXML()
	289	#self.storeMetrics()
[22]	290
[36]	291	#sys.exit(1)
[22]	292
[36]	293	xmlthread = threading.Thread( None, self.processXML, 'xmlthread' )
	294	storethread = threading.Thread( None, self.storeMetrics, 'storethread' )
[22]	295
[36]	296	while( 1 ):
	297
	298	if not xmlthread.isAlive():
	299	# Gather XML at the same interval as gmetad
	300
	301	# threaded call to: self.processXML()
	302	#
	303	xmlthread = threading.Thread( None, self.processXML, 'xmlthread' )
	304	debug_msg( 7, self.printTime() + ' - mainthread() - xmlthread() started' )
	305	xmlthread.start()
	306
	307	if not storethread.isAlive():
	308	# Store metrics every .. sec
	309
	310	# threaded call to: self.storeMetrics()
	311	#
	312	storethread = threading.Thread( None, self.storeMetrics, 'storethread' )
	313	debug_msg( 7, self.printTime() + ' - mainthread() - storethread() started' )
	314	storethread.start()
	315
	316	# Just sleep a sec here, to prevent daemon from going mad. We're all threads here anyway
	317	time.sleep( 1 )
	318
[33]	319	def storeMetrics( self ):
	320	"Store metrics retained in memory to disk"
[22]	321
[36]	322	STORE_INTERVAL = 30
[22]	323
[36]	324	debug_msg( 7, self.printTime() + ' - storethread() - Storing data..' )
	325
	326	# threaded call to: self.myHandler.storeMetrics()
	327	#
	328	storethread = threading.Thread( None, self.myHandler.storeMetrics(), 'storemetricthread' )
	329	storethread.start()
	330
	331	debug_msg( 7, self.printTime() + ' - storethread() - Sleeping.. (%ss)' %STORE_INTERVAL )
	332	time.sleep( STORE_INTERVAL )
	333	debug_msg( 7, self.printTime() + ' - storethread() - Done sleeping.' )
	334
	335	if storethread.isAlive():
	336
	337	debug_msg( 7, self.printTime() + ' - storethread() - storemetricthread() still running, waiting to finish..' )
	338	parsethread.join( 180 ) # Maximum time is 3 minutes for storing thread to finish - more then enough
	339	debug_msg( 7, self.printTime() + ' - storethread() - storemetricthread() finished.' )
	340
	341	debug_msg( 7, self.printTime() + ' - storethread() finished' )
	342
	343	return 0
	344
[8]	345	def processXML( self ):
	346	"Process XML"
	347
[36]	348	debug_msg( 7, self.printTime() + ' - xmlthread() - Parsing..' )
[8]	349
[36]	350	# threaded call to: self.myParser.parse( self.myXMLGatherer.getFileObject() )
	351	#
	352	parsethread = threading.Thread( None, self.myParser.parse, 'parsethread', [ self.myXMLGatherer.getFileObject() ] )
	353	parsethread.start()
	354
	355	debug_msg( 7, self.printTime() + ' - xmlthread() - Sleeping.. (%ss)' %self.config.getLowestInterval() )
	356	time.sleep( float( self.config.getLowestInterval() ) )
	357	debug_msg( 7, self.printTime() + ' - xmlthread() - Done sleeping.' )
	358
	359	if parsethread.isAlive():
	360
	361	debug_msg( 7, self.printTime() + ' - xmlthread() - parsethread() still running, waiting to finish..' )
	362	parsethread.join( 180 ) # Maximum time is 3 minutes for XML thread to finish - more then enough
	363	debug_msg( 7, self.printTime() + ' - xmlthread() - parsethread() finished.' )
	364
	365	debug_msg( 7, self.printTime() + ' - xmlthread() finished.' )
	366
	367	return 0
	368
[9]	369	class GangliaConfigParser:
	370
[34]	371	sources = [ ]
[9]	372
	373	def __init__( self, config ):
[32]	374
[9]	375	self.config = config
	376	self.parseValues()
	377
[32]	378	def parseValues( self ):
[9]	379	"Parse certain values from gmetad.conf"
	380
	381	readcfg = open( self.config, 'r' )
	382
	383	for line in readcfg.readlines():
	384
	385	if line.count( '"' ) > 1:
	386
[10]	387	if line.find( 'data_source' ) != -1 and line[0] != '#':
[9]	388
[11]	389	source = { }
	390	source['name'] = line.split( '"' )[1]
[9]	391	source_words = line.split( '"' )[2].split( ' ' )
	392
	393	for word in source_words:
	394
	395	valid_interval = 1
	396
	397	for letter in word:
[32]	398
[9]	399	if letter not in string.digits:
[32]	400
[9]	401	valid_interval = 0
	402
[10]	403	if valid_interval and len(word) > 0:
[32]	404
[9]	405	source['interval'] = word
[12]	406	debug_msg( 9, 'polling interval for %s = %s' %(source['name'], source['interval'] ) )
[33]	407
	408	# No interval found, use Ganglia's default
	409	if not source.has_key( 'interval' ):
	410	source['interval'] = 15
	411	debug_msg( 9, 'polling interval for %s defaulted to 15' %(source['name']) )
[32]	412
[33]	413	self.sources.append( source )
[9]	414
	415	def getInterval( self, source_name ):
[32]	416
[9]	417	for source in self.sources:
[32]	418
[12]	419	if source['name'] == source_name:
[32]	420
[9]	421	return source['interval']
[32]	422
[9]	423	return None
	424
[34]	425	def getLowestInterval( self ):
	426
	427	lowest_interval = 0
	428
	429	for source in self.sources:
	430
	431	if not lowest_interval or source['interval'] <= lowest_interval:
	432
	433	lowest_interval = source['interval']
	434
	435	# Return 15 when nothing is found, so that the daemon won't go insane with 0 sec delays
	436	if lowest_interval:
	437	return lowest_interval
	438	else:
	439	return 15
	440
[9]	441	class RRDHandler:
	442
[32]	443	myMetrics = { }
[36]	444	slot = None
[32]	445
[33]	446	def __init__( self, config, cluster ):
[9]	447	self.cluster = cluster
[33]	448	self.config = config
[36]	449	self.slot = mutex.mutex()
[9]	450
[32]	451	def getClusterName( self ):
	452	return self.cluster
	453
	454	def memMetric( self, host, metric ):
	455
[34]	456	if self.myMetrics.has_key( host ):
[32]	457
[34]	458	if self.myMetrics[ host ].has_key( metric['name'] ):
[32]	459
[34]	460	for mymetric in self.myMetrics[ host ][ metric['name'] ]:
[32]	461
[34]	462	if mymetric['time'] == metric['time']:
[32]	463
[34]	464	# Allready have this metric, abort
	465	return 1
	466	else:
	467	self.myMetrics[ host ][ metric['name'] ] = [ ]
	468	else:
[32]	469	self.myMetrics[ host ] = { }
[34]	470	self.myMetrics[ host ][ metric['name'] ] = [ ]
[32]	471
[36]	472	self.slot.testandset()
[32]	473	self.myMetrics[ host ][ metric['name'] ].append( metric )
[36]	474	self.slot.unlock()
[32]	475
[35]	476	def makeUpdateString( self, host, metricname ):
[32]	477
	478	update_string = ''
	479
[35]	480	for m in self.myMetrics[ host ][ metricname ]:
[32]	481
[35]	482	update_string = update_string + ' %s:%s' %( m['time'], m['val'] )
[32]	483
	484	return update_string
	485
[33]	486	def storeMetrics( self ):
	487
	488	for hostname, mymetrics in self.myMetrics.items():
	489
	490	for metricname, mymetric in mymetrics.items():
	491
[35]	492	mytime = self.makeTimeSerial()
	493	self.createCheck( hostname, metricname, mytime )
[36]	494	update_ret = self.update( hostname, metricname, mytime )
[33]	495
[36]	496	if update_ret == 0:
[33]	497
[36]	498	self.slot.testandset()
[33]	499	del self.myMetrics[ hostname ][ metricname ]
[36]	500	self.slot.unlock()
[33]	501	debug_msg( 9, 'stored metric %s for %s' %( hostname, metricname ) )
	502	else:
	503	debug_msg( 9, 'metric update failed' )
	504
[36]	505	return 1
[33]	506
[17]	507	def makeTimeSerial( self ):
[32]	508	"Generate a time serial. Seconds since epoch"
[17]	509
	510	# Seconds since epoch
	511	mytime = int( time.time() )
	512
	513	return mytime
	514
[33]	515	def makeRrdPath( self, host, metricname=None, timeserial=None ):
[32]	516	"""
	517	Make a RRD location/path and filename
	518	If a metric or timeserial are supplied the complete locations
	519	will be made, else just the host directory
	520	"""
[17]	521
[20]	522	if not timeserial:
	523	rrd_dir = '%s/%s/%s' %( check_dir(ARCHIVE_PATH), self.cluster, host )
	524	else:
	525	rrd_dir = '%s/%s/%s/%s' %( check_dir(ARCHIVE_PATH), self.cluster, host, timeserial )
[35]	526	if metricname:
[33]	527	rrd_file = '%s/%s.rrd' %( rrd_dir, metricname )
[20]	528	else:
	529	rrd_file = None
[17]	530
	531	return rrd_dir, rrd_file
	532
[20]	533	def getLastRrdTimeSerial( self, host ):
[32]	534	"""
	535	Find the last timeserial (directory) for this host
	536	This is determined once every host
	537	"""
[17]	538
[20]	539	rrd_dir, rrd_file = self.makeRrdPath( host )
[17]	540
[19]	541	newest_timeserial = 0
	542
[17]	543	if os.path.exists( rrd_dir ):
[32]	544
[17]	545	for root, dirs, files in os.walk( rrd_dir ):
	546
[20]	547	for dir in dirs:
[17]	548
[20]	549	valid_dir = 1
[17]	550
[20]	551	for letter in dir:
	552	if letter not in string.digits:
	553	valid_dir = 0
[17]	554
[20]	555	if valid_dir:
	556	timeserial = dir
[17]	557	if timeserial > newest_timeserial:
	558	newest_timeserial = timeserial
	559
	560	if newest_timeserial:
[18]	561	return newest_timeserial
[17]	562	else:
	563	return 0
	564
[20]	565	def checkNewRrdPeriod( self, host, current_timeserial ):
[32]	566	"""
	567	Check if current timeserial belongs to recent time period
	568	or should become a new period (and file).
[17]	569
[32]	570	Returns the serial of the correct time period
	571	"""
	572
[20]	573	last_timeserial = int( self.getLastRrdTimeSerial( host ) )
[30]	574	debug_msg( 9, 'last timeserial of %s is %s' %( host, last_timeserial ) )
[17]	575
	576	if not last_timeserial:
[18]	577	serial = current_timeserial
	578	else:
[17]	579
[18]	580	archive_secs = ARCHIVE_HOURS_PER_RRD * (60 * 60)
[17]	581
[18]	582	if (current_timeserial - last_timeserial) >= archive_secs:
	583	serial = current_timeserial
	584	else:
	585	serial = last_timeserial
[17]	586
[18]	587	return serial
	588
[33]	589	def getFirstTime( self, host, metricname ):
	590	"Get the first time of a metric we know of"
	591
	592	first_time = 0
	593
	594	for metric in self.myMetrics[ host ][ metricname ]:
	595
	596	if not first_time or metric['time'] <= first_time:
	597
	598	first_time = metric['time']
	599
[35]	600	return first_time
	601
[33]	602	def createCheck( self, host, metricname, timeserial ):
[9]	603	"Check if an .rrd allready exists for this metric, create if not"
	604
[35]	605	debug_msg( 9, 'rrdcreate: using timeserial %s for %s/%s' %( timeserial, host, metricname ) )
[9]	606
[33]	607	rrd_dir, rrd_file = self.makeRrdPath( host, metricname, timeserial )
[17]	608
[9]	609	if not os.path.exists( rrd_dir ):
	610	os.makedirs( rrd_dir )
[14]	611	debug_msg( 9, 'created dir %s' %( str(rrd_dir) ) )
[9]	612
[14]	613	if not os.path.exists( rrd_file ):
[9]	614
[33]	615	interval = self.config.getInterval( self.cluster )
[14]	616	heartbeat = 8 * int(interval)
[9]	617
[14]	618	param_step1 = '--step'
	619	param_step2 = str( interval )
[12]	620
[14]	621	param_start1 = '--start'
[33]	622	param_start2 = str( int( self.getFirstTime( host, metricname ) ) - 1 )
[12]	623
[14]	624	param_ds = 'DS:sum:GAUGE:%d:U:U' %heartbeat
	625	param_rra = 'RRA:AVERAGE:0.5:1:%s' %(ARCHIVE_HOURS_PER_RRD * 240)
[12]	626
[14]	627	rrdtool.create( str(rrd_file), param_step1, param_step2, param_start1, param_start2, param_ds, param_rra )
[13]	628
[14]	629	debug_msg( 9, 'created rrd %s' %( str(rrd_file) ) )
	630
[33]	631	def update( self, host, metricname, timeserial ):
[9]	632
[35]	633	debug_msg( 9, 'rrdupdate: using timeserial %s for %s/%s' %( timeserial, host, metricname ) )
[9]	634
[33]	635	rrd_dir, rrd_file = self.makeRrdPath( host, metricname, timeserial )
[18]	636
[33]	637	#timestamp = metric['time']
	638	#val = metric['val']
[9]	639
[33]	640	#update_string = '%s:%s' %(timestamp, val)
	641	update_string = self.makeUpdateString( host, metricname )
[15]	642
[27]	643	try:
[32]	644
[27]	645	rrdtool.update( str(rrd_file), str(update_string) )
[32]	646
[28]	647	except rrdtool.error, detail:
[32]	648
[27]	649	debug_msg( 0, 'EXCEPTION! While trying to update rrd:' )
	650	debug_msg( 0, '\trrd %s with %s' %( str(rrd_file), update_string ) )
[28]	651	debug_msg( 0, str(detail) )
[32]	652
[33]	653	return 1
[27]	654
[15]	655	debug_msg( 9, 'updated rrd %s with %s' %( str(rrd_file), update_string ) )
	656
[36]	657	return 0
	658
[8]	659	def main():
	660	"Program startup"
	661
	662	myProcessor = GangliaXMLProcessor()
	663
[22]	664	if DAEMONIZE:
	665	myProcessor.daemon()
	666	else:
	667	myProcessor.run()
	668
[9]	669	def check_dir( directory ):
	670	"Check if directory is a proper directory. I.e.: Does _not_ end with a '/'"
	671
	672	if directory[-1] == '/':
	673	directory = directory[:-1]
	674
	675	return directory
	676
[12]	677	def debug_msg( level, msg ):
	678
	679	if (DEBUG_LEVEL >= level):
	680	sys.stderr.write( msg + '\n' )
	681
[5]	682	# Let's go
[9]	683	if __name__ == '__main__':
	684	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: