Context Navigation

source: trunk/daemon/togad.py @ 39

Last change on this file since 39 was 39, checked in by bastiaans, 19 years ago

daemon/togad.py:

Changed threading debug msg'es to be more understandable

File size: 16.1 KB

Rev	Line
[3]	1	#!/usr/bin/env python
	2
[5]	3	from xml.sax import make_parser
	4	from xml.sax.handler import ContentHandler
	5	import socket
	6	import sys
[9]	7	import string
[12]	8	import os
	9	import os.path
[17]	10	import time
	11	import re
[36]	12	import threading
	13	import mutex
[38]	14	import random
[3]	15
[8]	16	# Specify debugging level here;
	17	#
[38]	18	# 11 = XML: metrics
	19	# 10 = XML: host, cluster, grid, ganglia
	20	# 9 = RRD activity, gmetad config parsing
	21	# 8 = daemon threading
[8]	22	#
[38]	23	DEBUG_LEVEL = 8
[6]	24
[9]	25	# Where is the gmetad.conf located
	26	#
	27	GMETAD_CONF = '/etc/gmetad.conf'
	28
	29	# Where to store the archived rrd's
	30	#
	31	ARCHIVE_PATH = '/data/toga/rrds'
	32
	33	# List of data_source names to archive for
	34	#
	35	ARCHIVE_SOURCES = [ "LISA Cluster" ]
	36
[13]	37	# Amount of hours to store in one single archived .rrd
[9]	38	#
[38]	39	ARCHIVE_HOURS_PER_RRD = 1
[13]	40
[22]	41	# Wether or not to run as a daemon in background
	42	#
	43	DAEMONIZE = 0
	44
[17]	45	######################
	46	# #
	47	# Configuration ends #
	48	# #
	49	######################
[13]	50
[17]	51	# What XML data types not to store
[13]	52	#
[17]	53	UNSUPPORTED_ARCHIVE_TYPES = [ 'string' ]
[9]	54
[8]	55	"""
	56	This is TOrque-GAnglia's data Daemon
	57	"""
	58
[37]	59	class RRDMutator:
[38]	60	"A class for handling .rrd mutations"
[37]	61
	62	binary = '/usr/bin/rrdtool'
	63
	64	def __init__( self, binary=None ):
	65
	66	if binary:
	67	self.binary = binary
	68
	69	def create( self, filename, args ):
	70	return self.perform( 'create' + ' "' + filename + '"', args )
	71
	72	def update( self, filename, args ):
	73	return self.perform( 'update' + ' "' + filename + '"', args )
	74
	75	def perform( self, action, args ):
	76
	77	arg_string = None
	78
	79	for arg in args:
	80
	81	if not arg_string:
	82
	83	arg_string = arg
	84	else:
	85	arg_string = arg_string + ' ' + arg
	86
[38]	87	debug_msg( 9, 'rrdm.perform(): ' + self.binary + ' ' + action + ' ' + arg_string )
[37]	88
	89	for line in os.popen( self.binary + ' ' + action + ' ' + arg_string ).readlines():
	90
	91	if line.find( 'ERROR' ) != -1:
	92
	93	error_msg = string.join( line.split( ' ' )[1:] )
[38]	94	debug_msg( 9, error_msg )
[37]	95	return 1
	96
	97	return 0
	98
[6]	99	class GangliaXMLHandler( ContentHandler ):
[8]	100	"Parse Ganglia's XML"
[3]	101
[33]	102	def __init__( self, config ):
	103	self.config = config
[35]	104	self.clusters = { }
[33]	105
[6]	106	def startElement( self, name, attrs ):
[8]	107	"Store appropriate data from xml start tags"
[3]	108
[7]	109	if name == 'GANGLIA_XML':
[32]	110
	111	self.XMLSource = attrs.get( 'SOURCE', "" )
	112	self.gangliaVersion = attrs.get( 'VERSION', "" )
	113
[12]	114	debug_msg( 10, 'Found XML data: source %s version %s' %( self.XMLSource, self.gangliaVersion ) )
[6]	115
[7]	116	elif name == 'GRID':
[32]	117
	118	self.gridName = attrs.get( 'NAME', "" )
	119	self.time = attrs.get( 'LOCALTIME', "" )
	120
[12]	121	debug_msg( 10, '`-Grid found: %s' %( self.gridName ) )
[6]	122
[7]	123	elif name == 'CLUSTER':
[32]	124
	125	self.clusterName = attrs.get( 'NAME', "" )
	126	self.time = attrs.get( 'LOCALTIME', "" )
	127
[35]	128	if not self.clusters.has_key( self.clusterName ) and self.clusterName in ARCHIVE_SOURCES:
[32]	129
[34]	130	self.clusters[ self.clusterName ] = RRDHandler( self.config, self.clusterName )
[33]	131
[35]	132	debug_msg( 10, ' \|-Cluster found: %s' %( self.clusterName ) )
[6]	133
[14]	134	elif name == 'HOST' and self.clusterName in ARCHIVE_SOURCES:
[32]	135
	136	self.hostName = attrs.get( 'NAME', "" )
	137	self.hostIp = attrs.get( 'IP', "" )
	138	self.hostReported = attrs.get( 'REPORTED', "" )
	139
[12]	140	debug_msg( 10, ' \| \|-Host found: %s - ip %s reported %s' %( self.hostName, self.hostIp, self.hostReported ) )
[6]	141
[14]	142	elif name == 'METRIC' and self.clusterName in ARCHIVE_SOURCES:
[6]	143
[32]	144	type = attrs.get( 'TYPE', "" )
[6]	145
[32]	146	if type not in UNSUPPORTED_ARCHIVE_TYPES:
[3]	147
[32]	148	myMetric = { }
	149	myMetric['name'] = attrs.get( 'NAME', "" )
	150	myMetric['val'] = attrs.get( 'VAL', "" )
	151	myMetric['time'] = self.hostReported
[3]	152
[34]	153	self.clusters[ self.clusterName ].memMetric( self.hostName, myMetric )
[3]	154
[34]	155	debug_msg( 11, ' \| \| \|-metric: %s:%s' %( myMetric['name'], myMetric['val'] ) )
[6]	156
[34]	157	def storeMetrics( self ):
[9]	158
[34]	159	for clustername, rrdh in self.clusters.items():
[16]	160
[38]	161	ret = rrdh.storeMetrics()
[9]	162
[38]	163	if ret:
	164	debug_msg( 9, 'An error occured while storing metrics for cluster %s' %clustername )
	165	return 1
	166
	167	return 0
	168
[5]	169	class GangliaXMLGatherer:
[8]	170	"Setup a connection and file object to Ganglia's XML"
[3]	171
[8]	172	s = None
	173
	174	def __init__( self, host, port ):
	175	"Store host and port for connection"
	176
[5]	177	self.host = host
	178	self.port = port
[33]	179	self.connect()
[3]	180
[33]	181	def connect( self ):
	182	"Setup connection to XML source"
[8]	183
	184	for res in socket.getaddrinfo( self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM ):
[32]	185
[5]	186	af, socktype, proto, canonname, sa = res
[32]	187
[5]	188	try:
[32]	189
[8]	190	self.s = socket.socket( af, socktype, proto )
[32]	191
[5]	192	except socket.error, msg:
[32]	193
[8]	194	self.s = None
[5]	195	continue
[32]	196
[5]	197	try:
[32]	198
[8]	199	self.s.connect( sa )
[32]	200
[5]	201	except socket.error, msg:
[32]	202
[8]	203	self.s.close()
	204	self.s = None
[5]	205	continue
[32]	206
[5]	207	break
[3]	208
[8]	209	if self.s is None:
[32]	210
[33]	211	debug_msg( 0, 'Could not open socket' )
	212	sys.exit( 1 )
[5]	213
[33]	214	def disconnect( self ):
	215	"Close socket"
	216
	217	if self.s:
	218	self.s.close()
	219	self.s = None
	220
	221	def __del__( self ):
	222	"Kill the socket before we leave"
	223
	224	self.disconnect()
	225
	226	def getFileObject( self ):
	227	"Connect, and return a file object"
	228
[38]	229	if self.s:
	230	# Apearantly, only data is received when a connection is made
	231	# therefor, disconnect and connect
	232	#
	233	self.disconnect()
	234	self.connect()
[33]	235
[8]	236	return self.s.makefile( 'r' )
[5]	237
[8]	238	class GangliaXMLProcessor:
[5]	239
[33]	240	def __init__( self ):
	241	"Setup initial XML connection and handlers"
	242
	243	self.config = GangliaConfigParser( GMETAD_CONF )
	244
	245	self.myXMLGatherer = GangliaXMLGatherer( 'localhost', 8651 )
	246	self.myParser = make_parser()
	247	self.myHandler = GangliaXMLHandler( self.config )
	248	self.myParser.setContentHandler( self.myHandler )
	249
[9]	250	def daemon( self ):
[8]	251	"Run as daemon forever"
[5]	252
[8]	253	self.DAEMON = 1
[5]	254
[8]	255	# Fork the first child
	256	#
	257	pid = os.fork()
[32]	258
[8]	259	if pid > 0:
[7]	260
[32]	261	sys.exit(0) # end parent
	262
[8]	263	# creates a session and sets the process group ID
	264	#
	265	os.setsid()
[7]	266
[8]	267	# Fork the second child
	268	#
	269	pid = os.fork()
[32]	270
[8]	271	if pid > 0:
[5]	272
[32]	273	sys.exit(0) # end parent
	274
[8]	275	# Go to the root directory and set the umask
	276	#
	277	os.chdir('/')
	278	os.umask(0)
	279
	280	sys.stdin.close()
	281	sys.stdout.close()
[36]	282	#sys.stderr.close()
[8]	283
	284	os.open('/dev/null', 0)
	285	os.dup(0)
	286	os.dup(0)
	287
	288	self.run()
	289
[22]	290	def printTime( self ):
[33]	291	"Print current time in human readable format"
[22]	292
[33]	293	return time.strftime("%a %d %b %Y %H:%M:%S")
[22]	294
[9]	295	def run( self ):
[8]	296	"Main thread"
	297
[36]	298	xmlthread = threading.Thread( None, self.processXML, 'xmlthread' )
	299	storethread = threading.Thread( None, self.storeMetrics, 'storethread' )
[22]	300
[36]	301	while( 1 ):
	302
	303	if not xmlthread.isAlive():
	304	# Gather XML at the same interval as gmetad
	305
	306	# threaded call to: self.processXML()
	307	#
	308	xmlthread = threading.Thread( None, self.processXML, 'xmlthread' )
	309	xmlthread.start()
	310
	311	if not storethread.isAlive():
	312	# Store metrics every .. sec
	313
	314	# threaded call to: self.storeMetrics()
	315	#
	316	storethread = threading.Thread( None, self.storeMetrics, 'storethread' )
	317	storethread.start()
	318
	319	# Just sleep a sec here, to prevent daemon from going mad. We're all threads here anyway
	320	time.sleep( 1 )
	321
[33]	322	def storeMetrics( self ):
	323	"Store metrics retained in memory to disk"
[22]	324
[39]	325	debug_msg( 8, self.printTime() + ' - storethread(): started.' )
	326
[38]	327	# Store metrics somewhere between every 60 and 180 seconds
	328	#
	329	STORE_INTERVAL = random.randint( 60, 180 )
[22]	330
[39]	331	storethread = threading.Thread( None, self.storeThread, 'storemetricthread' )
[36]	332	storethread.start()
	333
[39]	334	debug_msg( 8, self.printTime() + ' - storethread(): Sleeping.. (%ss)' %STORE_INTERVAL )
[36]	335	time.sleep( STORE_INTERVAL )
[39]	336	debug_msg( 8, self.printTime() + ' - storethread(): Done sleeping.' )
[36]	337
	338	if storethread.isAlive():
	339
[39]	340	debug_msg( 8, self.printTime() + ' - storethread(): storemetricthread() still running, waiting to finish..' )
	341	parsethread.join( 180 ) # Maximum time is for storing thread to finish
	342	debug_msg( 8, self.printTime() + ' - storethread(): Done waiting.' )
[36]	343
[39]	344	debug_msg( 8, self.printTime() + ' - storethread(): finished.' )
[36]	345
	346	return 0
	347
[39]	348	def storeThread( self ):
	349
	350	debug_msg( 8, self.printTime() + ' - storemetricthread(): started.' )
	351	debug_msg( 8, self.printTime() + ' - storemetricthread(): Storing data..' )
	352	ret = self.myHandler.storeMetrics()
	353	debug_msg( 8, self.printTime() + ' - storemetricthread(): Done storing.' )
	354	debug_msg( 8, self.printTime() + ' - storemetricthread(): finished.' )
	355
	356	return ret
	357
[8]	358	def processXML( self ):
	359	"Process XML"
	360
[39]	361	debug_msg( 8, self.printTime() + ' - xmlthread(): started.' )
[8]	362
[39]	363	parsethread = threading.Thread( None, self.parseThread, 'parsethread' )
[36]	364	parsethread.start()
	365
[39]	366	debug_msg( 8, self.printTime() + ' - xmlthread(): Sleeping.. (%ss)' %self.config.getLowestInterval() )
[36]	367	time.sleep( float( self.config.getLowestInterval() ) )
[39]	368	debug_msg( 8, self.printTime() + ' - xmlthread(): Done sleeping.' )
[36]	369
	370	if parsethread.isAlive():
	371
[39]	372	debug_msg( 8, self.printTime() + ' - xmlthread(): parsethread() still running, waiting to finish..' )
	373	parsethread.join( 60 ) # Maximum time for XML thread to finish
	374	debug_msg( 8, self.printTime() + ' - xmlthread(): Done waiting.' )
[36]	375
[39]	376	debug_msg( 8, self.printTime() + ' - xmlthread(): finished.' )
[36]	377
	378	return 0
	379
[39]	380	def parseThread( self ):
	381
	382	debug_msg( 8, self.printTime() + ' - parsethread(): started.' )
	383	debug_msg( 8, self.printTime() + ' - parsethread(): Parsing XML..' )
	384	ret = self.myParser.parse( self.myXMLGatherer.getFileObject() )
	385	debug_msg( 8, self.printTime() + ' - parsethread(): Done parsing.' )
	386	debug_msg( 8, self.printTime() + ' - parsethread(): finished.' )
	387
	388	return ret
	389
[9]	390	class GangliaConfigParser:
	391
[34]	392	sources = [ ]
[9]	393
	394	def __init__( self, config ):
[32]	395
[9]	396	self.config = config
	397	self.parseValues()
	398
[32]	399	def parseValues( self ):
[9]	400	"Parse certain values from gmetad.conf"
	401
	402	readcfg = open( self.config, 'r' )
	403
	404	for line in readcfg.readlines():
	405
	406	if line.count( '"' ) > 1:
	407
[10]	408	if line.find( 'data_source' ) != -1 and line[0] != '#':
[9]	409
[11]	410	source = { }
	411	source['name'] = line.split( '"' )[1]
[9]	412	source_words = line.split( '"' )[2].split( ' ' )
	413
	414	for word in source_words:
	415
	416	valid_interval = 1
	417
	418	for letter in word:
[32]	419
[9]	420	if letter not in string.digits:
[32]	421
[9]	422	valid_interval = 0
	423
[10]	424	if valid_interval and len(word) > 0:
[32]	425
[9]	426	source['interval'] = word
[12]	427	debug_msg( 9, 'polling interval for %s = %s' %(source['name'], source['interval'] ) )
[33]	428
	429	# No interval found, use Ganglia's default
	430	if not source.has_key( 'interval' ):
	431	source['interval'] = 15
	432	debug_msg( 9, 'polling interval for %s defaulted to 15' %(source['name']) )
[32]	433
[33]	434	self.sources.append( source )
[9]	435
	436	def getInterval( self, source_name ):
[32]	437
[9]	438	for source in self.sources:
[32]	439
[12]	440	if source['name'] == source_name:
[32]	441
[9]	442	return source['interval']
[32]	443
[9]	444	return None
	445
[34]	446	def getLowestInterval( self ):
	447
	448	lowest_interval = 0
	449
	450	for source in self.sources:
	451
	452	if not lowest_interval or source['interval'] <= lowest_interval:
	453
	454	lowest_interval = source['interval']
	455
	456	# Return 15 when nothing is found, so that the daemon won't go insane with 0 sec delays
	457	if lowest_interval:
	458	return lowest_interval
	459	else:
	460	return 15
	461
[9]	462	class RRDHandler:
	463
[32]	464	myMetrics = { }
[36]	465	slot = None
[32]	466
[33]	467	def __init__( self, config, cluster ):
[9]	468	self.cluster = cluster
[33]	469	self.config = config
[36]	470	self.slot = mutex.mutex()
[37]	471	self.rrdm = RRDMutator()
[9]	472
[32]	473	def getClusterName( self ):
	474	return self.cluster
	475
	476	def memMetric( self, host, metric ):
	477
[34]	478	if self.myMetrics.has_key( host ):
[32]	479
[34]	480	if self.myMetrics[ host ].has_key( metric['name'] ):
[32]	481
[34]	482	for mymetric in self.myMetrics[ host ][ metric['name'] ]:
[32]	483
[34]	484	if mymetric['time'] == metric['time']:
[32]	485
[34]	486	# Allready have this metric, abort
	487	return 1
	488	else:
	489	self.myMetrics[ host ][ metric['name'] ] = [ ]
	490	else:
[32]	491	self.myMetrics[ host ] = { }
[34]	492	self.myMetrics[ host ][ metric['name'] ] = [ ]
[32]	493
[36]	494	self.slot.testandset()
[32]	495	self.myMetrics[ host ][ metric['name'] ].append( metric )
[36]	496	self.slot.unlock()
[32]	497
[37]	498	def makeUpdateList( self, host, metricname ):
	499
	500	update_list = [ ]
	501
	502	for metric in self.myMetrics[ host ][ metricname ]:
	503
	504	update_list.append( '%s:%s' %( metric['time'], metric['val'] ) )
	505
	506	return update_list
	507
[33]	508	def storeMetrics( self ):
	509
	510	for hostname, mymetrics in self.myMetrics.items():
	511
	512	for metricname, mymetric in mymetrics.items():
	513
[35]	514	mytime = self.makeTimeSerial()
	515	self.createCheck( hostname, metricname, mytime )
[36]	516	update_ret = self.update( hostname, metricname, mytime )
[33]	517
[36]	518	if update_ret == 0:
[33]	519
[36]	520	self.slot.testandset()
[33]	521	del self.myMetrics[ hostname ][ metricname ]
[36]	522	self.slot.unlock()
[33]	523	debug_msg( 9, 'stored metric %s for %s' %( hostname, metricname ) )
	524	else:
	525	debug_msg( 9, 'metric update failed' )
[38]	526	return 1
[33]	527
[17]	528	def makeTimeSerial( self ):
[32]	529	"Generate a time serial. Seconds since epoch"
[17]	530
	531	# Seconds since epoch
	532	mytime = int( time.time() )
	533
	534	return mytime
	535
[33]	536	def makeRrdPath( self, host, metricname=None, timeserial=None ):
[32]	537	"""
	538	Make a RRD location/path and filename
	539	If a metric or timeserial are supplied the complete locations
	540	will be made, else just the host directory
	541	"""
[17]	542
[20]	543	if not timeserial:
	544	rrd_dir = '%s/%s/%s' %( check_dir(ARCHIVE_PATH), self.cluster, host )
	545	else:
	546	rrd_dir = '%s/%s/%s/%s' %( check_dir(ARCHIVE_PATH), self.cluster, host, timeserial )
[35]	547	if metricname:
[33]	548	rrd_file = '%s/%s.rrd' %( rrd_dir, metricname )
[20]	549	else:
	550	rrd_file = None
[17]	551
	552	return rrd_dir, rrd_file
	553
[20]	554	def getLastRrdTimeSerial( self, host ):
[32]	555	"""
	556	Find the last timeserial (directory) for this host
	557	This is determined once every host
	558	"""
[17]	559
[20]	560	rrd_dir, rrd_file = self.makeRrdPath( host )
[17]	561
[19]	562	newest_timeserial = 0
	563
[17]	564	if os.path.exists( rrd_dir ):
[32]	565
[17]	566	for root, dirs, files in os.walk( rrd_dir ):
	567
[20]	568	for dir in dirs:
[17]	569
[20]	570	valid_dir = 1
[17]	571
[20]	572	for letter in dir:
	573	if letter not in string.digits:
	574	valid_dir = 0
[17]	575
[20]	576	if valid_dir:
	577	timeserial = dir
[17]	578	if timeserial > newest_timeserial:
	579	newest_timeserial = timeserial
	580
	581	if newest_timeserial:
[18]	582	return newest_timeserial
[17]	583	else:
	584	return 0
	585
[20]	586	def checkNewRrdPeriod( self, host, current_timeserial ):
[32]	587	"""
	588	Check if current timeserial belongs to recent time period
	589	or should become a new period (and file).
[17]	590
[32]	591	Returns the serial of the correct time period
	592	"""
	593
[20]	594	last_timeserial = int( self.getLastRrdTimeSerial( host ) )
[30]	595	debug_msg( 9, 'last timeserial of %s is %s' %( host, last_timeserial ) )
[17]	596
	597	if not last_timeserial:
[18]	598	serial = current_timeserial
	599	else:
[17]	600
[18]	601	archive_secs = ARCHIVE_HOURS_PER_RRD * (60 * 60)
[17]	602
[18]	603	if (current_timeserial - last_timeserial) >= archive_secs:
	604	serial = current_timeserial
	605	else:
	606	serial = last_timeserial
[17]	607
[18]	608	return serial
	609
[33]	610	def getFirstTime( self, host, metricname ):
	611	"Get the first time of a metric we know of"
	612
	613	first_time = 0
	614
	615	for metric in self.myMetrics[ host ][ metricname ]:
	616
	617	if not first_time or metric['time'] <= first_time:
	618
	619	first_time = metric['time']
	620
[35]	621	return first_time
	622
[33]	623	def createCheck( self, host, metricname, timeserial ):
[9]	624	"Check if an .rrd allready exists for this metric, create if not"
	625
[35]	626	debug_msg( 9, 'rrdcreate: using timeserial %s for %s/%s' %( timeserial, host, metricname ) )
[9]	627
[33]	628	rrd_dir, rrd_file = self.makeRrdPath( host, metricname, timeserial )
[17]	629
[9]	630	if not os.path.exists( rrd_dir ):
	631	os.makedirs( rrd_dir )
[14]	632	debug_msg( 9, 'created dir %s' %( str(rrd_dir) ) )
[9]	633
[14]	634	if not os.path.exists( rrd_file ):
[9]	635
[33]	636	interval = self.config.getInterval( self.cluster )
[14]	637	heartbeat = 8 * int(interval)
[9]	638
[37]	639	params = [ ]
[12]	640
[37]	641	params.append( '--step' )
	642	params.append( str( interval ) )
[12]	643
[37]	644	params.append( '--start' )
	645	params.append( str( int( self.getFirstTime( host, metricname ) ) - 1 ) )
[12]	646
[37]	647	params.append( 'DS:sum:GAUGE:%d:U:U' %heartbeat )
	648	params.append( 'RRA:AVERAGE:0.5:1:%s' %(ARCHIVE_HOURS_PER_RRD * 240) )
[13]	649
[37]	650	self.rrdm.create( str(rrd_file), params )
	651
[14]	652	debug_msg( 9, 'created rrd %s' %( str(rrd_file) ) )
	653
[33]	654	def update( self, host, metricname, timeserial ):
[9]	655
[35]	656	debug_msg( 9, 'rrdupdate: using timeserial %s for %s/%s' %( timeserial, host, metricname ) )
[9]	657
[33]	658	rrd_dir, rrd_file = self.makeRrdPath( host, metricname, timeserial )
[18]	659
[37]	660	update_list = self.makeUpdateList( host, metricname )
[15]	661
[38]	662	ret = self.rrdm.update( str(rrd_file), update_list )
[32]	663
[38]	664	if ret:
	665	return 1
[27]	666
[37]	667	debug_msg( 9, 'updated rrd %s with %s' %( str(rrd_file), string.join( update_list ) ) )
[15]	668
[36]	669	return 0
	670
[8]	671	def main():
	672	"Program startup"
	673
	674	myProcessor = GangliaXMLProcessor()
	675
[22]	676	if DAEMONIZE:
	677	myProcessor.daemon()
	678	else:
	679	myProcessor.run()
	680
[9]	681	def check_dir( directory ):
	682	"Check if directory is a proper directory. I.e.: Does _not_ end with a '/'"
	683
	684	if directory[-1] == '/':
	685	directory = directory[:-1]
	686
	687	return directory
	688
[12]	689	def debug_msg( level, msg ):
	690
	691	if (DEBUG_LEVEL >= level):
	692	sys.stderr.write( msg + '\n' )
	693
[5]	694	# Let's go
[9]	695	if __name__ == '__main__':
	696	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: