Context Navigation

source: trunk/email2trac.py.in @ 605

Last change on this file since 605 was 605, checked in by bas, 12 years ago
removed some obsolete code
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 85.2 KB

Rev	Line
[22]	1	#!@PYTHON@
	2	#
	3	# This file is part of the email2trac utils
	4	#
[595]	5	# Copyright 2002 SARA
[22]	6	#
[595]	7	# Licensed under the Apache License, Version 2.0 (the "License");
	8	# you may not use this file except in compliance with the License.
	9	# You may obtain a copy of the License at
[22]	10	#
[595]	11	# http://www.apache.org/licenses/LICENSE-2.0
[22]	12	#
[595]	13	# Unless required by applicable law or agreed to in writing, software
	14	# distributed under the License is distributed on an "AS IS" BASIS,
	15	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	16	# See the License for the specific language governing permissions and
	17	# limitations under the License.
	18	#
[80]	19	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
	20	#
[22]	21	"""
[547]	22	email2trac.py -- Email -> TRAC tickets
[22]	23
[547]	24	A MTA filter to create Trac tickets from inbound emails.
[22]	25
[547]	26	first proof of concept from:
	27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
	28	Copyright 2005, Edgewall Software
[22]	29
[282]	30	Authors:
	31	Bas van der Vlies <basv@sara.nl>
	32	Walter de Jong <walter@sara.nl>
[22]	33
	34	How to use
	35	----------
[282]	36	* See https://subtrac.sara.nl/oss/email2trac/
	37
[22]	38	* Commandline opions:
[205]	39	-h,--help
[507]	40	-d, --debug
[205]	41	-f,--file <configuration file>
[410]	42	-n,--dry-run
[205]	43	-p, --project <project name>
[415]	44	-t, --ticket_prefix <name>
[22]	45
	46	SVN Info:
	47	$Id: email2trac.py.in 605 2012-08-30 11:59:29Z bas $
	48	"""
	49	import os
	50	import sys
	51	import string
	52	import getopt
	53	import time
	54	import email
[136]	55	import email.Iterators
	56	import email.Header
[22]	57	import re
	58	import urllib
	59	import unicodedata
	60	import mimetypes
[96]	61	import traceback
[403]	62	import logging
	63	import logging.handlers
[404]	64	import UserDict
[498]	65	import tempfile
[433]	66
[520]	67	from datetime import timedelta, datetime
[433]	68	from stat import *
[22]	69
[593]	70	try:
	71	from trac import __version__ as trac_version
	72	from trac import config as trac_config
[403]	73
[593]	74	except ImportError, detail:
	75	print "Can not find a a valid trac installation, try setting PYTHONPATH"
	76	sys.exit(1)
[91]	77
[508]	78	## Some global variables
[96]	79	#
	80	m = None
[22]	81
[508]	82	# This is to for the function AlwaysNotifyReporter
	83	sender_email = None
	84
[404]	85	class SaraDict(UserDict.UserDict):
[507]	86	def __init__(self, dictin = None):
	87	UserDict.UserDict.__init__(self)
	88	self.name = None
	89
	90	if dictin:
	91	if dictin.has_key('name'):
	92	self.name = dictin['name']
	93	del dictin['name']
	94	self.data = dictin
	95
	96	def get_value(self, name):
	97	if self.has_key(name):
	98	return self[name]
	99	else:
	100	return None
	101
	102	def __repr__(self):
	103	return repr(self.data)
[182]	104
[507]	105	def __str__(self):
	106	return str(self.data)
	107
	108	def __getattr__(self, name):
	109	"""
	110	override the class attribute get method. Return the value
	111	from the dictionary
	112	"""
	113	if self.data.has_key(name):
	114	return self.data[name]
	115	else:
	116	return None
	117
	118	def __setattr__(self, name, value):
	119	"""
	120	override the class attribute set method only when the UserDict
	121	has set its class attribute
	122	"""
	123	if self.__dict__.has_key('data'):
	124	self.data[name] = value
	125	else:
	126	self.__dict__[name] = value
[404]	127
[507]	128	def __iter__(self):
	129	return iter(self.data.keys())
[404]	130
[22]	131	class TicketEmailParser(object):
[507]	132	env = None
	133	comment = '> '
[359]	134
[507]	135	def __init__(self, env, parameters, logger, version):
	136	self.env = env
[22]	137
[507]	138	# Database connection
	139	#
	140	self.db = None
[22]	141
[507]	142	# Save parameters
	143	#
	144	self.parameters = parameters
	145	self.logger = logger
[206]	146
[507]	147	# Some useful mail constants
	148	#
	149	self.email_name = None
	150	self.email_addr = None
	151	self.email_from = None
	152	self.author = None
	153	self.id = None
[587]	154
	155	## Will be set to True if the user has ben registered
	156	#
	157	self.allow_registered_user = False
[507]	158
	159	self.STRIP_CONTENT_TYPES = list()
[72]	160
[507]	161	## fields properties via body_text
	162	#
	163	self.properties = dict()
[452]	164
[507]	165	self.VERSION = version
[402]	166
[507]	167	self.get_config = self.env.config.get
[404]	168
[507]	169	## init function ##
	170	#
	171	self.setup_parameters()
[402]	172
[507]	173	def setup_parameters(self):
	174	if self.parameters.umask:
	175	os.umask(self.parameters.umask)
[402]	176
[507]	177	if not self.parameters.spam_level:
	178	self.parameters.spam_level = 0
[22]	179
[507]	180	if not self.parameters.spam_header:
	181	self.parameters.spam_header = 'X-Spam-Score'
[207]	182
[507]	183	if not self.parameters.email_quote:
	184	self.parameters.email_quote = '> '
[22]	185
[507]	186	if not self.parameters.ticket_update_by_subject_lookback:
	187	self.parameters.ticket_update_by_subject_lookback = 30
[360]	188
[507]	189	if self.parameters.verbatim_format == None:
	190	self.parameters.verbatim_format = 1
[118]	191
[507]	192	if self.parameters.reflow == None:
	193	self.parameters.reflow = 1
[231]	194
[507]	195	if self.parameters.binhex:
	196	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
[238]	197
[507]	198	if self.parameters.applesingle:
	199	self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]	200
[507]	201	if self.parameters.appledouble:
	202	self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]	203
[507]	204	if self.parameters.strip_content_types:
	205	items = self.parameters.strip_content_types.split(',')
	206	for item in items:
	207	self.STRIP_CONTENT_TYPES.append(item.strip())
[294]	208
[507]	209	if self.parameters.tmpdir:
	210	self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
	211	else:
	212	self.parameters.tmpdir = os.path.normcase('/tmp')
[257]	213
[507]	214	if self.parameters.email_triggers_workflow == None:
	215	self.parameters.email_triggers_workflow = 1
[191]	216
[507]	217	if not self.parameters.subject_field_separator:
	218	self.parameters.subject_field_separator = '&'
	219	else:
	220	self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
[297]	221
[507]	222	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
	223	self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
[305]	224
[484]	225
[507]	226	self.system = None
[359]	227
[341]	228	########## Email Header Functions ###########################################################
[339]	229
[507]	230	def spam(self, message):
	231	"""
	232	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
	233	# Note if Spam_level then '*' are included
	234	"""
[580]	235	self.logger.debug('function spam')
	236
[507]	237	spam = False
	238	if message.has_key(self.parameters.spam_header):
	239	spam_l = string.split(message[self.parameters.spam_header])
[578]	240	#self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
[22]	241
[507]	242	try:
	243	number = spam_l[0].count('*')
	244	except IndexError, detail:
	245	number = 0
	246
	247	if number >= self.parameters.spam_level:
	248	spam = True
	249
[580]	250	## treat virus mails as spam
[507]	251	#
	252	elif message.has_key('X-Virus-found'):
	253	spam = True
[194]	254
[580]	255	## Astaro firewall spam handling
[507]	256	#
[580]	257	elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
	258	spam = True
	259
	260	## How to handle SPAM messages
	261	#
[507]	262	if self.parameters.drop_spam and spam:
[194]	263
[507]	264	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
	265	return 'drop'
[194]	266
[507]	267	elif spam:
[194]	268
[507]	269	return 'Spam'
	270	else:
[22]	271
[507]	272	return False
[191]	273
[572]	274	def email_header_acl(self, keyword, message_field, default):
[507]	275	"""
	276	This function wil check if the email address is allowed or denied
	277	to send mail to the ticket list
	278	"""
	279	self.logger.debug('function email_header_acl: %s' %keyword)
[409]	280
[507]	281	try:
	282	mail_addresses = self.parameters[keyword]
[221]	283
[507]	284	# Check if we have an empty string
	285	#
	286	if not mail_addresses:
	287	return default
[221]	288
[507]	289	except KeyError, detail:
[572]	290	self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
[206]	291
[507]	292	return default
[206]	293
[507]	294	mail_addresses = string.split(mail_addresses, ',')
[572]	295	message_addresses = string.split(message_field, ',')
[221]	296
[507]	297	for entry in mail_addresses:
	298	entry = entry.strip()
	299	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
[149]	300
[572]	301	for addr in message_addresses:
	302	addr = addr.strip()
	303
	304	if self.parameters.compare_function_list in [ 'matches', 'match']:
	305	s = '\t%s matches %s' %(addr, entry)
	306	result = TO_RE.match(addr)
	307	else:
	308	s = '\t%s contains %s' %(addr, entry)
	309	result = TO_RE.search(addr)
	310
	311	if result:
	312	self.logger.debug(s)
	313	return True
	314
[507]	315	return False
[208]	316
[507]	317	def email_header_txt(self, m):
	318	"""
	319	Display To and CC addresses in description field
	320	"""
	321	s = ''
[334]	322
[507]	323	if m['To'] and len(m['To']) > 0:
	324	s = "'''To:''' %s\r\n" %(m['To'])
	325	if m['Cc'] and len(m['Cc']) > 0:
	326	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
[22]	327
[507]	328	return self.email_to_unicode(s)
[22]	329
[138]	330
[507]	331	def get_sender_info(self, message):
	332	"""
	333	Get the default author name and email address from the message
	334	"""
[576]	335	self.logger.debug('function get_sender_info')
[43]	336
[570]	337	to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
	338	self.email_to_addrs = list()
[226]	339
[570]	340	for n,e in to_addrs:
	341	self.email_to_addrs.append(e)
	342
	343	self.email_to_addrs = ','.join(self.email_to_addrs)
	344
	345
[576]	346	self.email_name, self.email_addr = email.Utils.parseaddr(message['from'])
[142]	347
[576]	348	## decode email name can contain charset
	349	#
	350	self.email_name = self.email_to_unicode(self.email_name)
	351
	352	dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr)
	353	self.logger.debug(dstr)
	354
[507]	355	## Trac can not handle author's name that contains spaces
	356	#
[518]	357	if self.email_addr.lower() == self.trac_smtp_from.lower():
[507]	358	if self.email_name:
	359	self.author = self.email_name
	360	else:
	361	self.author = "email2trac"
	362	else:
	363	self.author = self.email_addr
[304]	364
[507]	365	if self.parameters.ignore_trac_user_settings:
	366	return
[194]	367
[507]	368	# Is this a registered user, use email address as search key:
	369	# result:
	370	# u : login name
	371	# n : Name that the user has set in the settings tab
	372	# e : email address that the user has set in the settings tab
	373	#
	374	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
	375	if (
	376	(e and (e.lower() == self.email_addr.lower())) or
[518]	377	(u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
[507]	378	)
	379	]
[43]	380
[507]	381	if len(users) >= 1:
[587]	382	self.email_from = users[0][2]
[507]	383	self.author = users[0][0]
[45]	384
[590]	385	self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
	386
[587]	387	if self.parameters.white_list_registered_users:
[590]	388	self.logger.debug('\tget_sender_info: added registered user to white list')
[587]	389	self.allow_registered_user = True
	390
	391
[597]	392	def set_cc_fields(self, ticket, message, update=False):
[507]	393	"""
	394	Set all the right fields for a new ticket
	395	"""
[597]	396	self.logger.debug('function set_cc_fields')
[72]	397
[270]	398
[507]	399	# Put all CC-addresses in ticket CC field
	400	#
	401	if self.parameters.reply_all:
[43]	402
[507]	403	email_cc = ''
[299]	404
[507]	405	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
[299]	406
[507]	407	if not cc_addrs:
	408	return
[43]	409
[507]	410	## Build a list of forbidden CC addresses
	411	#
	412	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
	413	#to_list = list()
	414	#for n,e in to_addrs:
	415	# to_list.append(e)
[43]	416
[597]	417	if update:
	418	self.logger.debug("\tupdate cc-fields")
	419
[507]	420	for name,addr in cc_addrs:
	421
	422	## Prevent mail loop
	423	#
	424	if addr == self.trac_smtp_from:
[597]	425	self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
[507]	426	continue
[43]	427
[597]	428	## Alwyas remove reporter email address
	429	#
	430	elif addr == self.email_addr:
	431	self.logger.debug("\tSkipping reporter email address for CC-field")
	432	continue
	433
[507]	434	else:
[597]	435	if email_cc:
	436	email_cc = '%s, %s' %(email_cc, addr)
	437	else:
	438	email_cc = addr
[96]	439
[507]	440	if email_cc:
[597]	441	self.logger.debug('\tCC fields: %s' %email_cc)
[507]	442	ticket['cc'] = self.email_to_unicode(email_cc)
[299]	443
[339]	444
[588]	445	def acl_list_from_file(self, f, keyword):
	446	"""
	447	Read the email address from a file
	448	"""
	449	self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
	450
	451	if not os.path.isfile(f):
	452	self.logger.error('%s_file: %s does not exists' %(keyword, f) )
	453	else:
	454	## read whole file and replace '\n' with ''
	455	#
	456	addr_l = open(f, 'r').readlines()
	457	s = ','.join(addr_l).replace('\n','')
	458
	459	try:
	460	self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
	461	except KeyError, detail:
	462	self.parameters[keyword] = s
	463
[339]	464	########## DEBUG functions ###########################################################
	465
[555]	466	def debug_body(self, message_body, temp_file):
	467	"""
	468	"""
	469	self.logger.debug('function debug_body:')
[310]	470
[555]	471	body_file = "%s.body" %(temp_file)
	472	fx = open(body_file, 'wb')
	473
[507]	474	if self.parameters.dry_run:
[561]	475	self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
[507]	476	return
[331]	477
[555]	478	self.logger.debug('writing body to %s' %(body_file))
[507]	479	if not message_body:
	480	message_body = '(None)'
[310]	481
[507]	482	message_body = message_body.encode('utf-8')
[310]	483
[507]	484	fx.write(message_body)
	485	fx.close()
	486	try:
	487	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
	488	except OSError:
	489	pass
[310]	490
[555]	491	def debug_attachments(self, message_parts, temp_file):
[507]	492	"""
	493	"""
	494	self.logger.debug('function debug_attachments')
	495
	496	n = 0
	497	for item in message_parts:
[553]	498
	499	## Skip inline text parts
	500	#
[507]	501	if not isinstance(item, tuple):
	502	continue
	503
	504	(original, filename, part) = item
[555]	505	self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
[310]	506
[555]	507	dummy_filename, ext = os.path.splitext(filename)
	508
[507]	509	n = n + 1
[555]	510	part_name = 'part%d%s' %(n, ext)
	511	part_file = "%s.%s" %(temp_file, part_name)
	512
	513	s = 'writing %s: filename: %s' %(part_file, filename)
[507]	514	self.print_unicode(s)
[555]	515
	516	## Forbidden chars, just write part files instead of names
[507]	517	#
[555]	518	#filename = filename.replace('\\', '_')
	519	#filename = filename.replace('/', '_')
	520	#filename = filename + '.att_email2trac'
	521	# part_file = os.path.join(self.parameters.tmpdir, filename)
	522	#part_file = util.text.unicode_quote(part_file)
	523	#self.print_unicode(part_file)
[331]	524
[507]	525	if self.parameters.dry_run:
[561]	526	self.logger.info('DRY_RUN: NOT saving attachments')
[507]	527	continue
[331]	528
[507]	529	fx = open(part_file, 'wb')
	530	text = part.get_payload(decode=1)
[331]	531
[507]	532	if not text:
	533	text = '(None)'
[331]	534
[507]	535	fx.write(text)
	536	fx.close()
[331]	537
[507]	538	try:
	539	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
	540	except OSError:
	541	pass
[310]	542
[528]	543	def save_email_for_debug(self, message, project_name, create_tempfile=False):
[309]	544
[528]	545	if self.parameters.dry_run:
[555]	546	self.logger.debug('DRY_RUN: NOT saving email message')
[528]	547	return
	548
[507]	549	if create_tempfile:
[555]	550	(fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
[514]	551	fx = os.fdopen(fd, 'wb')
[507]	552	else:
[555]	553	tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
	554	fx = open(tmp_file, 'wb')
[173]	555
[44]	556
[555]	557	self.logger.debug('saving email to %s' %(tmp_file))
[528]	558	fx.write('%s' % message)
	559	fx.close()
	560
	561	try:
[555]	562	os.chmod(tmp_file, S_IRWXU\|S_IRWXG\|S_IRWXO)
[528]	563	except OSError:
	564	pass
[331]	565
[507]	566	message_parts = self.get_message_parts(message)
	567	message_parts = self.unique_attachment_names(message_parts)
	568	body_text = self.get_body_text(message_parts)
[555]	569	self.debug_body(body_text, tmp_file)
	570	self.debug_attachments(message_parts, tmp_file)
[309]	571
[339]	572	########## Conversion functions ###########################################################
	573
[507]	574	def email_to_unicode(self, message_str):
	575	"""
	576	Email has 7 bit ASCII code, convert it to unicode with the charset
	577	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
	578	understands it.
	579	"""
	580	self.logger.debug("function email_to_unicode")
[341]	581
[507]	582	results = email.Header.decode_header(message_str)
[341]	583
[507]	584	s = None
	585	for text,format in results:
	586	if format:
	587	try:
	588	temp = unicode(text, format)
	589	except UnicodeError, detail:
	590	# This always works
	591	#
[525]	592	msg = 'ERROR: Could not find charset: %s, please install' %format
	593	self.logger.error(msg)
[507]	594	temp = unicode(text, 'iso-8859-15')
	595	except LookupError, detail:
[514]	596	msg = 'ERROR: Could not find charset: %s, please install' %format
[525]	597	self.logger.error(msg)
[507]	598	#temp = unicode(text, 'iso-8859-15')
	599	temp = message_str
	600
	601	else:
	602	temp = string.strip(text)
	603	temp = unicode(text, 'iso-8859-15')
[341]	604
[507]	605	if s:
	606	s = '%s %s' %(s, temp)
	607	else:
	608	s = '%s' %temp
[341]	609
[507]	610	return s
[341]	611
[507]	612	def str_to_dict(self, s):
	613	"""
	614	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
	615	"""
	616	self.logger.debug("function str_to_dict")
[164]	617
[507]	618	fields = string.split(s, self.parameters.subject_field_separator)
[262]	619
[507]	620	result = dict()
	621	for field in fields:
	622	try:
	623	index, value = string.split(field, '=')
[169]	624
[507]	625	# We can not change the description of a ticket via the subject
	626	# line. The description is the body of the email
	627	#
	628	if index.lower() in ['description']:
	629	continue
[169]	630
[507]	631	if value:
	632	result[index.lower()] = value
[169]	633
[507]	634	except ValueError:
	635	pass
	636	return result
[167]	637
[507]	638	def print_unicode(self,s):
	639	"""
	640	This function prints unicode strings if possible else it will quote it
	641	"""
	642	try:
	643	self.logger.debug(s)
	644	except UnicodeEncodeError, detail:
	645	self.logger.debug(util.text.unicode_quote(s))
[379]	646
[503]	647
[507]	648	def html_2_txt(self, data):
	649	"""
	650	Various routines to convert html syntax to valid trac wiki syntax
	651	"""
	652	self.logger.debug('function html_2_txt')
[503]	653
[507]	654	## This routine make an safe html that can be include
	655	# in trac, but no further text processing can be done
	656	#
	657	# try:
	658	# from lxml.html.clean import Cleaner
	659	# tags_rm = list()
	660	# tags_rm.append('body')
[503]	661	#
[507]	662	# cleaner = Cleaner(remove_tags=tags_rm )
	663	# parsed_data = cleaner.clean_html(data)
	664	# parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
[503]	665	#
[507]	666	# return parsed_data
	667	#
	668	# except ImportError::
	669	# pass
[503]	670
[507]	671	parsed_data = None
	672	if self.parameters.html2text_cmd:
[514]	673	(fd, tmp_file) = tempfile.mkstemp('email2trac.html')
	674	f = os.fdopen(fd, 'w')
	675
[507]	676	cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
	677	self.logger.debug('\t html2text conversion %s'%(cmd))
	678
	679	if self.parameters.dry_run:
[561]	680	self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
[503]	681
[507]	682	else:
	683	f.write(data)
	684	f.close()
[503]	685
[507]	686	lines = os.popen(cmd).readlines()
	687	parsed_data = ''.join(lines)
[503]	688
[507]	689	os.unlink(tmp_file)
[503]	690
[507]	691	else:
	692	self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
[503]	693
[507]	694	return parsed_data
[503]	695
[549]	696	def check_filename_length(self, name):
	697	"""
[551]	698	To bypass a bug in Trac
	699	check if the filename lenght is not larger then OS limit.
[549]	700	yes : return truncated filename
	701	no : return unmodified filename
	702	"""
	703	self.logger.debug('function check_filename_length: ')
[550]	704
	705	if not name:
	706	return 'None'
[549]	707
	708	dummy_filename, ext = os.path.splitext(name)
	709
	710	## Trac uses this format
	711	#
	712	quote_format = util.text.unicode_quote(dummy_filename)
	713
	714	## Determine max filename lenght
	715	#
[603]	716	try:
	717	filemax_length = os.pathconf('/', 'PC_NAME_MAX')
	718	except AttributeError, detail:
	719	filemax_lenght = 240
[549]	720
	721	if len(quote_format) <= filemax_length:
	722	return name
	723
	724	else:
	725	## Truncate file to filemax_length and reserve room for extension
	726	# We must break on a boundry
	727	#
	728	length = filemax_length - 6
	729
	730	for i in range(0,10):
	731
	732	truncated = quote_format[ : (length - i)]
	733
	734	try:
	735	unqoute_truncated = util.text.unicode_unquote(truncated)
	736	unqoute_truncated = unqoute_truncated + ext
	737
	738	self.print_unicode('\t ' + unqoute_truncated)
	739
	740	break
	741
	742	except UnicodeDecodeError, detail:
	743	continue
	744
	745
	746	return unqoute_truncated
	747
[339]	748	########## TRAC ticket functions ###########################################################
	749
[546]	750	def mail_workflow(self, tkt):
[545]	751	"""
	752	"""
[546]	753	self.logger.debug('function mail_workflow: ')
[545]	754
	755	req = Mock(authname=self.author, perm=MockPerm(), args={})
	756	ticket_system = TicketSystem(self.env)
	757
	758	try:
	759	workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
	760
	761	except KeyError:
[553]	762	## fallback for compability (Will be deprecated)
	763	# workflow can be none.
[545]	764	#
	765	workflow = None
	766	if tkt['status'] in ['closed']:
	767	workflow = self.parameters.workflow
	768
	769	if workflow:
[556]	770
[557]	771	## process all workflow implementations
	772	#
[556]	773	tkt_module = TicketModule(self.env)
	774	field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
	775
	776	for field in field_changes.keys():
	777
	778	## We have already processed these fields
	779	#
[558]	780	if not field in ['summary', 'description']:
[556]	781	s = 'workflow : %s, field %s : %s, by %s' \
	782	%(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
	783	self.logger.debug(s)
	784
	785	tkt[field] = field_changes[field]['new']
	786
[545]	787	return True
	788
	789	else:
	790	return False
	791
[507]	792	def check_permission_participants(self, tkt, action):
	793	"""
	794	Check if the mailer is allowed to update the ticket
	795	"""
[538]	796	self.logger.debug('function check_permission_participants %s')
[398]	797
[518]	798	if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
[507]	799	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
[408]	800
[507]	801	return True
[398]	802
[507]	803	perm = PermissionSystem(self.env)
	804	if perm.check_permission(action, self.author):
	805	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
[408]	806
[507]	807	return True
	808
	809	# Is the updater in the CC?
	810	try:
	811	cc_list = tkt['cc'].split(',')
	812	for cc in cc_list:
[518]	813	if self.email_addr.lower() in cc.lower.strip():
[507]	814	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
[398]	815
[507]	816	return True
[398]	817
[507]	818	except KeyError:
	819	pass
[398]	820
[507]	821	return False
[474]	822
[507]	823	def check_permission(self, tkt, action):
	824	"""
	825	check if the reporter has the right permission for the action:
[397]	826	- TICKET_CREATE
	827	- TICKET_MODIFY
[474]	828	- TICKET_APPEND
	829	- TICKET_CHGPROP
[397]	830
[507]	831	There are three models:
	832	- None : no checking at all
	833	- trac : check the permission via trac permission model
	834	- email2trac: ....
	835	"""
[538]	836	self.logger.debug("function check_permission: %s" %(action))
[397]	837
[507]	838	if self.parameters.ticket_permission_system in ['trac']:
[398]	839
[539]	840	perm = PermissionCache(self.env, self.author)
	841	if perm.has_permission(action):
[507]	842	return True
	843	else:
	844	return False
[398]	845
[507]	846	elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
	847	return (self.check_permission_participants(tkt, action))
[398]	848
[507]	849	## Default is to allow everybody ticket updates and ticket creation
	850	#
	851	else:
	852	return True
[397]	853
	854
[507]	855	def update_ticket_fields(self, ticket, user_dict, new=None):
	856	"""
	857	This will update the ticket fields. It will check if the
	858	given fields are known and if the right values are specified
	859	It will only update the ticket field value:
	860	- If the field is known
	861	- If the value supplied is valid for the ticket field.
	862	If not then there are two options:
	863	1) Skip the value (new=None)
	864	2) Set default value for field (new=1)
	865	"""
	866	self.logger.debug("function update_ticket_fields")
[169]	867
[507]	868	## Check only permission model on ticket updates
	869	#
	870	if not new:
	871	if self.parameters.ticket_permission_system:
	872	if not self.check_permission(ticket, 'TICKET_CHGPROP'):
	873	self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
	874	return False
[474]	875
[507]	876	## Build a system dictionary from the ticket fields
	877	# with field as index and option as value
	878	#
	879	sys_dict = dict()
[530]	880
[507]	881	for field in ticket.fields:
[530]	882
[507]	883	try:
	884	sys_dict[field['name']] = field['options']
[169]	885
[507]	886	except KeyError:
[531]	887	sys_dict[field['name']] = None
[507]	888	pass
[169]	889
[507]	890	## Check user supplied fields an compare them with the
	891	# system one's
	892	#
[549]	893	for field,value in user_dict.items():
	894
	895	s = 'user_field\t %s = %s' %(field,value)
	896	self.print_unicode(s)
[524]	897
[531]	898	if not field in sys_dict.keys():
[524]	899	self.logger.debug('%s is not a valid field for tickets' %(field))
	900	continue
	901
[507]	902	## To prevent mail loop
	903	#
	904	if field == 'cc':
[301]	905
[507]	906	cc_list = user_dict['cc'].split(',')
[301]	907
[507]	908	if self.trac_smtp_from in cc_list:
	909	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
[408]	910
[507]	911	cc_list.remove(self.trac_smtp_from)
[301]	912
[507]	913	value = ','.join(cc_list)
	914
[301]	915
[507]	916	## Check if every value is allowed for this field
	917	#
[531]	918	if sys_dict[field]:
[169]	919
[507]	920	if value in sys_dict[field]:
	921	ticket[field] = value
	922	else:
	923	## Must we set a default if value is not allowed
	924	#
	925	if new:
	926	value = self.get_config('ticket', 'default_%s' %(field) )
[169]	927
[507]	928	else:
[553]	929
[507]	930	ticket[field] = value
[202]	931
[549]	932	s = 'ticket_field\t %s = %s' %(field, ticket[field])
	933	self.print_unicode(s)
[345]	934
[507]	935	def ticket_update(self, m, id, spam):
	936	"""
	937	If the current email is a reply to an existing ticket, this function
	938	will append the contents of this email to that ticket, instead of
	939	creating a new one.
	940	"""
	941	self.logger.debug("function ticket_update")
[202]	942
[507]	943	if not self.parameters.ticket_update:
	944	self.logger.debug("ticket_update disabled")
	945	return False
[456]	946
[507]	947	## Must we update ticket fields
	948	#
	949	update_fields = dict()
	950	try:
	951	id, keywords = string.split(id, '?')
[262]	952
[507]	953	update_fields = self.str_to_dict(keywords)
[165]	954
[507]	955	## Strip '#'
	956	#
	957	self.id = int(id[1:])
[165]	958
[507]	959	except ValueError:
[390]	960
[507]	961	## Strip '#'
	962	#
	963	self.id = int(id[1:])
[164]	964
[597]	965	self.logger.debug("\tticket id: %s" %id)
[71]	966
[507]	967	## When is the change committed
	968	#
	969	if self.VERSION < 0.11:
	970	when = int(time.time())
	971	else:
	972	when = datetime.now(util.datefmt.utc)
[77]	973
[507]	974	try:
	975	tkt = Ticket(self.env, self.id, self.db)
[390]	976
[507]	977	except util.TracError, detail:
[390]	978
[507]	979	## Not a valid ticket
	980	#
[604]	981	self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
[507]	982	self.id = None
	983	return False
[126]	984
[507]	985	## Check the permission of the reporter
	986	#
	987	if self.parameters.ticket_permission_system:
	988	if not self.check_permission(tkt, 'TICKET_APPEND'):
	989	self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
	990	return False
[398]	991
[507]	992	## How many changes has this ticket
	993	#
	994	grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
	995	cnum = sum(1 for e in grouped) + 1
[288]	996
[486]	997
[507]	998	## reopen the ticket if it is was closed
	999	# We must use the ticket workflow framework
	1000	#
	1001	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[257]	1002
[546]	1003	if not self.mail_workflow(tkt):
[538]	1004
[507]	1005	if tkt['status'] in ['closed']:
[526]	1006	tkt['status'] = 'reopened'
	1007	tkt['resolution'] = ''
	1008
[507]	1009	## Old pre 0.11 situation
	1010	#
	1011	elif self.parameters.email_triggers_workflow:
[470]	1012
[597]	1013	self.logger.debug('\temail triggers workflow pre trac 0.11')
[470]	1014
[507]	1015	if tkt['status'] in ['closed']:
	1016	tkt['status'] = 'reopened'
	1017	tkt['resolution'] = ''
[257]	1018
[507]	1019	else:
[597]	1020	self.logger.debug('\temail triggers workflow disabled')
[469]	1021
[507]	1022	## Must we update some ticket fields properties via subject line
	1023	#
	1024	if update_fields:
	1025	self.update_ticket_fields(tkt, update_fields)
[166]	1026
[597]	1027
[507]	1028	message_parts = self.get_message_parts(m)
	1029	message_parts = self.unique_attachment_names(message_parts)
[210]	1030
[597]	1031	## Must we update some ticket fields properties via inline comments
	1032	# in body_text
[507]	1033	#
	1034	if self.properties:
	1035	self.update_ticket_fields(tkt, self.properties)
[309]	1036
[597]	1037	## Must we update the CC ticket field
	1038	#
	1039	self.set_cc_fields(tkt, m, update=True)
	1040
[507]	1041	if self.parameters.email_header:
	1042	message_parts.insert(0, self.email_header_txt(m))
[76]	1043
[507]	1044	body_text = self.get_body_text(message_parts)
[236]	1045
[507]	1046	error_with_attachments = self.attach_attachments(message_parts)
[348]	1047
[507]	1048	if body_text.strip() or update_fields or self.properties:
[543]	1049
[507]	1050	if self.parameters.dry_run:
[561]	1051	s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
	1052	self.logger.info(s)
[543]	1053
[507]	1054	else:
	1055	if error_with_attachments:
	1056	body_text = '%s\\%s' %(error_with_attachments, body_text)
[543]	1057
[597]	1058	self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
[507]	1059	tkt.save_changes(self.author, body_text, when, None, str(cnum))
	1060
	1061	if not spam:
	1062	self.notify(tkt, False, when)
[72]	1063
[507]	1064	return True
[71]	1065
[507]	1066	def set_ticket_fields(self, ticket):
	1067	"""
	1068	set the ticket fields to value specified
	1069	- /etc/email2trac.conf with <prefix>_<field>
	1070	- trac default values, trac.ini
	1071	"""
	1072	self.logger.debug('function set_ticket_fields')
[410]	1073
[507]	1074	user_dict = dict()
[202]	1075
[507]	1076	for field in ticket.fields:
[202]	1077
[507]	1078	name = field['name']
[202]	1079
[507]	1080	## default trac value
	1081	#
	1082	if not field.get('custom'):
	1083	value = self.get_config('ticket', 'default_%s' %(name) )
[599]	1084
	1085	## skip this field can only be set by email2trac.conf
	1086	#
	1087	if name in ['resolution']:
[507]	1088	value = None
[599]	1089
[507]	1090	else:
	1091	## Else get the default value for reporter
	1092	#
	1093	value = field.get('value')
	1094	options = field.get('options')
[345]	1095
[507]	1096	if value and options and (value not in options):
	1097	value = options[int(value)]
	1098
[549]	1099	s = 'trac[%s] = %s' %(name, value)
	1100	self.print_unicode(s)
[202]	1101
[507]	1102	## email2trac.conf settings
	1103	#
	1104	prefix = self.parameters.ticket_prefix
	1105	try:
	1106	value = self.parameters['%s_%s' %(prefix, name)]
[202]	1107
[549]	1108	s = 'email2trac[%s] = %s ' %(name, value)
	1109	self.print_unicode(s)
	1110
[507]	1111	except KeyError, detail:
	1112	pass
	1113
	1114	if value:
	1115	user_dict[name] = value
[202]	1116
[550]	1117	s = 'used %s = %s' %(name, value)
	1118	self.print_unicode(s)
	1119
[507]	1120	self.update_ticket_fields(ticket, user_dict, new=1)
[202]	1121
[507]	1122	if 'status' not in user_dict.keys():
	1123	ticket['status'] = 'new'
[202]	1124
	1125
[507]	1126	def ticket_update_by_subject(self, subject):
	1127	"""
	1128	This list of Re: prefixes is probably incomplete. Taken from
	1129	wikipedia. Here is how the subject is matched
	1130	- Re: <subject>
	1131	- Re: (<Mail list label>:)+ <subject>
[202]	1132
[507]	1133	So we must have the last column
	1134	"""
	1135	self.logger.debug('function ticket_update_by_subject')
[356]	1136
[533]	1137	found_id = None
[507]	1138	if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
	1139
	1140	SUBJECT_RE = re.compile(r'^(?:(?:RE\|AW\|VS\|SV\|FW\|FWD):\s)+(.)', re.IGNORECASE)
	1141	result = SUBJECT_RE.search(subject)
[356]	1142
[507]	1143	if result:
	1144	## This is a reply
	1145	#
	1146	orig_subject = result.group(1)
[356]	1147
[507]	1148	self.logger.debug('subject search string: %s' %(orig_subject))
[356]	1149
[507]	1150	cursor = self.db.cursor()
	1151	summaries = [orig_subject, '%%: %s' % orig_subject]
[356]	1152
[520]	1153	## Time resolution is in micoseconds
[507]	1154	#
[520]	1155	search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
[356]	1156
[529]	1157	if self.VERSION < 0.12:
	1158	lookback = util.datefmt.to_timestamp(search_date)
	1159	else:
	1160	lookback = util.datefmt.to_utimestamp(search_date)
	1161
[507]	1162	for summary in summaries:
	1163	self.logger.debug('Looking for summary matching: "%s"' % summary)
[408]	1164
[533]	1165	sql = """SELECT id, reporter FROM ticket
[507]	1166	WHERE changetime >= %s AND summary LIKE %s
	1167	ORDER BY changetime DESC"""
[533]	1168
[507]	1169	cursor.execute(sql, [lookback, summary.strip()])
[356]	1170
[507]	1171	for row in cursor:
[408]	1172
[533]	1173	(matched_id, sql_reporter) = row
[408]	1174
[533]	1175	## Save first entry.
	1176	#
	1177	if not found_id:
	1178	found_id = matched_id
	1179
	1180	## If subject and reporter are the same. The we certainly have found the right ticket
	1181	#
	1182	if sql_reporter == self.author:
	1183	self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
	1184	found_id = matched_id
	1185	break
[356]	1186
[533]	1187	if found_id:
	1188	self.logger.debug('Found matching ticket id: %d' % found_id)
	1189	found_id = '#%d' % found_id
	1190	return (found_id, orig_subject)
	1191
[553]	1192	# obsolete !?? 12 Aug 2011
[533]	1193	#subject = orig_subject
[356]	1194
[533]	1195	return (found_id, subject)
[356]	1196
	1197
[507]	1198	def new_ticket(self, msg, subject, spam, set_fields = None):
	1199	"""
	1200	Create a new ticket
	1201	"""
	1202	self.logger.debug('function new_ticket')
[250]	1203
[507]	1204	tkt = Ticket(self.env)
[326]	1205
[597]	1206	## self.author can be email address of an username
	1207	#
	1208	tkt['reporter'] = self.author
[326]	1209
[597]	1210	self.set_cc_fields(tkt, msg)
	1211
[507]	1212	self.set_ticket_fields(tkt)
[202]	1213
[507]	1214	## Check the permission of the reporter
	1215	#
	1216	if self.parameters.ticket_permission_system:
	1217	if not self.check_permission(tkt, 'TICKET_CREATE'):
	1218	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
	1219	return False
[397]	1220
[507]	1221	## Old style setting for component, will be removed
	1222	#
	1223	if spam:
	1224	tkt['component'] = 'Spam'
[204]	1225
[507]	1226	elif self.parameters.has_key('component'):
	1227	tkt['component'] = self.parameters['component']
[201]	1228
[507]	1229	if not msg['Subject']:
	1230	tkt['summary'] = u'(No subject)'
	1231	else:
	1232	tkt['summary'] = subject
[22]	1233
	1234
[507]	1235	if set_fields:
	1236	rest, keywords = string.split(set_fields, '?')
[262]	1237
[507]	1238	if keywords:
	1239	update_fields = self.str_to_dict(keywords)
	1240	self.update_ticket_fields(tkt, update_fields)
[262]	1241
[296]	1242
[604]	1243	message_parts = self.get_message_parts(msg, True)
[309]	1244
[507]	1245	## Must we update some ticket fields properties via body_text
	1246	#
	1247	if self.properties:
	1248	self.update_ticket_fields(tkt, self.properties)
[309]	1249
[507]	1250	message_parts = self.unique_attachment_names(message_parts)
	1251
	1252	## produce e-mail like header
	1253	#
	1254	head = ''
	1255	if self.parameters.email_header:
	1256	head = self.email_header_txt(msg)
	1257	message_parts.insert(0, head)
	1258
	1259	body_text = self.get_body_text(message_parts)
[45]	1260
[507]	1261	tkt['description'] = body_text
[90]	1262
[507]	1263	## When is the change committed
	1264	#
	1265	if self.VERSION < 0.11:
	1266	when = int(time.time())
	1267	else:
	1268	when = datetime.now(util.datefmt.utc)
[45]	1269
[507]	1270	if self.parameters.dry_run:
[561]	1271	self.logger.info('DRY_RUN: tkt.insert()')
[507]	1272	else:
	1273	self.id = tkt.insert()
	1274
	1275	changed = False
	1276	comment = ''
[77]	1277
[507]	1278	## some routines in trac are dependend on ticket id
	1279	# like alternate notify template
	1280	#
	1281	if self.parameters.alternate_notify_template:
	1282	tkt['id'] = self.id
	1283	changed = True
[273]	1284
[507]	1285	## Rewrite the description if we have mailto enabled
	1286	#
	1287	if self.parameters.mailto_link:
	1288	changed = True
	1289	comment = u'\nadded mailto line\n'
[592]	1290
[581]	1291	#mailto = self.html_mailto_link( m['Subject'])
	1292	mailto = self.html_mailto_link(subject)
[253]	1293
[507]	1294	tkt['description'] = u'%s\r\n%s%s\r\n' \
	1295	%(head, mailto, body_text)
	1296
	1297	## Save the attachments to the ticket
	1298	#
	1299	error_with_attachments = self.attach_attachments(message_parts)
[295]	1300
[507]	1301	if error_with_attachments:
	1302	changed = True
	1303	comment = '%s\n%s\n' %(comment, error_with_attachments)
[45]	1304
[545]	1305	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[546]	1306	if self.mail_workflow(tkt):
[545]	1307	changed = True
	1308
[507]	1309	if changed:
	1310	if self.parameters.dry_run:
[561]	1311	s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
	1312	self.logger.info(s)
	1313
[507]	1314	else:
	1315	tkt.save_changes(tkt['reporter'], comment)
[90]	1316
[507]	1317	if not spam:
[545]	1318	self.notify(tkt, True)
[45]	1319
[260]	1320
[507]	1321	def attach_attachments(self, message_parts, update=False):
	1322	'''
	1323	save any attachments as files in the ticket's directory
	1324	'''
	1325	self.logger.debug('function attach_attachments()')
[342]	1326
[507]	1327	if self.parameters.dry_run:
[555]	1328	self.logger.debug("DRY_RUN: no attachments attached to tickets")
[507]	1329	return ''
[342]	1330
[507]	1331	count = 0
[342]	1332
[507]	1333	## Get Maxium attachment size
	1334	#
	1335	max_size = int(self.get_config('attachment', 'max_size'))
	1336	status = None
	1337
	1338	for item in message_parts:
	1339	## Skip body parts
	1340	#
	1341	if not isinstance(item, tuple):
	1342	continue
	1343
	1344	(original, filename, part) = item
[477]	1345
[548]	1346	## We have to determine the size so we use this temporary solution.
[507]	1347	#
[548]	1348	path, fd = util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
[507]	1349	text = part.get_payload(decode=1)
	1350	if not text:
	1351	text = '(None)'
	1352	fd.write(text)
	1353	fd.close()
[342]	1354
[507]	1355	## get the file_size
	1356	#
	1357	stats = os.lstat(path)
	1358	file_size = stats[ST_SIZE]
[342]	1359
[507]	1360	## Check if the attachment size is allowed
	1361	#
	1362	if (max_size != -1) and (file_size > max_size):
	1363	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
	1364	%(status, original, file_size, max_size)
[342]	1365
[507]	1366	os.unlink(path)
	1367	continue
	1368	else:
	1369	count = count + 1
	1370
	1371	## Insert the attachment
	1372	#
	1373	fd = open(path, 'rb')
	1374	if self.system == 'discussion':
[553]	1375	att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
[585]	1376	elif self.system == 'blog':
	1377	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
[507]	1378	else:
[548]	1379	s = 'Attach %s to ticket %d' %(filename, self.id)
	1380	self.print_unicode(s)
[507]	1381	att = attachment.Attachment(self.env, 'ticket', self.id)
[359]	1382
[507]	1383	## This will break the ticket_update system, the body_text is vaporized
	1384	# ;-(
	1385	#
	1386	if not update:
	1387	att.author = self.author
	1388	att.description = self.email_to_unicode('Added by email2trac')
[342]	1389
[553]	1390	try:
[549]	1391
[507]	1392	self.logger.debug('Insert atachment')
	1393	att.insert(filename, fd, file_size)
[553]	1394
[507]	1395	except OSError, detail:
[553]	1396
[507]	1397	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
	1398	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
[342]	1399
[507]	1400	## Remove the created temporary filename
	1401	#
	1402	fd.close()
	1403	os.unlink(path)
[342]	1404
[507]	1405	## return error
	1406	#
	1407	return status
[342]	1408
[359]	1409	########## Fullblog functions #################################################
[339]	1410
[585]	1411	def blog(self, msg, subject, id, params):
[507]	1412	"""
	1413	The blog create/update function
	1414	"""
	1415	## import the modules
	1416	#
	1417	from tracfullblog.core import FullBlogCore
	1418	from tracfullblog.model import BlogPost, BlogComment
[260]	1419
[507]	1420	## instantiate blog core
	1421	#
	1422	blog = FullBlogCore(self.env)
	1423	req = Mock(authname='anonymous', perm=MockPerm(), args={})
[312]	1424
[585]	1425	## parameters from the subject
	1426	#
	1427	params = self.str_to_dict((params or '').lstrip('?'))
[260]	1428
[585]	1429	## preferably get the time from the email date header, or else
	1430	# use current date and time
	1431	date = email.Utils.parsedate_tz(msg.get('date'))
	1432	if date:
	1433	dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
	1434	else:
	1435	self.logger.warn("No valid date header found")
	1436	dt = util.datefmt.to_datetime(None, util.datefmt.utc)
	1437
	1438	## blog entry affected
	1439	#
	1440	self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
	1441
	1442	## check wether a blog post exists
	1443	#
	1444	post = BlogPost(self.env, self.id)
	1445	force_update = self.properties.get('update', params.get('update'))
	1446
	1447	## message parts
	1448	#
	1449	message_parts = self.get_message_parts(msg)
	1450	message_parts = self.unique_attachment_names(message_parts)
	1451
	1452	if post.get_versions() and not force_update:
	1453
	1454	## add comment to blog entry
[507]	1455	#
[585]	1456	comment = BlogComment(self.env, self.id)
	1457	comment.author = self.properties.get('author', params.get('author', self.author))
[507]	1458	comment.comment = self.get_body_text(message_parts)
[585]	1459	comment.time = dt
[312]	1460
[578]	1461	if self.parameters.dry_run:
	1462	self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
	1463	return
[585]	1464	warnings = blog.create_comment(req, comment)
[260]	1465
[507]	1466	else:
[585]	1467	## create or update blog entry
[507]	1468	#
[585]	1469	post.author = self.properties.get('author', params.get('author', self.author))
	1470	post.categories = self.properties.get('categories', params.get('categories', ''))
[578]	1471	post.title = subject.strip()
[585]	1472	post.publish_time = dt
[507]	1473	post.body = self.get_body_text(message_parts)
	1474
[578]	1475	if self.parameters.dry_run:
	1476	self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
	1477	return
[585]	1478	warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
[260]	1479
[585]	1480	## check for problems
	1481	#
	1482	if warnings:
	1483	raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
	1484
	1485	## all seems well, attach attachments
	1486	#
	1487	self.attach_attachments(message_parts)
[260]	1488
[585]	1489
[359]	1490	########## Discussion functions ##############################################
[342]	1491
[507]	1492	def discussion_topic(self, content, subject):
[342]	1493
[507]	1494	## Import modules.
	1495	#
	1496	from tracdiscussion.api import DiscussionApi
	1497	from trac.util.datefmt import to_timestamp, utc
[359]	1498
[507]	1499	self.logger.debug('Creating a new topic in forum:', self.id)
[359]	1500
[507]	1501	## Get dissussion API component.
	1502	#
	1503	api = self.env[DiscussionApi]
	1504	context = self._create_context(content, subject)
[359]	1505
[507]	1506	## Get forum for new topic.
	1507	#
	1508	forum = api.get_forum(context, self.id)
[359]	1509
[507]	1510	if not forum:
	1511	self.logger.error("ERROR: Replied forum doesn't exist")
[359]	1512
[507]	1513	## Prepare topic.
	1514	#
	1515	topic = {'forum' : forum['id'],
	1516	'subject' : context.subject,
	1517	'time': to_timestamp(datetime.now(utc)),
	1518	'author' : self.author,
	1519	'subscribers' : [self.email_addr],
	1520	'body' : self.get_body_text(context.content_parts)}
[359]	1521
[507]	1522	## Add topic to DB and commit it.
	1523	#
	1524	self._add_topic(api, context, topic)
	1525	self.db.commit()
[359]	1526
[507]	1527	def discussion_topic_reply(self, content, subject):
[359]	1528
[507]	1529	## Import modules.
	1530	#
	1531	from tracdiscussion.api import DiscussionApi
	1532	from trac.util.datefmt import to_timestamp, utc
[359]	1533
[507]	1534	self.logger.debug('Replying to discussion topic', self.id)
[359]	1535
[507]	1536	## Get dissussion API component.
	1537	#
	1538	api = self.env[DiscussionApi]
	1539	context = self._create_context(content, subject)
[359]	1540
[507]	1541	## Get replied topic.
	1542	#
	1543	topic = api.get_topic(context, self.id)
[359]	1544
[507]	1545	if not topic:
	1546	self.logger.error("ERROR: Replied topic doesn't exist")
[359]	1547
[507]	1548	## Prepare message.
	1549	#
	1550	message = {'forum' : topic['forum'],
	1551	'topic' : topic['id'],
	1552	'replyto' : -1,
	1553	'time' : to_timestamp(datetime.now(utc)),
	1554	'author' : self.author,
	1555	'body' : self.get_body_text(context.content_parts)}
[359]	1556
[507]	1557	## Add message to DB and commit it.
	1558	#
	1559	self._add_message(api, context, message)
	1560	self.db.commit()
[359]	1561
[507]	1562	def discussion_message_reply(self, content, subject):
[359]	1563
[507]	1564	## Import modules.
	1565	#
	1566	from tracdiscussion.api import DiscussionApi
	1567	from trac.util.datefmt import to_timestamp, utc
[359]	1568
[507]	1569	self.logger.debug('Replying to discussion message', self.id)
[359]	1570
[507]	1571	## Get dissussion API component.
	1572	#
	1573	api = self.env[DiscussionApi]
	1574	context = self._create_context(content, subject)
[359]	1575
[507]	1576	## Get replied message.
	1577	#
	1578	message = api.get_message(context, self.id)
[359]	1579
[507]	1580	if not message:
	1581	self.logger.error("ERROR: Replied message doesn't exist")
[359]	1582
[507]	1583	## Prepare message.
	1584	#
	1585	message = {'forum' : message['forum'],
	1586	'topic' : message['topic'],
	1587	'replyto' : message['id'],
	1588	'time' : to_timestamp(datetime.now(utc)),
	1589	'author' : self.author,
	1590	'body' : self.get_body_text(context.content_parts)}
[359]	1591
[507]	1592	## Add message to DB and commit it.
	1593	#
	1594	self._add_message(api, context, message)
	1595	self.db.commit()
[359]	1596
[507]	1597	def _create_context(self, content, subject):
[359]	1598
[507]	1599	## Import modules.
	1600	#
	1601	from trac.mimeview import Context
	1602	from trac.web.api import Request
	1603	from trac.perm import PermissionCache
[359]	1604
[507]	1605	## TODO: Read server base URL from config.
	1606	# Create request object to mockup context creation.
	1607	#
	1608	environ = {'SERVER_PORT' : 80,
	1609	'SERVER_NAME' : 'test',
	1610	'REQUEST_METHOD' : 'POST',
	1611	'wsgi.url_scheme' : 'http',
	1612	'wsgi.input' : sys.stdin}
	1613	chrome = {'links': {},
	1614	'scripts': [],
	1615	'ctxtnav': [],
	1616	'warnings': [],
	1617	'notices': []}
[359]	1618
[507]	1619	if self.env.base_url_for_redirect:
	1620	environ['trac.base_url'] = self.env.base_url
[359]	1621
[507]	1622	req = Request(environ, None)
	1623	req.chrome = chrome
	1624	req.tz = 'missing'
	1625	req.authname = self.author
	1626	req.perm = PermissionCache(self.env, self.author)
	1627	req.locale = None
[359]	1628
[507]	1629	## Create and return context.
	1630	#
	1631	context = Context.from_request(req)
	1632	context.realm = 'discussion-email2trac'
	1633	context.cursor = self.db.cursor()
	1634	context.content = content
	1635	context.subject = subject
[359]	1636
[507]	1637	## Read content parts from content.
	1638	#
	1639	context.content_parts = self.get_message_parts(content)
	1640	context.content_parts = self.unique_attachment_names(
	1641	context.content_parts)
[359]	1642
[507]	1643	return context
[359]	1644
[507]	1645	def _add_topic(self, api, context, topic):
	1646	context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]	1647
[507]	1648	## Filter topic.
	1649	#
	1650	for discussion_filter in api.discussion_filters:
	1651	accept, topic_or_error = discussion_filter.filter_topic(
	1652	context, topic)
	1653	if accept:
	1654	topic = topic_or_error
	1655	else:
	1656	raise TracError(topic_or_error)
[359]	1657
[507]	1658	## Add a new topic.
	1659	#
	1660	api.add_topic(context, topic)
[359]	1661
[507]	1662	## Get inserted topic with new ID.
	1663	#
	1664	topic = api.get_topic_by_time(context, topic['time'])
[359]	1665
[507]	1666	## Attach attachments.
	1667	#
	1668	self.id = topic['id']
	1669	self.attach_attachments(context.content_parts, True)
[359]	1670
[507]	1671	## Notify change listeners.
	1672	#
	1673	for listener in api.topic_change_listeners:
	1674	listener.topic_created(context, topic)
[359]	1675
[507]	1676	def _add_message(self, api, context, message):
	1677	context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]	1678
[507]	1679	## Filter message.
	1680	#
	1681	for discussion_filter in api.discussion_filters:
	1682	accept, message_or_error = discussion_filter.filter_message(
	1683	context, message)
	1684	if accept:
	1685	message = message_or_error
	1686	else:
	1687	raise TracError(message_or_error)
[359]	1688
[507]	1689	## Add message.
	1690	#
	1691	api.add_message(context, message)
[359]	1692
[507]	1693	## Get inserted message with new ID.
	1694	#
	1695	message = api.get_message_by_time(context, message['time'])
[359]	1696
[507]	1697	## Attach attachments.
	1698	#
[549]	1699
[507]	1700	self.attach_attachments(context.content_parts, True)
[359]	1701
[507]	1702	## Notify change listeners.
	1703	#
	1704	for listener in api.message_change_listeners:
	1705	listener.message_created(context, message)
[359]	1706
	1707	########## MAIN function ######################################################
	1708
[507]	1709	def parse(self, fp):
	1710	"""
	1711	"""
	1712	self.logger.debug('Main function parse')
	1713	global m
[96]	1714
[507]	1715	m = email.message_from_file(fp)
	1716
	1717	if not m:
	1718	self.logger.debug('This is not a valid email message format')
	1719	return
	1720
	1721	## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
	1722	#
	1723	try:
	1724	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
	1725	except AttributeError, detail:
	1726	pass
[239]	1727
[507]	1728	if self.parameters.debug: # save the entire e-mail message text
[528]	1729	self.save_email_for_debug(m, self.parameters.project_name, True)
[77]	1730
[507]	1731	self.db = self.env.get_db_cnx()
	1732	self.get_sender_info(m)
[152]	1733
[587]	1734
[588]	1735	if self.parameters.white_list_file:
	1736	self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
	1737
[587]	1738	if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
	1739
[507]	1740	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
	1741	return False
[77]	1742
[507]	1743	if self.email_header_acl('black_list', self.email_addr, False):
	1744	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
	1745	return False
[221]	1746
[570]	1747	if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
	1748	self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
[507]	1749	return False
[226]	1750
[507]	1751	## If spam drop the message
	1752	#
	1753	if self.spam(m) == 'drop':
	1754	return False
[194]	1755
[507]	1756	elif self.spam(m) == 'spam':
	1757	spam_msg = True
	1758	else:
	1759	spam_msg = False
[204]	1760
[507]	1761	if not m['Subject']:
	1762	subject = 'No Subject'
	1763	else:
[564]	1764	subject = self.email_to_unicode(m['Subject'])
[304]	1765
[564]	1766	## Bug in python logging module <2.6
	1767	#
	1768	self.logger.info('subject: %s' %repr(subject))
[359]	1769
[507]	1770	## [hic] #1529: Re: LRZ
	1771	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
	1772	#
	1773	ticket_regex = r'''
	1774	(?P<new_fields>[#][?].*)
	1775	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
	1776	'''
	1777	## Check if FullBlogPlugin is installed
	1778	#
	1779	blog_enabled = None
	1780	blog_regex = ''
	1781	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
	1782	self.logger.debug('Trac BLOG support enabled')
	1783	blog_enabled = True
[585]	1784	blog_regex = '''\|(?P<blog>blog(?P<blog_params>[?][^:])?:(?P<blog_id>\S))'''
[329]	1785
[77]	1786
[507]	1787	## Check if DiscussionPlugin is installed
	1788	#
	1789	discussion_enabled = None
	1790	discussion_regex = ''
	1791	if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
	1792	self.logger.debug('Trac Discussion support enabled')
	1793	discussion_enabled = True
	1794	discussion_regex = r'''
	1795	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
	1796	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
	1797	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
	1798	'''
[77]	1799
[359]	1800
[507]	1801	regex_str = ticket_regex + blog_regex + discussion_regex
	1802	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
[359]	1803
[507]	1804	## Find out if this is a ticket, a blog or a discussion
	1805	#
	1806	result = SYSTEM_RE.search(subject)
[390]	1807
[507]	1808	if result:
	1809	## update ticket + fields
	1810	#
	1811	if result.group('reply'):
	1812	self.system = 'ticket'
[260]	1813
[507]	1814	## Skip the last ':' character
	1815	#
	1816	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
	1817	self.new_ticket(m, subject, spam_msg)
[390]	1818
[507]	1819	## New ticket + fields
	1820	#
	1821	elif result.group('new_fields'):
	1822	self.system = 'ticket'
	1823	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
[262]	1824
[507]	1825	if blog_enabled:
	1826	if result.group('blog'):
	1827	self.system = 'blog'
[585]	1828	self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
[359]	1829
[507]	1830	if discussion_enabled:
	1831	## New topic.
	1832	#
	1833	if result.group('forum'):
	1834	self.system = 'discussion'
	1835	self.id = int(result.group('forum_id'))
	1836	self.discussion_topic(m, subject[result.end('forum'):])
[359]	1837
[507]	1838	## Reply to topic.
	1839	#
	1840	elif result.group('topic'):
	1841	self.system = 'discussion'
	1842	self.id = int(result.group('topic_id'))
	1843	self.discussion_topic_reply(m, subject[result.end('topic'):])
[359]	1844
[507]	1845	## Reply to topic message.
	1846	#
	1847	elif result.group('message'):
	1848	self.system = 'discussion'
	1849	self.id = int(result.group('message_id'))
	1850	self.discussion_message_reply(m, subject[result.end('message'):])
[359]	1851
[507]	1852	else:
[553]	1853
[507]	1854	self.system = 'ticket'
	1855	(matched_id, subject) = self.ticket_update_by_subject(subject)
[553]	1856
[507]	1857	if matched_id:
[553]	1858
[507]	1859	if not self.ticket_update(m, matched_id, spam_msg):
	1860	self.new_ticket(m, subject, spam_msg)
[553]	1861
[507]	1862	else:
	1863	## No update by subject, so just create a new ticket
	1864	#
	1865	self.new_ticket(m, subject, spam_msg)
[353]	1866
[356]	1867
[343]	1868	########## BODY TEXT functions ###########################################################
	1869
[507]	1870	def strip_signature(self, text):
	1871	"""
	1872	Strip signature from message, inspired by Mailman software
	1873	"""
	1874	self.logger.debug('function strip_signature')
[462]	1875
[507]	1876	body = []
	1877	for line in text.splitlines():
	1878	if line == '-- ':
	1879	break
	1880	body.append(line)
[136]	1881
[507]	1882	return ('\n'.join(body))
[136]	1883
[507]	1884	def reflow(self, text, delsp = 0):
	1885	"""
	1886	Reflow the message based on the format="flowed" specification (RFC 3676)
	1887	"""
	1888	flowedlines = []
	1889	quotelevel = 0
	1890	prevflowed = 0
[231]	1891
[507]	1892	for line in text.splitlines():
	1893	from re import match
	1894
	1895	## Figure out the quote level and the content of the current line
	1896	#
	1897	m = match('(>)( ?)(.)', line)
	1898	linequotelevel = len(m.group(1))
	1899	line = m.group(3)
[231]	1900
[507]	1901	## Determine whether this line is flowed
	1902	#
	1903	if line and line != '-- ' and line[-1] == ' ':
	1904	flowed = 1
	1905	else:
	1906	flowed = 0
[231]	1907
[507]	1908	if flowed and delsp and line and line[-1] == ' ':
	1909	line = line[:-1]
[231]	1910
[507]	1911	## If the previous line is flowed, append this line to it
	1912	#
	1913	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
	1914	flowedlines[-1] += line
[477]	1915
[507]	1916	## Otherwise, start a new line
	1917	#
	1918	else:
	1919	flowedlines.append('>' * linequotelevel + line)
[231]	1920
[507]	1921	prevflowed = flowed
	1922
[231]	1923
[507]	1924	return '\n'.join(flowedlines)
[231]	1925
[507]	1926	def strip_quotes(self, text):
	1927	"""
	1928	Strip quotes from message by Nicolas Mendoza
	1929	"""
	1930	self.logger.debug('function strip_quotes')
[462]	1931
[507]	1932	body = []
	1933	for line in text.splitlines():
	1934	try:
[462]	1935
[507]	1936	if line.startswith(self.parameters.email_quote):
	1937	continue
[462]	1938
[507]	1939	except UnicodeDecodeError:
[462]	1940
[507]	1941	tmp_line = self.email_to_unicode(line)
	1942	if tmp_line.startswith(self.parameters.email_quote):
	1943	continue
	1944
	1945	body.append(line)
[151]	1946
[507]	1947	return ('\n'.join(body))
[191]	1948
[507]	1949	def inline_properties(self, text):
	1950	"""
	1951	Parse text if we use inline keywords to set ticket fields
	1952	"""
	1953	self.logger.debug('function inline_properties')
[309]	1954
[507]	1955	properties = dict()
	1956	body = list()
[309]	1957
[507]	1958	INLINE_EXP = re.compile('\s[@]\s(\w+)\s:(.)$')
[309]	1959
[507]	1960	for line in text.splitlines():
	1961	match = INLINE_EXP.match(line)
	1962	if match:
	1963	keyword, value = match.groups()
[509]	1964
	1965	if self.parameters.inline_properties_first_wins:
	1966	if keyword in self.properties.keys():
	1967	continue
	1968
[507]	1969	self.properties[keyword] = value.strip()
[509]	1970	self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
[408]	1971
[507]	1972	else:
	1973	body.append(line)
	1974
	1975	return '\n'.join(body)
[309]	1976
	1977
[507]	1978	def wrap_text(self, text, replace_whitespace = False):
	1979	"""
	1980	Will break a lines longer then given length into several small
	1981	lines of size given length
	1982	"""
	1983	import textwrap
[154]	1984
[507]	1985	LINESEPARATOR = '\n'
	1986	reformat = ''
[151]	1987
[507]	1988	for s in text.split(LINESEPARATOR):
	1989	tmp = textwrap.fill(s, self.parameters.use_textwrap)
	1990	if tmp:
	1991	reformat = '%s\n%s' %(reformat,tmp)
	1992	else:
	1993	reformat = '%s\n' %reformat
[153]	1994
[507]	1995	return reformat
[153]	1996
[507]	1997	# Python2.4 and higher
	1998	#
	1999	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
	2000	#
[154]	2001
[343]	2002	########## EMAIL attachements functions ###########################################################
	2003
[507]	2004	def inline_part(self, part):
	2005	"""
	2006	"""
	2007	self.logger.debug('function inline_part()')
[154]	2008
[507]	2009	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
[340]	2010
[604]	2011	def get_message_parts(self, msg, new_email=False):
[507]	2012	"""
	2013	parses the email message and returns a list of body parts and attachments
	2014	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
	2015	"""
	2016	self.logger.debug('function get_message_parts()')
[317]	2017
[507]	2018	message_parts = list()
	2019
	2020	ALTERNATIVE_MULTIPART = False
[278]	2021
[507]	2022	for part in msg.walk():
	2023	content_maintype = part.get_content_maintype()
	2024	content_type = part.get_content_type()
[278]	2025
[507]	2026	self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
	2027	self.logger.debug('\t Message part: Content-Type: %s' % content_type)
[498]	2028
[507]	2029	## Check content type
	2030	#
	2031	if content_type in self.STRIP_CONTENT_TYPES:
	2032	self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
	2033	continue
[238]	2034
[507]	2035	## Catch some mulitpart execptions
	2036	#
	2037	if content_type == 'multipart/alternative':
	2038	ALTERNATIVE_MULTIPART = True
	2039	continue
[278]	2040
[507]	2041	## Skip multipart containers
	2042	#
	2043	if content_maintype == 'multipart':
	2044	self.logger.debug("\t Skipping multipart container")
	2045	continue
	2046
[569]	2047	## Check if this is an inline part. It's inline if there is co Cont-Disp header,
	2048	# or if there is one and it says "inline"
[507]	2049	#
	2050	inline = self.inline_part(part)
[236]	2051
[507]	2052	## Drop HTML message
	2053	#
	2054	if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
[553]	2055
[507]	2056	if content_type == 'text/html':
	2057	self.logger.debug('\t Skipping alternative HTML message')
	2058	ALTERNATIVE_MULTIPART = False
	2059	continue
[278]	2060
[569]	2061	filename = part.get_filename()
	2062	s = '\t unicode filename: %s' %(filename)
	2063	self.print_unicode(s)
	2064	self.logger.debug('\t raw filename: %s' %repr(filename))
[498]	2065
[569]	2066	filename = self.check_filename_length(filename)
	2067
[507]	2068	## Save all non plain text message as attachment
	2069	#
	2070	if not content_type in ['text/plain']:
[498]	2071
[569]	2072	message_parts.append( (filename, part) )
[498]	2073
[569]	2074	## We only convert html messages
[507]	2075	#
	2076	if not content_type == 'text/html':
[569]	2077	self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
[507]	2078	continue
[500]	2079
[569]	2080
	2081	## We have an text or html message
	2082	#
[507]	2083	if not inline:
[569]	2084	self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
	2085	message_parts.append( (filename, part) )
[507]	2086	continue
	2087
	2088	## Try to decode message part. We have a html or plain text messafe
	2089	#
	2090	body_text = part.get_payload(decode=1)
	2091	if not body_text:
	2092	body_text = part.get_payload(decode=0)
[236]	2093
[507]	2094	## Try to convert html message
	2095	#
	2096	if content_type == 'text/html':
[502]	2097
[507]	2098	body_text = self.html_2_txt(body_text)
	2099	if not body_text:
	2100	continue
[498]	2101
[507]	2102	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
	2103	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
[231]	2104
[507]	2105	if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
	2106	body_text = self.reflow(body_text, delsp == 'yes')
[22]	2107
[604]	2108	if new_email and self.parameters.only_strip_on_update:
	2109	self.logger.debug('Skip signature/quote stripping for new messages')
	2110	else:
	2111	if self.parameters.strip_signature:
	2112	body_text = self.strip_signature(body_text)
[191]	2113
[604]	2114	if self.parameters.strip_quotes:
	2115	body_text = self.strip_quotes(body_text)
	2116
[507]	2117	if self.parameters.inline_properties:
	2118	body_text = self.inline_properties(body_text)
[309]	2119
[507]	2120	if self.parameters.use_textwrap:
	2121	body_text = self.wrap_text(body_text)
[148]	2122
[507]	2123	## Get contents charset (iso-8859-15 if not defined in mail headers)
	2124	#
	2125	charset = part.get_content_charset()
	2126	if not charset:
	2127	charset = 'iso-8859-15'
[102]	2128
[507]	2129	try:
	2130	ubody_text = unicode(body_text, charset)
[100]	2131
[507]	2132	except UnicodeError, detail:
	2133	ubody_text = unicode(body_text, 'iso-8859-15')
[89]	2134
[507]	2135	except LookupError, detail:
	2136	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
[100]	2137
[507]	2138	if self.parameters.verbatim_format:
	2139	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
	2140	else:
	2141	message_parts.append('%s' %ubody_text)
[22]	2142
[507]	2143	return message_parts
	2144
	2145	def unique_attachment_names(self, message_parts):
	2146	"""
	2147	Make sure we have unique names attachments:
	2148	- check if it contains illegal characters
	2149	- Rename "None" filenames to "untitled-part"
	2150	"""
	2151	self.logger.debug('function unique_attachment_names()')
	2152	renamed_parts = []
	2153	attachment_names = set()
[296]	2154
[507]	2155	for item in message_parts:
	2156
	2157	## If not an attachment, leave it alone
	2158	#
	2159	if not isinstance(item, tuple):
	2160	renamed_parts.append(item)
	2161	continue
	2162
	2163	(filename, part) = item
[295]	2164
[507]	2165	## If filename = None, use a default one
	2166	#
	2167	if filename in [ 'None']:
	2168	filename = 'untitled-part'
[550]	2169	self.logger.info('\t Rename filename "None" to: %s' %filename)
[22]	2170
[507]	2171	## Guess the extension from the content type, use non strict mode
	2172	# some additional non-standard but commonly used MIME types
	2173	# are also recognized
	2174	#
	2175	ext = mimetypes.guess_extension(part.get_content_type(), False)
	2176	if not ext:
	2177	ext = '.bin'
[22]	2178
[507]	2179	filename = '%s%s' % (filename, ext)
[22]	2180
[507]	2181	## Discard relative paths for windows/unix in attachment names
	2182	#
	2183	filename = filename.replace('\\', '_')
	2184	filename = filename.replace('/', '_')
[347]	2185
[507]	2186	## remove linefeed char
	2187	#
	2188	for forbidden_char in ['\r', '\n']:
	2189	filename = filename.replace(forbidden_char,'')
[465]	2190
[507]	2191	## We try to normalize the filename to utf-8 NFC if we can.
	2192	# Files uploaded from OS X might be in NFD.
	2193	# Check python version and then try it
	2194	#
	2195	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
	2196	# try:
	2197	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
	2198	# except TypeError:
	2199	# pass
[100]	2200
[507]	2201	## Make the filename unique for this ticket
	2202	#
	2203	num = 0
	2204	unique_filename = filename
	2205	dummy_filename, ext = os.path.splitext(filename)
[134]	2206
[507]	2207	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
	2208	num += 1
	2209	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
	2210
[550]	2211	s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
[549]	2212	self.print_unicode(s)
[100]	2213
[507]	2214	attachment_names.add(unique_filename)
[236]	2215
[507]	2216	renamed_parts.append((filename, unique_filename, part))
	2217
	2218	return renamed_parts
	2219
	2220
	2221	def attachment_exists(self, filename):
[250]	2222
[549]	2223	self.logger.debug("function attachment_exists")
[250]	2224
[549]	2225	s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
	2226	self.print_unicode(s)
	2227
[553]	2228	## Do we have a valid ticket id
[507]	2229	#
	2230	if not self.id:
	2231	return False
[250]	2232
[507]	2233	try:
	2234	if self.system == 'discussion':
[553]	2235
	2236	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
	2237
[585]	2238	elif self.system == 'blog':
	2239
	2240	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
	2241
[507]	2242	else:
[553]	2243
	2244	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
	2245
[507]	2246	return True
[553]	2247
[507]	2248	except attachment.ResourceNotFound:
[553]	2249
[507]	2250	return False
[343]	2251
	2252	########## TRAC Ticket Text ###########################################################
[507]	2253
	2254	def get_body_text(self, message_parts):
	2255	"""
	2256	"""
	2257	self.logger.debug('function get_body_text()')
[498]	2258
[507]	2259	body_text = []
	2260
	2261	for part in message_parts:
	2262
	2263	## Plain text part, append it
	2264	#
	2265	if not isinstance(part, tuple):
	2266	body_text.extend(part.strip().splitlines())
	2267	body_text.append("")
	2268	continue
[490]	2269
[507]	2270	(original, filename, part) = part
	2271	inline = self.inline_part(part)
[491]	2272
[507]	2273	## Skip generation of attachment link if html is converted to text
	2274	#
	2275	if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
	2276	s = 'Skipping attachment link for html part: %s' %(filename)
	2277	self.print_unicode(s)
	2278	continue
	2279
	2280	if part.get_content_maintype() == 'image' and inline:
[553]	2281
[507]	2282	if self.system != 'discussion':
	2283	s = 'wiki image link for: %s' %(filename)
	2284	self.print_unicode(s)
	2285	body_text.append('[[Image(%s)]]' % filename)
[553]	2286
[507]	2287	body_text.append("")
[553]	2288
[507]	2289	else:
[553]	2290
[507]	2291	if self.system != 'discussion':
[553]	2292
[507]	2293	s = 'wiki attachment link for: %s' %(filename)
	2294	self.print_unicode(s)
	2295	body_text.append('[attachment:"%s"]' % filename)
[553]	2296
[507]	2297	body_text.append("")
[490]	2298
[507]	2299	## Convert list body_texts to string
	2300	#
	2301	body_text = '\r\n'.join(body_text)
	2302	return body_text
[236]	2303
[507]	2304	def html_mailto_link(self, subject):
	2305	"""
	2306	This function returns a HTML mailto tag with the ticket id and author email address
	2307	"""
	2308	self.logger.debug("function html_mailto_link")
[576]	2309
[507]	2310	if not self.author:
	2311	author = self.email_addr
	2312	else:
	2313	author = self.author
[343]	2314
[507]	2315	if not self.parameters.mailto_cc:
	2316	self.parameters.mailto_cc = ''
[592]	2317
	2318	## Bug in urllib.quote function
	2319	#
	2320	if isinstance(subject, unicode):
	2321	subject = subject.encode('utf-8')
[434]	2322
[507]	2323	## use urllib to escape the chars
	2324	#
[517]	2325	s = '%s?Subject=%s&cc=%s' %(
[507]	2326	urllib.quote(self.email_addr),
	2327	urllib.quote('Re: #%s: %s' %(self.id, subject)),
	2328	urllib.quote(self.parameters.mailto_cc)
	2329	)
[343]	2330
[507]	2331	if self.VERSION in [ 0.10 ]:
	2332	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
	2333	else:
	2334	s = '[mailto:"%s" Reply to: %s]' %(s, author)
[506]	2335
[507]	2336	self.logger.debug("\tmailto link %s" %s)
	2337	return s
[343]	2338
	2339	########## TRAC notify section ###########################################################
	2340
[507]	2341	def notify(self, tkt, new=True, modtime=0):
	2342	"""
	2343	A wrapper for the TRAC notify function. So we can use templates
	2344	"""
	2345	self.logger.debug('function notify()')
[392]	2346
[509]	2347	if self.parameters.notify_reporter:
	2348	self.logger.debug('\t Notify reporter set')
[508]	2349	global sender_email
	2350	sender_email = self.email_addr
	2351
	2352	self.logger.debug('\t Using Email2TracNotification function AlwaysNotifyReporter')
	2353	import trac.notification as Email2TracNotification
	2354	Email2TracNotification.Notify.notify = AlwaysNotifyReporter
	2355
	2356	if self.parameters.dry_run :
[561]	2357	self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
[507]	2358	return
	2359	try:
[392]	2360
[507]	2361	#from trac.ticket.notification import TicketNotificationSystem
	2362	#tn_sys = TicketNotificationSystem(self.env)
	2363	#print tn_sys
	2364	#print tn_sys.__dict__
	2365	#sys.exit(0)
[392]	2366
[507]	2367	## create false {abs_}href properties, to trick Notify()
	2368	#
[540]	2369	if self.VERSION in [0.10]:
[507]	2370	self.env.abs_href = Href(self.get_config('project', 'url'))
	2371	self.env.href = Href(self.get_config('project', 'url'))
[22]	2372
[507]	2373	tn = TicketNotifyEmail(self.env)
[213]	2374
[507]	2375	if self.parameters.alternate_notify_template:
[222]	2376
[507]	2377	if self.VERSION >= 0.11:
[222]	2378
[507]	2379	from trac.web.chrome import Chrome
[222]	2380
[507]	2381	if self.parameters.alternate_notify_template_update and not new:
	2382	tn.template_name = self.parameters.alternate_notify_template_update
	2383	else:
	2384	tn.template_name = self.parameters.alternate_notify_template
[222]	2385
[507]	2386	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
	2387
	2388	else:
[222]	2389
[507]	2390	tn.template_name = self.parameters.alternate_notify_template
[42]	2391
[507]	2392	tn.notify(tkt, new, modtime)
[41]	2393
[507]	2394	except Exception, e:
	2395	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
[41]	2396
[508]	2397	########## END Class Definition ########################################################
[22]	2398
[508]	2399	########## Global Notificaition Function ################################################
	2400	def AlwaysNotifyReporter(self, resid):
	2401	"""
	2402	Copy of def notify() to manipulate recipents to always include reporter for the
	2403	notification.
	2404	"""
[569]	2405	#print sender_email, resid
[508]	2406	(torcpts, ccrcpts) = self.get_recipients(resid)
[569]	2407	#print "get_recipients finished"
[74]	2408
[508]	2409	if not tktparser.email_header_acl('notify_reporter_black_list', sender_email, False):
	2410	## additionally append sender (regardeless of settings in trac.ini)
	2411	#
	2412	if not sender_email in torcpts:
	2413	torcpts.append(sender_email)
	2414
	2415	self.begin_send()
	2416	self.send(torcpts, ccrcpts)
	2417	self.finish_send()
	2418
[343]	2419	########## Parse Config File ###########################################################
[22]	2420
	2421	def ReadConfig(file, name):
[507]	2422	"""
	2423	Parse the config file
	2424	"""
	2425	if not os.path.isfile(file):
	2426	print 'File %s does not exist' %file
	2427	sys.exit(1)
[22]	2428
[507]	2429	config = trac_config.Configuration(file)
	2430
	2431	parentdir = config.get('DEFAULT', 'parentdir')
	2432	sections = config.sections()
[22]	2433
[507]	2434	## use some trac internals to get the defaults
	2435	#
	2436	tmp = config.parser.defaults()
	2437	project = SaraDict()
[485]	2438
[507]	2439	for option, value in tmp.items():
	2440	try:
	2441	project[option] = int(value)
	2442	except ValueError:
	2443	project[option] = value
[485]	2444
[507]	2445	if name:
	2446	if name in sections:
	2447	project = SaraDict()
	2448	for option, value in config.options(name):
	2449	try:
	2450	project[option] = int(value)
	2451	except ValueError:
	2452	project[option] = value
[485]	2453
[507]	2454	elif not parentdir:
	2455	print "Not a valid project name: %s, valid names are: %s" %(name, sections)
	2456	print "or set parentdir in the [DEFAULT] section"
	2457	sys.exit(1)
[22]	2458
[507]	2459	## If parentdir then set project dir to parentdir + name
	2460	#
	2461	if not project.has_key('project'):
	2462	if not parentdir:
	2463	print "You must set project or parentdir in your configuration file"
	2464	sys.exit(1)
	2465	elif not name:
	2466	print "You must configure a project section in your configuration file"
	2467	else:
	2468	project['project'] = os.path.join(parentdir, name)
[22]	2469
[528]	2470	##
	2471	# Save the project name
	2472	#
	2473	project['project_name'] = os.path.basename(project['project'])
	2474
[507]	2475	return project
[22]	2476
[410]	2477	########## Setup Logging ###############################################################
[87]	2478
[419]	2479	def setup_log(parameters, project_name, interactive=None):
[507]	2480	"""
[516]	2481	Setup logging
[410]	2482
[507]	2483	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
[420]	2484	would be interpreted by the ConfigParser itself.
[507]	2485	"""
	2486	logger = logging.getLogger('email2trac %s' %project_name)
[410]	2487
[507]	2488	if interactive:
	2489	parameters.log_type = 'stderr'
[414]	2490
[507]	2491	if not parameters.log_type:
	2492	if sys.platform in ['win32', 'cygwin']:
[521]	2493	parameters.log_type = 'eventlog'
[507]	2494	else:
	2495	parameters.log_type = 'syslog'
[410]	2496
[507]	2497	if parameters.log_type == 'file':
[415]	2498
[507]	2499	if not parameters.log_file:
	2500	parameters.log_file = 'email2trac.log'
[415]	2501
[507]	2502	if not os.path.isabs(parameters.log_file):
	2503	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
[410]	2504
[507]	2505	log_handler = logging.FileHandler(parameters.log_file)
[415]	2506
[507]	2507	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
	2508	## Requires win32 extensions
	2509	#
[532]	2510	logid = "email2trac"
[507]	2511	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
[410]	2512
[507]	2513	elif parameters.log_type in ('syslog', 'unix'):
	2514	log_handler = logging.handlers.SysLogHandler('/dev/log')
[410]	2515
[507]	2516	elif parameters.log_type in ('stderr'):
	2517	log_handler = logging.StreamHandler(sys.stderr)
[410]	2518
[507]	2519	else:
	2520	log_handler = logging.handlers.BufferingHandler(0)
[410]	2521
[507]	2522	if parameters.log_format:
	2523	parameters.log_format = parameters.log_format.replace('$(', '%(')
	2524	else:
	2525	parameters.log_format = '%(name)s: %(message)s'
[516]	2526	if parameters.log_type in ('file', 'stderr'):
	2527	parameters.log_format = '%(asctime)s ' + parameters.log_format
[410]	2528
[507]	2529	log_formatter = logging.Formatter(parameters.log_format)
	2530	log_handler.setFormatter(log_formatter)
	2531	logger.addHandler(log_handler)
[410]	2532
[507]	2533	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
	2534	logger.setLevel(logging.DEBUG)
	2535	parameters.debug = 1
[410]	2536
[507]	2537	elif parameters.log_level in ['INFO'] or parameters.verbose:
	2538	logger.setLevel(logging.INFO)
[410]	2539
[507]	2540	elif parameters.log_level in ['WARNING']:
	2541	logger.setLevel(logging.WARNING)
[410]	2542
[507]	2543	elif parameters.log_level in ['ERROR']:
	2544	logger.setLevel(logging.ERROR)
[410]	2545
[507]	2546	elif parameters.log_level in ['CRITICAL']:
	2547	logger.setLevel(logging.CRITICAL)
[410]	2548
[507]	2549	else:
	2550	logger.setLevel(logging.INFO)
[410]	2551
[507]	2552	return logger
[410]	2553
	2554
[22]	2555	if __name__ == '__main__':
[507]	2556	## Default config file
	2557	#
	2558	configfile = '@email2trac_conf@'
	2559	project = ''
	2560	component = ''
	2561	ticket_prefix = 'default'
	2562	dry_run = None
	2563	verbose = None
	2564	debug_interactive = None
[202]	2565
[507]	2566	SHORT_OPT = 'cdhf:np:t:v'
	2567	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
[201]	2568
[507]	2569	try:
	2570	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
	2571	except getopt.error,detail:
	2572	print __doc__
	2573	print detail
	2574	sys.exit(1)
	2575
	2576	project_name = None
	2577	for opt,value in opts:
	2578	if opt in [ '-h', '--help']:
	2579	print __doc__
	2580	sys.exit(0)
	2581	elif opt in ['-c', '--component']:
	2582	component = value
	2583	elif opt in ['-d', '--debug']:
	2584	debug_interactive = 1
	2585	elif opt in ['-f', '--file']:
	2586	configfile = value
	2587	elif opt in ['-n', '--dry-run']:
	2588	dry_run = True
	2589	elif opt in ['-p', '--project']:
	2590	project_name = value
	2591	elif opt in ['-t', '--ticket_prefix']:
	2592	ticket_prefix = value
	2593	elif opt in ['-v', '--verbose']:
	2594	verbose = True
	2595
	2596	settings = ReadConfig(configfile, project_name)
[410]	2597
[507]	2598	## The default prefix for ticket values in email2trac.conf
	2599	#
	2600	settings.ticket_prefix = ticket_prefix
	2601	settings.dry_run = dry_run
	2602	settings.verbose = verbose
[410]	2603
[507]	2604	if not settings.debug and debug_interactive:
	2605	settings.debug = debug_interactive
[412]	2606
[507]	2607	if not settings.project:
	2608	print __doc__
	2609	print 'No Trac project is defined in the email2trac config file.'
	2610	sys.exit(1)
[419]	2611
[507]	2612	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
	2613
	2614	if component:
	2615	settings['component'] = component
[202]	2616
[507]	2617	## Determine major trac version used to be in email2trac.conf
	2618	# Quick hack for 0.12
	2619	#
	2620	version = '0.%s' %(trac_version.split('.')[1])
	2621	if version.startswith('0.12'):
	2622	version = '0.12'
[541]	2623	elif version.startswith('0.13'):
	2624	version = '0.13'
[373]	2625
[507]	2626	logger.debug("Found trac version: %s" %(version))
	2627
	2628	try:
	2629	if version == '0.10':
	2630	from trac import attachment
	2631	from trac.env import Environment
	2632	from trac.ticket import Ticket
	2633	from trac.web.href import Href
	2634	from trac import util
	2635	from trac.ticket.web_ui import TicketModule
[486]	2636
[507]	2637	#
	2638	# return util.text.to_unicode(str)
	2639	#
	2640	# see http://projects.edgewall.com/trac/changeset/2799
	2641	from trac.ticket.notification import TicketNotifyEmail
	2642	from trac import config as trac_config
	2643	from trac.core import TracError
[359]	2644
[554]	2645	elif version in ['0.11', '0.12', '0.13']:
[507]	2646	from trac import attachment
	2647	from trac import config as trac_config
	2648	from trac import util
	2649	from trac.core import TracError
[554]	2650	from trac.env import Environment
[507]	2651	from trac.perm import PermissionSystem
[539]	2652	from trac.perm import PermissionCache
[538]	2653	from trac.test import Mock, MockPerm
[507]	2654	from trac.ticket import Ticket
[554]	2655	from trac.ticket.api import TicketSystem
[507]	2656	from trac.ticket.web_ui import TicketModule
[541]	2657	from trac.web.href import Href
[368]	2658
[541]	2659	#
	2660	# return util.text.to_unicode(str)
	2661	#
	2662	# see http://projects.edgewall.com/trac/changeset/2799
	2663	from trac.ticket.notification import TicketNotifyEmail
	2664
[507]	2665	else:
	2666	logger.error('TRAC version %s is not supported' %version)
[519]	2667	sys.exit(0)
[182]	2668
[507]	2669	## Must be set before environment is created
	2670	#
	2671	if settings.has_key('python_egg_cache'):
	2672	python_egg_cache = str(settings['python_egg_cache'])
	2673	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
[291]	2674
[507]	2675	if settings.debug > 0:
	2676	logger.debug('Loading environment %s', settings.project)
[359]	2677
[507]	2678	try:
	2679	env = Environment(settings['project'], create=0)
	2680	except IOError, detail:
[534]	2681	logger.error("trac error: %s" %detail)
[519]	2682	sys.exit(0)
[333]	2683
[507]	2684	tktparser = TicketEmailParser(env, settings, logger, float(version))
	2685	tktparser.parse(sys.stdin)
[22]	2686
[507]	2687	## Catch all errors and use the logging module
	2688	#
	2689	except Exception, error:
[187]	2690
[507]	2691	etype, evalue, etb = sys.exc_info()
	2692	for e in traceback.format_exception(etype, evalue, etb):
	2693	logger.critical(e)
[187]	2694
[507]	2695	if m:
[528]	2696	tktparser.save_email_for_debug(m, settings.project_name, True)
[97]	2697
[507]	2698	sys.exit(1)
[22]	2699	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format