#!@PYTHON@ # Copyright (C) 2002 # # This file is part of the email2trac utils # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2, or (at your option) any # later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA # # For vi/emacs or other use tabstop=4 (vi: set ts=4) # """ email2trac.py -- Email tickets to Trac. A simple MTA filter to create Trac tickets from inbound emails. Copyright 2005, Daniel Lundin Copyright 2005, Edgewall Software Changed By: Bas van der Vlies Date : 13 September 2005 Descr. : Added config file and command line options, spam level detection, reply address and mailto option. Unicode support Changed By: Walter de Jong Descr. : multipart-message code and trac attachments The scripts reads emails from stdin and inserts directly into a Trac database. MIME headers are mapped as follows: * From: => Reporter => CC (Optional via reply_all option) * Subject: => Summary * Body => Description * Component => Can be set to SPAM via spam_level option How to use ---------- * Create an config file: [DEFAULT] # REQUIRED project : /data/trac/test # REQUIRED debug : 1 # OPTIONAL, if set print some DEBUG info spam_level : 4 # OPTIONAL, if set check for SPAM mail reply_all : 1 # OPTIONAL, if set then fill in ticket CC field umask : 022 # OPTIONAL, if set then use this umask for creation of the attachments mailto_link : 1 # OPTIONAL, if set then [mailto:<>] in description mailto_cc : basv@sara.nl # OPTIONAL, use this address as CC in mailto line ticket_update: 1 # OPTIONAL, if set then check if this is an update for a ticket trac_version : 0.9 # OPTIONAL, default is 0.10 [jouvin] # OPTIONAL project declaration, if set both fields necessary project : /data/trac/jouvin # use -p|--project jouvin. * default config file is : /etc/email2trac.conf * Commandline opions: -h | --help -c | --component= -f | --file= -p | --project= SVN Info: $Id: email2trac.py.in 187 2007-08-13 20:04:09Z bas $ """ import os import sys import string import getopt import stat import time import email import email.Iterators import email.Header import re import urllib import unicodedata import ConfigParser from stat import * import mimetypes import syslog import traceback from datetime import tzinfo, timedelta, datetime # Some global variables # trac_default_version = 0.10 m = None # A UTC class needed for trac version 0.11, added by # tbaschak at ktc dot mb dot ca # class UTC(tzinfo): """UTC""" ZERO = timedelta(0) HOUR = timedelta(hours=1) def utcoffset(self, dt): return self.ZERO def tzname(self, dt): return "UTC" def dst(self, dt): return self.ZERO class TicketEmailParser(object): env = None comment = '> ' def __init__(self, env, parameters, version): self.env = env # Database connection # self.db = None # Some useful mail constants # self.author = None self.email_addr = None self.email_from = None self.VERSION = version self.get_config = self.env.config.get if parameters.has_key('umask'): os.umask(int(parameters['umask'], 8)) if parameters.has_key('debug'): self.DEBUG = int(parameters['debug']) else: self.DEBUG = 0 if parameters.has_key('mailto_link'): self.MAILTO = int(parameters['mailto_link']) if parameters.has_key('mailto_cc'): self.MAILTO_CC = parameters['mailto_cc'] else: self.MAILTO_CC = '' else: self.MAILTO = 0 if parameters.has_key('spam_level'): self.SPAM_LEVEL = int(parameters['spam_level']) else: self.SPAM_LEVEL = 0 if parameters.has_key('email_comment'): self.comment = str(parameters['email_comment']) if parameters.has_key('email_header'): self.EMAIL_HEADER = int(parameters['email_header']) else: self.EMAIL_HEADER = 0 if parameters.has_key('alternate_notify_template'): self.notify_template = str(parameters['alternate_notify_template']) else: self.notify_template = None if parameters.has_key('reply_all'): self.REPLY_ALL = int(parameters['reply_all']) else: self.REPLY_ALL = 0 if parameters.has_key('ticket_update'): self.TICKET_UPDATE = int(parameters['ticket_update']) else: self.TICKET_UPDATE = 0 if parameters.has_key('drop_spam'): self.DROP_SPAM = int(parameters['drop_spam']) else: self.DROP_SPAM = 0 if parameters.has_key('verbatim_format'): self.VERBATIM_FORMAT = int(parameters['verbatim_format']) else: self.VERBATIM_FORMAT = 1 if parameters.has_key('strip_signature'): self.STRIP_SIGNATURE = int(parameters['strip_signature']) else: self.STRIP_SIGNATURE = 0 if parameters.has_key('use_textwrap'): self.USE_TEXTWRAP = int(parameters['use_textwrap']) else: self.USE_TEXTWRAP = 0 if parameters.has_key('python_egg_cache'): self.python_egg_cache = str(parameters['python_egg_cache']) os.environ['PYTHON_EGG_CACHE'] = self.python_egg_cache # Use OS independend functions # self.TMPDIR = os.path.normcase('/tmp') if parameters.has_key('tmpdir'): self.TMPDIR = os.path.normcase(str(parameters['tmpdir'])) # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_ # Note if Spam_level then '*' are included def spam(self, message): if message.has_key('X-Spam-Score'): spam_l = string.split(message['X-Spam-Score']) number = spam_l[0].count('*') if number >= self.SPAM_LEVEL: return 'Spam' elif message.has_key('X-Virus-found'): # treat virus mails as spam return 'Spam' return self.get_config('ticket', 'default_component') def blacklisted_from(self): FROM_RE = re.compile(r""" MAILER-DAEMON@ """, re.VERBOSE) result = FROM_RE.search(self.email_addr) if result: return True else: return False def email_to_unicode(self, message_str): """ Email has 7 bit ASCII code, convert it to unicode with the charset that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac understands it. """ results = email.Header.decode_header(message_str) str = None for text,format in results: if format: try: temp = unicode(text, format) except UnicodeError, detail: # This always works # temp = unicode(text, 'iso-8859-15') except LookupError, detail: #text = 'ERROR: Could not find charset: %s, please install' %format #temp = unicode(text, 'iso-8859-15') temp = message_str else: temp = string.strip(text) temp = unicode(text, 'iso-8859-15') if str: str = '%s %s' %(str, temp) else: str = '%s' %temp #str = str.encode('utf-8') return str def debug_attachments(self, message): n = 0 for part in message.walk(): if part.get_content_maintype() == 'multipart': # multipart/* is just a container print 'TD: multipart container' continue n = n + 1 print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type()) print 'TD: part%d: filename: %s' % (n, part.get_filename()) if part.is_multipart(): print 'TD: this part is multipart' payload = part.get_payload(decode=1) print 'TD: payload:', payload else: print 'TD: this part is not multipart' file = 'part%d' %n part_file = os.path.join(self.TMPDIR, file) #part_file = '/var/tmp/part%d' % n print 'TD: writing part%d (%s)' % (n,part_file) fx = open(part_file, 'wb') text = part.get_payload(decode=1) if not text: text = '(None)' fx.write(text) fx.close() try: os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO) except OSError: pass def email_header_txt(self, m): """ Display To and CC addresses in description field """ str = '' if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl': str = "'''To:''' %s [[BR]]" %(m['To']) if m['Cc'] and len(m['Cc']) > 0: str = "%s'''Cc:''' %s [[BR]]" % (str, m['Cc']) return self.email_to_unicode(str) def set_owner(self, ticket): """ Select default owner for ticket component """ #### return self.get_config('ticket', 'default_component') cursor = self.db.cursor() sql = "SELECT owner FROM component WHERE name='%s'" % ticket['component'] cursor.execute(sql) try: ticket['owner'] = cursor.fetchone()[0] except TypeError, detail: ticket['owner'] = None def get_author_emailaddrs(self, message): """ Get the default author name and email address from the message """ temp = self.email_to_unicode(message['from']) #print temp.encode('utf-8') self.author, self.email_addr = email.Utils.parseaddr(temp) #print self.author.encode('utf-8', 'replace') # Look for email address in registered trac users # users = [ u for (u, n, e) in self.env.get_known_users(self.db) if e == self.email_addr ] if len(users) == 1: self.email_from = users[0] else: self.email_from = self.email_to_unicode(message['from']) #self.email_from = self.email_to_unicode(self.email_addr) def set_reply_fields(self, ticket, message): """ Set all the right fields for a new ticket """ ticket['reporter'] = self.email_from # Put all CC-addresses in ticket CC field # if self.REPLY_ALL: #tos = message.get_all('to', []) ccs = message.get_all('cc', []) addrs = email.Utils.getaddresses(ccs) if not addrs: return # Remove reporter email address if notification is # on # if self.notification: try: addrs.remove((self.author, self.email_addr)) except ValueError, detail: pass for name,mail in addrs: try: mail_list = '%s, %s' %(mail_list, mail) except: mail_list = mail if mail_list: ticket['cc'] = self.email_to_unicode(mail_list) def save_email_for_debug(self, message, tempfile=False): if tempfile: import tempfile msg_file = tempfile.mktemp('.email2trac') else: #msg_file = '/var/tmp/msg.txt' msg_file = os.path.join(self.TMPDIR, 'msg.txt') print 'TD: saving email to %s' % msg_file fx = open(msg_file, 'wb') fx.write('%s' % message) fx.close() try: os.chmod(msg_file,S_IRWXU|S_IRWXG|S_IRWXO) except OSError: pass def str_to_dict(self, str): """ Transfrom a str of the form [=]+ to dict[] = """ # Skip the last ':' character # fields = string.split(str[:-1], ',') result = dict() for field in fields: try: index, value = string.split(field,'=') # We can not change the description of a ticket via the subject # line. The description is the body of the email # if index.lower() in ['description']: continue if value: result[index.lower()] = value except ValueError: pass return result def update_ticket_fields(self, ticket, user_dict): """ This will update the ticket fields when supplied via the subject mail line. It will only update the ticket field: - If the field is known - If the value supplied is valid for the ticket field - Else we skip it and no error is given """ # Build a system dictionary from the ticket fields # with field as index and option as value # sys_dict = dict() for field in ticket.fields: try: sys_dict[field['name']] = field['options'] except KeyError: sys_dict[field['name']] = None pass # Check user supplied fields an compare them with the # system one's # for field,value in user_dict.items(): if self.DEBUG >= 5: print 'user field : %s=%s' %(field,value) if sys_dict.has_key(field): if self.DEBUG >= 5: print 'sys field : ', sys_dict[field] # Check if value is an allowed system option, if TypeError then # every value is allowed # try: if value in sys_dict[field]: ticket[field] = value except TypeError: ticket[field] = value def ticket_update(self, m): """ If the current email is a reply to an existing ticket, this function will append the contents of this email to that ticket, instead of creating a new one. """ if not m['Subject']: return False else: subject = self.email_to_unicode(m['Subject']) # [hic] #1529: Re: LRZ # [hic] #1529?owner=bas,priority=medium: Re: LRZ # TICKET_RE = re.compile(r""" (?P[#][0-9]+:) |(?P[#][\d]+\?.*:) """, re.VERBOSE) result = TICKET_RE.search(subject) if not result: return False # Must we update ticket fields # update_tkt_fields = dict() try: nr, keywords = string.split(result.group('ticketnr_fields'), '?') update_tkt_fields = self.str_to_dict(keywords) # Strip '#' # ticket_id = int(nr[1:]) except AttributeError: # Strip '#' and ':' # nr = result.group('ticketnr') ticket_id = int(nr[1:-1]) # Get current time # when = int(time.time()) try: tkt = Ticket(self.env, ticket_id, self.db) except util.TracError, detail: return False # Must we update some ticket fields properties # if update_tkt_fields: self.update_ticket_fields(tkt, update_tkt_fields) body_text = self.get_body_text(m) if self.EMAIL_HEADER: head = self.email_header_txt(m) body_text = u"\r\n%s \r\n%s" %(head, body_text) #if self.MAILTO: # mailto = self.html_mailto_link(tkt['summary'], ticket_id, body_text) # body_text = u"\r\n%s \r\n%s" %(mailto, body_text) tkt.save_changes(self.author, body_text, when) tkt['id'] = ticket_id if self.VERSION == 0.9: str = self.attachments(m, tkt, True) else: str = self.attachments(m, tkt) if self.notification: self.notify(tkt, False, when) return True def new_ticket(self, msg): """ Create a new ticket """ tkt = Ticket(self.env) tkt['status'] = 'new' # Some defaults # tkt['milestone'] = self.get_config('ticket', 'default_milestone') tkt['priority'] = self.get_config('ticket', 'default_priority') tkt['severity'] = self.get_config('ticket', 'default_severity') tkt['version'] = self.get_config('ticket', 'default_version') if not msg['Subject']: tkt['summary'] = u'(No subject)' else: tkt['summary'] = self.email_to_unicode(msg['Subject']) if settings.has_key('component'): tkt['component'] = settings['component'] else: tkt['component'] = self.spam(msg) # Discard SPAM messages. # if self.DROP_SPAM and (tkt['component'] == 'Spam'): if self.DEBUG > 2 : print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL return False # Set default owner for component, HvB # Is not necessary, because if component is set. The trac code # will find the owner: self.set_owner(tkt) # self.set_reply_fields(tkt, msg) # produce e-mail like header # head = '' if self.EMAIL_HEADER > 0: head = self.email_header_txt(msg) body_text = self.get_body_text(msg) tkt['description'] = '\r\n%s\r\n%s' \ %(head, body_text) #when = int(time.time()) utc = UTC() when = datetime.now(utc) ticket_id = tkt.insert() #try: # ticket_id = tkt.insert() #except OperationalError, detail: # syslog.openlog('email2trac', syslog.LOG_NOWAIT) # syslog.syslog('catch tkt insert problem %s' %detail) # syslog.closelog() # # ticket_id = tkt.insert() tkt['id'] = ticket_id changed = False comment = '' # Rewrite the description if we have mailto enabled # if self.MAILTO: changed = True comment = u'\nadded mailto line\n' mailto = self.html_mailto_link(tkt['summary'], ticket_id, body_text) tkt['description'] = u'\r\n%s\r\n%s%s\r\n' \ %(head, mailto, body_text) str = self.attachments(msg, tkt) if str: changed = True comment = '%s\n%s\n' %(comment, str) if changed: tkt.save_changes(self.author, comment) #print tkt.get_changelog(self.db, when) if self.notification: self.notify(tkt, True) #self.notify(tkt, False) def parse(self, fp): global m m = email.message_from_file(fp) if not m: return if self.DEBUG > 1: # save the entire e-mail message text self.save_email_for_debug(m) self.debug_attachments(m) self.db = self.env.get_db_cnx() self.get_author_emailaddrs(m) if self.blacklisted_from(): if self.DEBUG > 1 : print 'Message rejected : From: in blacklist' return False if self.get_config('notification', 'smtp_enabled') in ['true']: self.notification = 1 else: self.notification = 0 # Must we update existing tickets # if self.TICKET_UPDATE > 0: if self.ticket_update(m): return True self.new_ticket(m) def strip_signature(self, text): """ Strip signature from message, inspired by Mailman software """ body = [] for line in text.splitlines(): if line == '-- ': break body.append(line) return ('\n'.join(body)) def wrap_text(self, text, replace_whitespace = False): """ Will break a lines longer then given length into several small lines of size given length """ import textwrap LINESEPARATOR = '\n' reformat = '' for s in text.split(LINESEPARATOR): tmp = textwrap.fill(s,self.USE_TEXTWRAP) if tmp: reformat = '%s\n%s' %(reformat,tmp) else: reformat = '%s\n' %reformat return reformat # Python2.4 and higher # #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR)) # def get_body_text(self, msg): """ put the message text in the ticket description or in the changes field. message text can be plain text or html or something else """ has_description = 0 encoding = True ubody_text = u'No plain text message' for part in msg.walk(): # 'multipart/*' is a container for multipart messages # if part.get_content_maintype() == 'multipart': continue if part.get_content_type() == 'text/plain': # Try to decode, if fails then do not decode # body_text = part.get_payload(decode=1) if not body_text: body_text = part.get_payload(decode=0) if self.STRIP_SIGNATURE: body_text = self.strip_signature(body_text) if self.USE_TEXTWRAP: body_text = self.wrap_text(body_text) # Get contents charset (iso-8859-15 if not defined in mail headers) # charset = part.get_content_charset() if not charset: charset = 'iso-8859-15' try: ubody_text = unicode(body_text, charset) except UnicodeError, detail: ubody_text = unicode(body_text, 'iso-8859-15') except LookupError, detail: ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset) elif part.get_content_type() == 'text/html': ubody_text = '(see attachment for HTML mail message)' else: ubody_text = '(see attachment for message)' has_description = 1 break # we have the description, so break if not has_description: ubody_text = '(see attachment for message)' # A patch so that the web-interface will not update the description # field of a ticket # ubody_text = ('\r\n'.join(ubody_text.splitlines())) # If we can unicode it try to encode it for trac # else we a lot of garbage # #if encoding: # ubody_text = ubody_text.encode('utf-8') if self.VERBATIM_FORMAT: ubody_text = '{{{\r\n%s\r\n}}}' %ubody_text else: ubody_text = '%s' %ubody_text return ubody_text def notify(self, tkt , new=True, modtime=0): """ A wrapper for the TRAC notify function. So we can use templates """ if tkt['component'] == 'Spam': return try: # create false {abs_}href properties, to trick Notify() # self.env.abs_href = Href(self.get_config('project', 'url')) self.env.href = Href(self.get_config('project', 'url')) tn = TicketNotifyEmail(self.env) if self.notify_template: tn.template_name = self.notify_template; tn.notify(tkt, new, modtime) except Exception, e: print 'TD: Failure sending notification on creation of ticket #%s: %s' %(tkt['id'], e) def mail_line(self, str): return '%s %s' % (self.comment, str) def html_mailto_link(self, subject, id, body): if not self.author: author = self.email_addr else: author = self.author # Must find a fix # #arr = string.split(body, '\n') #arr = map(self.mail_line, arr) #body = string.join(arr, '\n') #body = '%s wrote:\n%s' %(author, body) # Temporary fix # str = 'mailto:%s?Subject=%s&Cc=%s' %( urllib.quote(self.email_addr), urllib.quote('Re: #%s: %s' %(id, subject)), urllib.quote(self.MAILTO_CC) ) str = '\r\n{{{\r\n#!html\r\nReply to: %s\r\n}}}\r\n' %(str, author) return str def attachments(self, message, ticket, update=False): ''' save any attachments as files in the ticket's directory ''' count = 0 first = 0 number = 0 # Get Maxium attachment size # max_size = int(self.get_config('attachment', 'max_size')) status = '' for part in message.walk(): if part.get_content_maintype() == 'multipart': # multipart/* is just a container continue if not first: # first content is the message first = 1 if part.get_content_type() == 'text/plain': # if first is text, is was already put in the description continue filename = part.get_filename() if not filename: number = number + 1 filename = 'part%04d' % number ext = mimetypes.guess_extension(part.get_content_type()) if not ext: ext = '.bin' filename = '%s%s' % (filename, ext) else: filename = self.email_to_unicode(filename) # From the trac code # filename = filename.replace('\\', '/').replace(':', '/') filename = os.path.basename(filename) # We try to normalize the filename to utf-8 NFC if we can. # Files uploaded from OS X might be in NFD. # Check python version and then try it # if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3): try: filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') except TypeError: pass url_filename = urllib.quote(filename) # # Must be tuneables HvB # path, fd = util.create_unique_file(os.path.join(self.TMPDIR, url_filename)) text = part.get_payload(decode=1) if not text: text = '(None)' fd.write(text) fd.close() # get the file_size # stats = os.lstat(path) file_size = stats[stat.ST_SIZE] # Check if the attachment size is allowed # if (max_size != -1) and (file_size > max_size): status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \ %(status, filename, file_size, max_size) os.unlink(path) continue else: count = count + 1 # Insert the attachment # fd = open(path) att = attachment.Attachment(self.env, 'ticket', ticket['id']) # This will break the ticket_update system, the body_text is vaporized # ;-( # if not update: att.author = self.author att.description = self.email_to_unicode('Added by email2trac') att.insert(url_filename, fd, file_size) #except util.TracError, detail: # print detail # Remove the created temporary filename # fd.close() os.unlink(path) # Return how many attachments # status = 'This message has %d attachment(s)\n%s' %(count, status) return status def mkdir_p(dir, mode): '''do a mkdir -p''' arr = string.split(dir, '/') path = '' for part in arr: path = '%s/%s' % (path, part) try: stats = os.stat(path) except OSError: os.mkdir(path, mode) def ReadConfig(file, name): """ Parse the config file """ if not os.path.isfile(file): print 'File %s does not exist' %file sys.exit(1) config = ConfigParser.ConfigParser() try: config.read(file) except ConfigParser.MissingSectionHeaderError,detail: print detail sys.exit(1) # Use given project name else use defaults # if name: if not config.has_section(name): print "Not a valid project name: %s" %name print "Valid names: %s" %config.sections() sys.exit(1) project = dict() for option in config.options(name): project[option] = config.get(name, option) else: project = config.defaults() return project if __name__ == '__main__': # Default config file # configfile = '@email2trac_conf@' project = '' component = '' ENABLE_SYSLOG = 0 try: opts, args = getopt.getopt(sys.argv[1:], 'chf:p:', ['component=','help', 'file=', 'project=']) except getopt.error,detail: print __doc__ print detail sys.exit(1) project_name = None for opt,value in opts: if opt in [ '-h', '--help']: print __doc__ sys.exit(0) elif opt in ['-c', '--component']: component = value elif opt in ['-f', '--file']: configfile = value elif opt in ['-p', '--project']: project_name = value settings = ReadConfig(configfile, project_name) if not settings.has_key('project'): print __doc__ print 'No Trac project is defined in the email2trac config file.' sys.exit(1) if component: settings['component'] = component if settings.has_key('trac_version'): version = float(settings['trac_version']) else: version = trac_default_version if settings.has_key('enable_syslog'): ENABLE_SYSLOG = float(settings['enable_syslog']) #debug HvB #print settings try: if version == 0.9: from trac import attachment from trac.env import Environment from trac.ticket import Ticket from trac.web.href import Href from trac import util from trac.Notify import TicketNotifyEmail elif version == 0.10: from trac import attachment from trac.env import Environment from trac.ticket import Ticket from trac.web.href import Href from trac import util # # return util.text.to_unicode(str) # # see http://projects.edgewall.com/trac/changeset/2799 from trac.ticket.notification import TicketNotifyEmail elif version == 0.11: from trac import attachment from trac.env import Environment from trac.ticket import Ticket from trac.web.href import Href from trac import util # # return util.text.to_unicode(str) # # see http://projects.edgewall.com/trac/changeset/2799 from trac.ticket.notification import TicketNotifyEmail env = Environment(settings['project'], create=0) tktparser = TicketEmailParser(env, settings, version) tktparser.parse(sys.stdin) # Catch all errors ans log to SYSLOG if we have enabled this # else stdout # except Exception, error: if ENABLE_SYSLOG: syslog.openlog('email2trac', syslog.LOG_NOWAIT) etype, evalue, etb = sys.exc_info() for e in traceback.format_exception(etype, evalue, etb): syslog.syslog(e) syslog.closelog() else: traceback.print_exc() if m: tktparser.save_email_for_debug(m, True) # EOB