source: trunk/email2trac.py.in @ 671

Last change on this file since 671 was 671, checked in by bas, 8 years ago

finally finalized the trac wildcard address support, see #297

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 94.4 KB
Line 
1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
5#       Copyright 2002 SURFsara
6#
7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
10#
11#       http://www.apache.org/licenses/LICENSE-2.0
12#
13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
21"""
22email2trac.py -- Email -> TRAC tickets
23
24A MTA filter to create Trac tickets from inbound emails.
25
26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
29
30Authors:
31  Bas van der Vlies <bas.vandervlies@surfsara.nl>
32  Walter de Jong <walter.dejong@surfsara.nl>
33
34How to use
35----------
36 * See https://oss.trac.surfsara.nl/email2trac/
37
38 * Commandline opions:
39                -h,--help
40                -A, --agilo
41                -B, --bh_product <name>
42                -d, --debug
43                -E, --virtualenv <path>
44                -f,--file  <configuration file>
45                -n,--dry-run
46                -p, --project <project name>
47                -t, --ticket_prefix <name>
48                -v, --verbose
49
50SVN Info:
51        $Id: email2trac.py.in 671 2015-10-19 10:18:11Z bas $
52"""
53import os
54import sys
55import string
56import getopt
57import time
58import email
59import email.Iterators
60import email.Header
61import re
62import urllib
63import unicodedata
64import mimetypes
65import traceback
66import logging
67import logging.handlers
68import UserDict
69import tempfile
70
71from datetime import timedelta, datetime
72from stat import *
73
74## Some global variables
75#
76m = None
77
78class SaraDict(UserDict.UserDict):
79    def __init__(self, dictin = None):
80        UserDict.UserDict.__init__(self)
81        self.name = None
82       
83        if dictin:
84            if dictin.has_key('name'):
85                self.name = dictin['name']
86                del dictin['name']
87            self.data = dictin
88           
89    def get_value(self, name):
90        if self.has_key(name):
91            return self[name]
92        else:
93            return None
94               
95    def __repr__(self):
96        return repr(self.data)
97
98    def __str__(self):
99        return str(self.data)
100           
101    def __getattr__(self, name):
102        """
103        override the class attribute get method. Return the value
104        from the dictionary
105        """
106        if self.data.has_key(name):
107            return self.data[name]
108        else:
109            return None
110           
111    def __setattr__(self, name, value):
112        """
113        override the class attribute set method only when the UserDict
114        has set its class attribute
115        """
116        if self.__dict__.has_key('data'):
117            self.data[name] = value
118        else:
119            self.__dict__[name] = value
120
121    def __iter__(self):
122        return iter(self.data.keys())
123
124
125
126class TicketEmailParser(object):
127    env = None
128    comment = '> '
129
130    def __init__(self, env, parameters, logger, version):
131        self.env = env
132
133        # Database connection
134        #
135        self.db = None
136
137        # Save parameters
138        #
139        self.parameters = parameters
140        self.logger = logger
141
142        # Some useful mail constants
143        #
144        self.email_name = None
145        self.email_addr = None
146        self.email_from = None
147        self.author     = None
148        self.id         = None
149
150        ## Will be set to True if the user has ben registered
151        #
152        self.allow_registered_user = False
153       
154        self.STRIP_CONTENT_TYPES = list()
155
156        ## fields properties via body_text
157        #
158        self.properties = dict()
159
160        self.VERSION = version
161
162        self.get_config = self.env.config.get
163
164        ## init function ##
165        #
166        self.setup_parameters()
167
168    def setup_parameters(self):
169        if self.parameters.umask:
170            os.umask(self.parameters.umask)
171
172        if not self.parameters.spam_level:
173            self.parameters.spam_level = 0
174
175        if not self.parameters.spam_header:
176            self.parameters.spam_header = 'X-Spam-Score'
177
178        if not self.parameters.email_quote:
179            self.parameters.email_quote = '^> .*'
180            self.parameters.skip_line_regex = '^> .*'
181
182        if not self.parameters.ticket_update_by_subject_lookback:
183            self.parameters.ticket_update_by_subject_lookback = 30
184
185        if self.parameters.verbatim_format == None:
186            self.parameters.verbatim_format = 1
187
188        if self.parameters.reflow == None:
189            self.parameters.reflow = 1
190
191        if self.parameters.binhex:
192            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
193
194        if self.parameters.applesingle:
195            self.STRIP_CONTENT_TYPES.append('application/applefile')
196
197        if self.parameters.appledouble:
198            self.STRIP_CONTENT_TYPES.append('application/applefile')
199
200        if self.parameters.strip_content_types:
201            items = self.parameters.strip_content_types.split(',')
202            for item in items:
203                self.STRIP_CONTENT_TYPES.append(item.strip())
204
205        if self.parameters.tmpdir:
206            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
207        else:
208            self.parameters.tmpdir = os.path.normcase('/tmp')
209
210        if self.parameters.email_triggers_workflow == None:
211            self.parameters.email_triggers_workflow = 1
212
213        if not self.parameters.subject_field_separator:
214            self.parameters.subject_field_separator = '&'
215        else:
216            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
217
218        if not self.parameters.strip_signature_regex:
219            self.parameters.strip_signature_regex = '^-- $'
220            self.parameters.cut_line_regex = '^-- $'
221
222        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
223        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
224        self.smtp_replyto = self.get_config('notification', 'smtp_replyto')
225        self.trac_smtp_always_cc = self.get_config('notification', 'smtp_always_cc')
226        self.trac_smtp_always_bcc = self.get_config('notification', 'smtp_always_bcc')
227
228        self.system = None
229
230########## Email Header Functions ###########################################################
231
232    def spam(self, message):
233        """
234        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
235        # Note if Spam_level then '*' are included
236        """
237        self.logger.debug('function spam')
238
239        spam = False
240        if message.has_key(self.parameters.spam_header):
241            spam_l = string.split(message[self.parameters.spam_header])
242            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
243
244            try:
245                number = spam_l[0].count('*')
246            except IndexError, detail:
247                number = 0
248               
249            if number >= self.parameters.spam_level:
250                spam = True
251               
252        ## treat virus mails as spam
253        #
254        elif message.has_key('X-Virus-found'):         
255            spam = True
256
257        ## Astaro firewall spam handling
258        #
259        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
260            spam = True
261
262        ## How to handle SPAM messages
263        #
264        if self.parameters.drop_spam and spam:
265
266            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
267            return 'drop'   
268
269        elif spam:
270
271            return 'Spam'   
272        else:
273
274            return False
275
276    def email_header_acl(self, keyword, message_field, default):
277        """
278        This function wil check if the email address is allowed or denied
279        to send mail to the ticket list
280        """
281        self.logger.debug('function email_header_acl: %s' %keyword)
282
283        try:
284            mail_addresses = self.parameters[keyword]
285
286            # Check if we have an empty string
287            #
288            if not mail_addresses:
289                return default
290
291        except KeyError, detail:
292            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
293
294            return default
295
296        mail_addresses = string.split(mail_addresses, ',')
297        message_addresses = string.split(message_field, ',')
298
299        for entry in mail_addresses:
300            entry = entry.strip()
301            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
302
303            for addr in message_addresses:
304                addr = addr.strip()
305
306                if self.parameters.compare_function_list in [ 'matches', 'match']:
307                    s = '\t%s matches %s' %(addr, entry)
308                    result =  TO_RE.match(addr)
309                else:
310                    s = '\t%s contains %s' %(addr, entry)
311                    result =  TO_RE.search(addr)
312
313                if result:
314                    self.logger.debug(s)
315                    return True
316
317        return False
318
319    def email_header_txt(self, m):
320        """
321        Display To and CC addresses in description field
322        """
323        s = ''
324
325        if m['To'] and len(m['To']) > 0:
326            s = "'''To:''' %s\r\n" %(m['To'])
327        if m['Cc'] and len(m['Cc']) > 0:
328            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
329
330        return  self.email_to_unicode(s)
331
332
333    def get_sender_info(self, message):
334        """
335        Get the default author name and email address from the message
336        """
337        self.logger.debug('function get_sender_info')
338
339        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
340        self.email_to_addrs = list()
341
342        for n,e in to_addrs:
343            self.email_to_addrs.append(e)
344
345        self.email_to_addrs = ','.join(self.email_to_addrs)
346
347
348        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
349
350        ## decode email name can contain charset
351        #
352        self.email_name = self.email_to_unicode(self.email_name)
353
354        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
355        self.logger.debug(dstr)
356
357        ## Trac can not handle author's name that contains spaces
358        #
359        if self.email_addr.lower() == self.trac_smtp_from.lower():
360            if self.email_name:
361                self.author = self.email_name
362            else:
363                self.author = "email2trac"
364        else:
365            self.author = self.email_addr
366
367        if self.parameters.ignore_trac_user_settings:
368            return
369
370        # Is this a registered user, use email address as search key:
371        # result:
372        #   u : login name
373        #   n : Name that the user has set in the settings tab
374        #   e : email address that the user has set in the settings tab
375        #
376        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
377            if (
378                (e and (e.lower() == self.email_addr.lower())) or
379                (u and (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower()))
380            )
381            ]
382
383        if len(users) >= 1:
384            self.email_from = users[0][2]
385            self.author = users[0][0]
386
387            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
388
389            if self.parameters.white_list_registered_users:
390                self.logger.debug('\tget_sender_info: added registered user to white list')
391                self.allow_registered_user = True
392                   
393
394    def set_cc_fields(self, ticket, message, update=False):
395        """
396        Set all the right fields for a new ticket
397        """
398        self.logger.debug('function set_cc_fields')
399
400
401        # Put all CC-addresses in ticket CC field
402        #
403        if self.parameters.reply_all:
404
405            ticket_cc = ''
406
407            msg_cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
408
409            if not msg_cc_addrs:
410                return
411
412            if update:
413                self.logger.debug("\tupdate ticket cc-field")
414                ticket_cc = ticket['cc']
415                ticket_cc_list = ticket_cc.split(',')
416
417            for name,addr in msg_cc_addrs:
418       
419                ## Prevent mail loop
420                #
421                if addr == self.trac_smtp_from:
422                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
423                    continue
424
425                ## Alwyas remove reporter email address
426                #
427                elif addr == self.email_addr:
428                    self.logger.debug("\tSkipping reporter email address for CC-field")
429                    continue
430
431                ## Always remove the always_cc address
432                #
433                elif addr == self.trac_smtp_always_cc:
434                    self.logger.debug("\tSkipping smtp_always_cc email address for CC-field")
435                    continue
436
437                ## Always remove the always_bcc address
438                #
439                elif addr == self.trac_smtp_always_bcc:
440                    self.logger.debug("\tSkipping smtp_always_bcc email address for CC-field")
441                    continue
442
443                ## Skip also addr in cc_black_list (email2trac.conf)
444                #
445                if self.email_header_acl('cc_black_list', addr, False):
446                    self.logger.debug("\tSkipping address for CC-field due to cc_black_list")
447                    continue
448
449                else:
450                    ## On update, prevent adding duplicates
451                    #
452                    if update:
453                      if addr in ticket_cc_list:
454                          continue
455                   
456                    if ticket_cc:
457                        ticket_cc = '%s, %s' %(ticket_cc, addr)
458                    else:
459                        ticket_cc = addr
460
461            if ticket_cc:
462                self.logger.debug('\tCC fields: %s' %ticket_cc)
463                ticket['cc'] = self.email_to_unicode(ticket_cc)
464
465
466    def acl_list_from_file(self, f, keyword):
467        """
468        Read the email address from a file
469        """
470        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
471       
472        if  not os.path.isfile(f):
473            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
474        else:
475            ## read whole file and replace '\n' with ''
476            #
477            addr_l = open(f, 'r').readlines()
478            s = ','.join(addr_l).replace('\n','')
479
480            try:
481                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
482            except KeyError, detail:
483                self.parameters[keyword] = s
484
485########## DEBUG functions  ###########################################################
486
487    def debug_body(self, message_body, temp_file):
488        """
489        """
490        self.logger.debug('function debug_body:')
491
492        body_file = "%s.body" %(temp_file)
493        fx = open(body_file, 'wb')
494
495        if self.parameters.dry_run:
496            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
497            return
498
499        self.logger.debug('writing body to %s' %(body_file))
500        if not message_body:
501                message_body = '(None)'
502
503        message_body = message_body.encode('utf-8')
504
505        fx.write(message_body)
506        fx.close()
507        try:
508            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
509        except OSError:
510            pass
511
512    def debug_attachments(self, message_parts, temp_file):
513        """
514        """
515        self.logger.debug('function debug_attachments')
516       
517        n = 0
518        for item in message_parts:
519
520            ## Skip inline text parts
521            #
522            if not isinstance(item, tuple):
523                continue
524               
525            (original, filename, part) = item
526            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
527
528            dummy_filename, ext = os.path.splitext(filename)
529
530            n = n + 1
531            part_name = 'part%d%s' %(n, ext)
532            part_file = "%s.%s" %(temp_file, part_name)
533
534            s = 'writing %s: filename: %s' %(part_file, filename)
535            self.print_unicode(s)
536
537            ## Forbidden chars, just write part files instead of names
538            #
539            #filename = filename.replace('\\', '_')
540            #filename = filename.replace('/', '_')
541            #filename = filename + '.att_email2trac'
542            # part_file = os.path.join(self.parameters.tmpdir, filename)
543            #part_file = util.text.unicode_quote(part_file)
544            #self.print_unicode(part_file)
545
546            if self.parameters.dry_run:
547                self.logger.info('DRY_RUN: NOT saving attachments')
548                continue
549
550            fx = open(part_file, 'wb')
551            text = part.get_payload(decode=1)
552
553            if not text:
554                text = '(None)'
555
556            fx.write(text)
557            fx.close()
558
559            try:
560                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
561            except OSError:
562                pass
563
564    def save_email_for_debug(self, message, project_name, save_only_raw_message=False):
565
566        if self.parameters.dry_run:
567            self.logger.debug('DRY_RUN: NOT saving email message')
568            return
569
570        (fd, tmp_file) = tempfile.mkstemp('.email2trac', project_name, self.parameters.tmpdir)
571        fx = os.fdopen(fd, 'wb')
572
573        self.logger.debug('saving email to %s' %(tmp_file))
574        fx.write('%s' % message)
575        fx.close()
576       
577        try:
578            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
579        except OSError:
580            pass
581
582        if not save_only_raw_message:
583            message_parts = self.get_message_parts(message)
584            message_parts = self.unique_attachment_names(message_parts)
585            body_text = self.get_body_text(message_parts)
586            self.debug_body(body_text, tmp_file)
587            self.debug_attachments(message_parts, tmp_file)
588
589########## Conversion functions  ###########################################################
590
591    def email_to_unicode(self, message_str):
592        """
593        Email has 7 bit ASCII code, convert it to unicode with the charset
594        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
595        understands it.
596        """
597        self.logger.debug("function email_to_unicode")
598
599        self.logger.debug("\t repr:%s type:%s" %(repr(message_str), type(message_str)))
600
601        ## Skip unicode strings, there are already converted
602        #
603        if type(message_str) is unicode:
604            return message_str
605
606        results =  email.Header.decode_header(message_str)
607
608        s = None
609        for text,format in results:
610            if format:
611                try:
612                    temp = unicode(text, format)
613                except UnicodeError, detail:
614                    # This always works
615                    #
616                    msg = 'ERROR: Could not find charset: %s, please install' %format
617                    self.logger.error(msg)
618                    temp = unicode(text, 'iso-8859-15')
619                except LookupError, detail:
620                    msg = 'ERROR: Could not find charset: %s, please install' %format
621                    self.logger.error(msg)
622                    #temp = unicode(text, 'iso-8859-15')
623                    temp = message_str
624                       
625            else:
626                temp = string.strip(text)
627                temp = unicode(text, 'iso-8859-15')
628
629            if s:
630                s = '%s %s' %(s, temp)
631            else:
632                s = '%s' %temp
633
634        return s
635
636    def str_to_dict(self, s):
637        """
638        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
639        """
640        self.logger.debug("function str_to_dict")
641
642        fields = string.split(s, self.parameters.subject_field_separator)
643
644        result = dict()
645        for field in fields:
646            try:
647                index, value = string.split(field, '=')
648
649                # We can not change the description of a ticket via the subject
650                # line. The description is the body of the email
651                #
652                if index.lower() in ['description']:
653                    continue
654
655                if value:
656                    result[index.lower()] = value
657
658            except ValueError:
659                pass
660        return result
661
662    def print_unicode(self,s):
663        """
664        This function prints unicode strings if possible else it will quote it
665        """
666        try:
667            self.logger.debug(s)
668        except UnicodeEncodeError, detail:
669            self.logger.debug(util.text.unicode_quote(s))
670
671
672    def html_2_txt(self, data):
673        """
674        Various routines to convert html syntax to valid trac wiki syntax
675        """
676        self.logger.debug('function html_2_txt')
677
678        ## This routine make an safe html that can be include
679        #  in trac, but no further text processing can be done
680        #
681#       try:
682#           from lxml.html.clean import Cleaner
683#           tags_rm = list()
684#           tags_rm.append('body')
685#
686#           cleaner = Cleaner(remove_tags=tags_rm )
687#           parsed_data = cleaner.clean_html(data)
688#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
689#
690#           return parsed_data
691#           
692#       except ImportError::
693#           pass
694
695        parsed_data = None
696        if self.parameters.html2text_cmd:
697            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
698            f = os.fdopen(fd, 'w')
699
700            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
701            self.logger.debug('\t html2text conversion %s'%(cmd))
702   
703            if self.parameters.dry_run:
704                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
705
706            else:
707                f.write(data)
708                f.close()
709
710                lines = os.popen(cmd).readlines()
711                parsed_data =  ''.join(lines)
712
713                os.unlink(tmp_file)
714
715        else:
716            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
717
718        return parsed_data
719
720    def check_filename_length(self, name):
721        """
722        To bypass a bug in Trac
723        check if the filename length is not larger then OS limit.
724          yes : return truncated filename
725          no  : return unmodified filename
726        """
727        self.logger.debug('function check_filename_length: ')
728
729        if not name:
730            return 'None'
731       
732        dummy_filename, ext = os.path.splitext(name)
733
734        ## Trac uses this format
735        #
736        try:
737            quote_format = util.text.unicode_quote(dummy_filename)
738
739        except UnicodeDecodeError, detail:
740            ## last resort convert to unicode
741            #
742            dummy_filename = util.text.to_unicode(dummy_filename)
743            quote_format = util.text.unicode_quote(dummy_filename)
744
745        ## Determine max filename length
746        #
747        try:
748            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
749        except AttributeError, detail:
750            filemax_length = 240
751
752        if len(quote_format) <= filemax_length:
753            return name
754
755        else:
756            ## Truncate file to filemax_length and reserve room for extension
757            #  We must break on a boundry
758            #
759            length = filemax_length - 6
760
761            for i in range(0,10):
762
763                truncated = quote_format[ : (length - i)]
764
765                try:
766                    unqoute_truncated = util.text.unicode_unquote(truncated)
767                    unqoute_truncated = unqoute_truncated + ext
768
769                    self.print_unicode('\t ' + unqoute_truncated)
770
771                    break
772
773                except UnicodeDecodeError, detail:
774                    continue
775
776
777            return unqoute_truncated
778
779########## TRAC ticket functions  ###########################################################
780
781    def mail_workflow(self, tkt):
782        """
783        """
784        self.logger.debug('function mail_workflow: ')
785       
786        req = Mock(authname=self.author, perm=MockPerm(), args={})
787        ticket_system = TicketSystem(self.env)
788       
789        try:
790            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
791
792        except KeyError:
793            ## fallback for compability (Will be deprecated)
794            #  workflow can be none.
795            #
796            workflow = None
797            if tkt['status'] in ['closed']:
798                workflow = self.parameters.workflow
799
800        if workflow:
801
802            ## process all workflow implementations
803            #
804            tkt_module = TicketModule(self.env)
805            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
806
807            for field in field_changes.keys():
808
809                ## We have already processed these fields
810                #
811                if not field in ['summary', 'description']:
812                    s = 'workflow : %s, field %s : %s, by %s' \
813                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
814                    self.logger.debug(s)
815
816                    tkt[field] = field_changes[field]['new']
817
818            return True
819
820        else:
821            return False
822
823    def check_permission_participants(self, tkt, action):
824        """
825        Check if the mailer is allowed to update the ticket
826        """
827        self.logger.debug('function check_permission_participants %s')
828
829        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
830            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
831
832            return True
833
834        perm = PermissionSystem(self.env)
835        if perm.check_permission(action, self.author):
836            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
837
838            return True
839       
840        # Is the updater in the CC?
841        try:
842            cc_list = tkt['cc'].split(',')
843            for cc in cc_list:
844                if self.email_addr.lower() in cc.lower().strip():
845                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
846
847                    return True
848
849        except KeyError:
850            pass
851
852        return False
853
854    def check_permission(self, tkt, action):
855        """
856        check if the reporter has the right permission for the action:
857          - TICKET_CREATE
858          - TICKET_MODIFY
859          - TICKET_APPEND
860          - TICKET_CHGPROP
861
862        There are three models:
863            - None      : no checking at all
864            - trac      : check the permission via trac permission model
865            - email2trac: ....
866        """
867        self.logger.debug("function check_permission: %s" %(action))
868
869        if self.parameters.ticket_permission_system in ['trac']:
870
871            perm = PermissionCache(self.env, self.author)
872            if perm.has_permission(action):
873                return True
874            else:
875                return False
876
877        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
878            return (self.check_permission_participants(tkt, action))   
879
880        ## Default is to allow everybody ticket updates and ticket creation
881        #
882        else:
883                return True
884
885
886    def update_ticket_fields(self, ticket, user_dict, new=None):
887        """
888        This will update the ticket fields. It will check if the
889        given fields are known and if the right values are specified
890        It will only update the ticket field value:
891            - If the field is known
892            - If the value supplied is valid for the ticket field.
893              If not then there are two options:
894               1) Skip the value (new=None)
895               2) Set default value for field (new=1)
896        """
897        self.logger.debug("function update_ticket_fields")
898
899        if self.parameters.bh_product:
900            if 'product' in user_dict:
901                if user_dict['product'] != ticket.env.product.prefix:
902                    self.logging.warning("bloodhound products cannot be changed "
903                                         "- ignoring")
904                user_dict.pop('product')
905
906        ## Check only permission model on ticket updates
907        #
908        if not new:
909            if self.parameters.ticket_permission_system:
910                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
911                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
912                    return False
913
914        ## Build a system dictionary from the ticket fields
915        #  with field as index and option as value
916        #
917        sys_dict = dict()
918
919        for field in ticket.fields:
920
921            try:
922                sys_dict[field['name']] = field['options']
923
924            except KeyError:
925                sys_dict[field['name']] = None
926                pass
927
928        ## Check user supplied fields an compare them with the
929        #  system one's
930        #
931        for field,value in user_dict.items():
932       
933            s = 'user_field\t %s = %s' %(field,value)
934            self.print_unicode(s)
935
936            if not field in sys_dict.keys(): 
937                self.logger.debug('%s is not a valid field for tickets' %(field))
938                continue
939
940            ## To prevent mail loop
941            #
942            if field == 'cc':
943
944                cc_list = user_dict['cc'].split(',')
945
946                if self.trac_smtp_from in cc_list:
947                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
948
949                    cc_list.remove(self.trac_smtp_from)
950
951                value = ','.join(cc_list)
952               
953
954            ## Check if every value is allowed for this field
955            #
956            if sys_dict[field]:
957
958                if value in sys_dict[field]:
959                    ticket[field] = value
960                else:
961                    ## Must we set a default if value is not allowed
962                    #
963                    if new:
964                        value = self.get_config('ticket', 'default_%s' %(field) )
965
966            else:
967
968                ticket[field] = value
969
970            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
971            self.print_unicode(s)
972
973    def ticket_update(self, m, id, spam):
974        """
975        If the current email is a reply to an existing ticket, this function
976        will append the contents of this email to that ticket, instead of
977        creating a new one.
978        """
979        self.logger.debug("function ticket_update")
980
981        if not self.parameters.ticket_update:
982            self.logger.debug("ticket_update disabled")
983            return False
984
985        ## Must we update ticket fields
986        #
987        update_fields = dict()
988        try:
989            id, keywords = string.split(id, '?')
990
991            update_fields = self.str_to_dict(keywords)
992
993            ## Strip '#'
994            #
995            self.id = int(id[1:])
996
997        except ValueError:
998
999            ## Strip '#'
1000            #
1001            self.id = int(id[1:])
1002
1003        self.logger.debug("\tticket id: %s" %id)
1004
1005        ## When is the change committed
1006        #
1007        if self.VERSION < 0.11:
1008            when = int(time.time())
1009        else:
1010            when = datetime.now(util.datefmt.utc)
1011
1012        try:
1013            tkt = Ticket(self.env, self.id, self.db)
1014
1015        except util.TracError, detail:
1016
1017            ## Not a valid ticket
1018            #
1019            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
1020            self.id = None
1021            return False
1022
1023        ## Check the permission of the reporter
1024        #
1025        if self.parameters.ticket_permission_system:
1026            if not self.check_permission(tkt, 'TICKET_APPEND'):
1027                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
1028                return False
1029
1030        ## How many changes has this ticket
1031        #
1032        grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1033        cnum = sum(1 for e in grouped) + 1
1034
1035
1036        ## reopen the ticket if it is was closed
1037        #  We must use the ticket workflow framework
1038        #
1039        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1040
1041            if not self.mail_workflow(tkt):
1042
1043                if tkt['status'] in ['closed']:
1044                    tkt['status'] = 'reopened'
1045                    tkt['resolution'] = ''
1046
1047        ## Old pre 0.11 situation
1048        #
1049        elif self.parameters.email_triggers_workflow:
1050
1051            self.logger.debug('\temail triggers workflow pre trac 0.11')
1052
1053            if tkt['status'] in ['closed']:
1054                tkt['status'] = 'reopened'
1055                tkt['resolution'] = ''
1056
1057        else:
1058            self.logger.debug('\temail triggers workflow disabled')
1059
1060        ## Must we update some ticket fields properties via subject line
1061        #
1062        if update_fields:
1063            self.update_ticket_fields(tkt, update_fields)
1064
1065
1066        message_parts = self.get_message_parts(m)
1067        message_parts = self.unique_attachment_names(message_parts)
1068
1069        ## Must we update some ticket fields properties via inline comments
1070        # in body_text
1071        #
1072        if self.properties:
1073                self.update_ticket_fields(tkt, self.properties)
1074
1075        ## Must we update the CC ticket field
1076        #
1077        self.set_cc_fields(tkt, m, update=True)
1078
1079        if self.parameters.email_header:
1080            message_parts.insert(0, self.email_header_txt(m))
1081
1082        body_text = self.get_body_text(message_parts)
1083
1084        error_with_attachments = self.attach_attachments(message_parts)
1085
1086        if body_text.strip() or update_fields or self.properties:
1087
1088            if self.parameters.dry_run:
1089                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1090                self.logger.info(s)
1091
1092            else:
1093                if error_with_attachments:
1094                    body_text = '%s\\%s' %(error_with_attachments, body_text)
1095
1096                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1097                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1098           
1099        if not spam:
1100            self.notify(tkt, False, when)
1101
1102        return True
1103
1104    def set_ticket_fields(self, ticket):
1105        """
1106        set the ticket fields to value specified
1107            - /etc/email2trac.conf with <prefix>_<field>
1108            - trac default values, trac.ini
1109        """
1110        self.logger.debug('function set_ticket_fields')
1111
1112        user_dict = dict()
1113
1114        for field in ticket.fields:
1115
1116            name = field['name']
1117
1118            ## default trac value
1119            #
1120            CUSTOM_FIELD = False
1121            if not field.get('custom'):
1122                value = self.get_config('ticket', 'default_%s' %(name) )
1123
1124                ## skip this field can only be set by email2trac.conf
1125                #
1126                if name in ['resolution']:
1127                    value = None
1128
1129            else:
1130                ##  Else get the default value for custom fields
1131                #
1132                CUSTOM_FIELD = True
1133                value = field.get('value')
1134                options = field.get('options')
1135
1136                if value and options and (value not in options):
1137                     value = options[int(value)]
1138   
1139            s = 'trac[%s] = %s' %(name, value)
1140            self.print_unicode(s)
1141
1142            ## email2trac.conf settings
1143            #
1144            prefix = self.parameters.ticket_prefix
1145            try:
1146                value = self.parameters['%s_%s' %(prefix, name)]
1147
1148                s = 'email2trac[%s] = %s ' %(name, value)
1149                self.print_unicode(s)
1150
1151            except KeyError, detail:
1152                pass
1153       
1154            if value:
1155                user_dict[name] = value
1156
1157                s = 'used %s = %s' %(name, value)
1158                self.print_unicode(s)
1159
1160            else:
1161                ## custom fields need some initialisation
1162                #
1163                if CUSTOM_FIELD:
1164                    user_dict[name] = ''
1165                   
1166        self.update_ticket_fields(ticket, user_dict, new=1)
1167
1168        if 'status' not in user_dict.keys():
1169            ticket['status'] = 'new'
1170
1171    def ticket_update_by_subject(self, subject):
1172        """
1173        This list of Re: prefixes is probably incomplete. Taken from
1174        wikipedia. Here is how the subject is matched
1175          - Re: <subject>
1176          - Re: (<Mail list label>:)+ <subject>
1177
1178        So we must have the last column
1179        """
1180        self.logger.debug('function ticket_update_by_subject')
1181
1182        found_id = None
1183        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1184               
1185            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1186            result = SUBJECT_RE.search(subject)
1187
1188            if result:
1189                ## This is a reply
1190                #
1191                orig_subject = result.group(1)
1192
1193                self.logger.debug('subject search string: %s' %(orig_subject))
1194
1195                cursor = self.db.cursor()
1196                summaries = [orig_subject, '%%: %s' % orig_subject]
1197
1198                ## Time resolution is in micoseconds
1199                #
1200                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1201
1202                if self.VERSION < 0.12:
1203                    lookback = util.datefmt.to_timestamp(search_date)
1204                else:
1205                    lookback = util.datefmt.to_utimestamp(search_date)
1206
1207                for summary in summaries:
1208                    self.logger.debug('Looking for summary matching: "%s"' % summary)
1209
1210                    sql = """SELECT id, reporter FROM ticket
1211                            WHERE changetime >= %s AND summary LIKE %s
1212                            ORDER BY changetime DESC"""
1213
1214                    cursor.execute(sql, [lookback, summary.strip()])
1215
1216                    for row in cursor:
1217
1218                        (matched_id, sql_reporter) = row
1219
1220                        ## Save first entry.
1221                        #
1222                        if not found_id:
1223                            found_id = matched_id
1224                           
1225                        ## If subject and reporter are the same. The we certainly have found the right ticket
1226                        #
1227                        if sql_reporter == self.author:
1228                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1229                            found_id = matched_id
1230                            break
1231
1232                    if found_id:
1233                        self.logger.debug('Found matching ticket id: %d' % found_id)
1234                        found_id = '#%d' % found_id
1235                        return (found_id, orig_subject)
1236                   
1237        return (found_id, subject)
1238
1239
1240    def new_ticket(self, msg, subject, spam, set_fields = None):
1241        """
1242        Create a new ticket
1243        """
1244        self.logger.debug('function new_ticket')
1245
1246        tkt = Ticket(self.env)
1247
1248        ## self.author can be email address of an username
1249        #
1250        tkt['reporter'] = self.author
1251
1252        self.set_cc_fields(tkt, msg)
1253
1254        self.set_ticket_fields(tkt)
1255
1256        ## Check the permission of the reporter
1257        #
1258        if self.parameters.ticket_permission_system:
1259            if not self.check_permission(tkt, 'TICKET_CREATE'):
1260                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1261                return False
1262
1263        ## Old style setting for component, will be removed
1264        #
1265        if spam:
1266            tkt['component'] = 'Spam'
1267
1268        elif self.parameters.has_key('component'):
1269            tkt['component'] = self.parameters['component']
1270
1271        if not msg['Subject']:
1272            tkt['summary'] = u'(No subject)'
1273        else:
1274            tkt['summary'] = subject
1275
1276
1277        if set_fields:
1278            rest, keywords = string.split(set_fields, '?')
1279
1280            if keywords:
1281                update_fields = self.str_to_dict(keywords)
1282                self.update_ticket_fields(tkt, update_fields)
1283
1284
1285        message_parts = self.get_message_parts(msg, True)
1286
1287        ## Must we update some ticket fields properties via body_text
1288        #
1289        if self.properties:
1290                self.update_ticket_fields(tkt, self.properties)
1291
1292        message_parts = self.unique_attachment_names(message_parts)
1293       
1294        ## produce e-mail like header
1295        #
1296        head = ''
1297        if self.parameters.email_header:
1298            head = self.email_header_txt(msg)
1299            message_parts.insert(0, head)
1300           
1301        body_text = self.get_body_text(message_parts)
1302
1303        tkt['description'] = body_text
1304
1305        ## When is the change committed
1306        #
1307        if self.VERSION < 0.11:
1308            when = int(time.time())
1309        else:
1310            when = datetime.now(util.datefmt.utc)
1311
1312        if self.parameters.dry_run:
1313            self.logger.info('DRY_RUN: tkt.insert()')
1314        else:
1315            self.id = tkt.insert()
1316   
1317        changed = False
1318        comment = ''
1319
1320        ## some routines in trac are dependend on ticket id
1321        #  like alternate notify template
1322        #
1323        if self.parameters.alternate_notify_template:
1324            tkt['id'] = self.id
1325            changed = True
1326
1327        ## Rewrite the description if we have mailto enabled
1328        #
1329        if self.parameters.mailto_link:
1330            changed = True
1331            comment = u'\nadded mailto line\n'
1332
1333            #mailto = self.html_mailto_link( m['Subject'])
1334            mailto = self.html_mailto_link(subject)
1335
1336            tkt['description'] = u'%s\r\n%s%s\r\n' \
1337                %(head, mailto, body_text)
1338   
1339        ## Save the attachments to the ticket   
1340        #
1341        error_with_attachments =  self.attach_attachments(message_parts)
1342
1343        if error_with_attachments:
1344            changed = True
1345            comment = '%s\n%s\n' %(comment, error_with_attachments)
1346
1347        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1348            if self.mail_workflow(tkt):
1349                changed = True
1350
1351        if changed:
1352            if self.parameters.dry_run:
1353                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1354                self.logger.info(s)
1355
1356            else:
1357                tkt.save_changes(tkt['reporter'], comment)
1358
1359        if not spam:
1360            self.notify(tkt, True)
1361
1362
1363    def attach_attachments(self, message_parts, update=False):
1364        '''
1365        save any attachments as files in the ticket's directory
1366        '''
1367        self.logger.debug('function attach_attachments()')
1368
1369        if self.parameters.dry_run:
1370            self.logger.debug("DRY_RUN: no attachments attached to tickets")
1371            return ''
1372
1373        count = 0
1374
1375        ## Get Maxium attachment size
1376        #
1377        max_size = int(self.get_config('attachment', 'max_size'))
1378        status   = None
1379       
1380        for item in message_parts:
1381            ## Skip body parts
1382            #
1383            if not isinstance(item, tuple):
1384                continue
1385               
1386            (original, filename, part) = item
1387
1388            ## We have to determine the size so we use this temporary solution.
1389            #
1390            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1391            text = part.get_payload(decode=1)
1392            if not text:
1393                text = '(None)'
1394            fd.write(text)
1395            fd.close()
1396
1397            ## get the file_size
1398            #
1399            stats = os.lstat(path)
1400            file_size = stats[ST_SIZE]
1401
1402            ## Check if the attachment size is allowed
1403            #
1404            if (max_size != -1) and (file_size > max_size):
1405                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1406                    %(status, original, file_size, max_size)
1407
1408                os.unlink(path)
1409                continue
1410            else:
1411                count = count + 1
1412                   
1413            ## Insert the attachment
1414            #
1415            fd = open(path, 'rb')
1416            if self.system == 'discussion':
1417                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1418            elif self.system == 'blog':
1419                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1420            else:
1421                s = 'Attach %s to ticket %d' %(filename, self.id)
1422                self.print_unicode(s)
1423                att = attachment.Attachment(self.env, 'ticket', self.id)
1424 
1425            ## This will break the ticket_update system, the body_text is vaporized
1426            #  ;-(
1427            #
1428            if not update:
1429                att.author = self.author
1430                att.description = self.email_to_unicode('Added by email2trac')
1431
1432            try:
1433
1434                self.logger.debug('Insert atachment')
1435                att.insert(filename, fd, file_size)
1436
1437            except OSError, detail:
1438
1439                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1440                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1441
1442            ## Remove the created temporary filename
1443            #
1444            fd.close()
1445            os.unlink(path)
1446
1447        ## return error
1448        #
1449        return status
1450
1451########## Fullblog functions  #################################################
1452
1453    def blog(self, msg, subject, id, params):
1454        """
1455        The blog create/update function
1456        """
1457        ## import the modules
1458        #
1459        from tracfullblog.core import FullBlogCore
1460        from tracfullblog.model import BlogPost, BlogComment
1461
1462        ## instantiate blog core
1463        #
1464        blog = FullBlogCore(self.env)
1465        req = Mock(authname='anonymous', perm=MockPerm(), args={})
1466
1467        ## parameters from the subject
1468        #
1469        params = self.str_to_dict((params or '').lstrip('?'))
1470
1471        ## preferably get the time from the email date header, or else
1472        #  use current date and time
1473        date = email.Utils.parsedate_tz(msg.get('date'))
1474        if date:
1475            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1476        else:
1477            self.logger.warn("No valid date header found")
1478            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1479
1480        ## blog entry affected
1481        #
1482        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1483
1484        ## check wether a blog post exists
1485        #
1486        post = BlogPost(self.env, self.id)
1487        force_update = self.properties.get('update', params.get('update'))
1488
1489        ## message parts
1490        #
1491        message_parts = self.get_message_parts(msg)
1492        message_parts = self.unique_attachment_names(message_parts)
1493
1494        if post.get_versions() and not force_update:
1495
1496            ## add comment to blog entry
1497            #
1498            comment = BlogComment(self.env, self.id)
1499            comment.author = self.properties.get('author', params.get('author', self.author))
1500            comment.comment = self.get_body_text(message_parts)
1501            comment.time = dt
1502
1503            if self.parameters.dry_run:
1504                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1505                return
1506            warnings = blog.create_comment(req, comment)
1507
1508        else:
1509            ## create or update blog entry
1510            #
1511            post.author = self.properties.get('author', params.get('author', self.author))
1512            post.categories = self.properties.get('categories', params.get('categories', ''))
1513            post.title = subject.strip()
1514            post.publish_time = dt
1515            post.body = self.get_body_text(message_parts)
1516           
1517            if self.parameters.dry_run:
1518                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1519                return
1520            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1521
1522        ## check for problems
1523        #
1524        if warnings:
1525            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1526       
1527        ## all seems well, attach attachments
1528        #
1529        self.attach_attachments(message_parts)
1530
1531
1532########## Discussion functions  ##############################################
1533
1534    def discussion_topic(self, content, subject):
1535
1536        ## Import modules.
1537        #
1538        from tracdiscussion.api import DiscussionApi
1539        from trac.util.datefmt import to_timestamp, utc
1540
1541        self.logger.debug('Creating a new topic in forum:', self.id)
1542
1543        ## Get dissussion API component.
1544        #
1545        api = self.env[DiscussionApi]
1546        args = {'forum' : self.id}
1547        context = self._create_context(api, args, content, subject)
1548
1549        ## Get forum for new topic.
1550        #
1551        forum = context.forum
1552
1553        if not forum:
1554            self.logger.error("ERROR: Replied forum doesn't exist")
1555
1556        ## Prepare topic.
1557        #
1558        topic = {'forum' : forum['id'],
1559                 'subject' : context.subject,
1560                 'time': to_timestamp(datetime.now(utc)),
1561                 'author' : self.author,
1562                 'subscribers' : [self.email_addr],
1563                 'body' : self.get_body_text(context.content_parts)}
1564
1565        ## Add topic to DB and commit it.
1566        #
1567        self._add_topic(api, context, topic)
1568
1569    def discussion_topic_reply(self, content, subject):
1570
1571        ## Import modules.
1572        #
1573        from tracdiscussion.api import DiscussionApi
1574        from trac.util.datefmt import to_timestamp, utc
1575
1576        self.logger.debug('Replying to discussion topic', self.id)
1577
1578        ## Get dissussion API component.
1579        #
1580        api = self.env[DiscussionApi]
1581        args = {'topic' : self.id}
1582        context = self._create_context(api, args, content, subject)
1583
1584        ## Get replied topic.
1585        #
1586        topic = context.topic
1587
1588        if not topic:
1589            self.logger.error("ERROR: Replied topic doesn't exist")
1590
1591        ## Prepare message.
1592        #
1593        message = {'forum' : topic['forum'],
1594                   'topic' : topic['id'],
1595                   'replyto' : -1,
1596                   'time' : to_timestamp(datetime.now(utc)),
1597                   'author' : self.author,
1598                   'body' : self.get_body_text(context.content_parts)}
1599
1600        ## Add message to DB and commit it.
1601        #
1602        self._add_message(api, context, message)
1603
1604    def discussion_message_reply(self, content, subject):
1605
1606        ## Import modules.
1607        #
1608        from tracdiscussion.api import DiscussionApi
1609        from trac.util.datefmt import to_timestamp, utc
1610
1611        self.logger.debug('Replying to discussion message', self.id)
1612
1613        ## Get dissussion API component.
1614        #
1615        api = self.env[DiscussionApi]
1616        args = {'message' : self.id}
1617        context = self._create_context(api, args, content, subject)
1618
1619        ## Get replied message.
1620        #
1621        message = context.message
1622
1623        if not message:
1624            self.logger.error("ERROR: Replied message doesn't exist")
1625
1626        ## Prepare message.
1627        #
1628        message = {'forum' : message['forum'],
1629                   'topic' : message['topic'],
1630                   'replyto' : message['id'],
1631                   'time' : to_timestamp(datetime.now(utc)),
1632                   'author' : self.author,
1633                   'body' : self.get_body_text(context.content_parts)}
1634
1635        ## Add message to DB and commit it.
1636        #
1637        self._add_message(api, context, message)
1638
1639    def _create_context(self, api, args, content, subject):
1640
1641        ## Import modules.
1642        #
1643        from trac.mimeview import Context
1644        from trac.web.api import Request
1645        from trac.web.session import Session
1646        from trac.perm import PermissionCache
1647
1648        ## TODO: Read server base URL from config.
1649        #  Create request object to mockup context creation.
1650        #
1651        environ = {'SERVER_PORT' : 80,
1652                   'SERVER_NAME' : 'test',
1653                   'REQUEST_METHOD' : 'POST',
1654                   'wsgi.url_scheme' : 'http',
1655                   'wsgi.input' : sys.stdin}
1656        chrome =  {'links': {},
1657                   'scripts': [],
1658                   'ctxtnav': [],
1659                   'warnings': [],
1660                   'notices': []}
1661
1662        if self.env.base_url_for_redirect:
1663            environ['trac.base_url'] = self.env.base_url
1664
1665        req = Request(environ, None)
1666        req.chrome = chrome
1667        req.tz = 'missing'
1668        req.authname = self.author
1669        req.perm = PermissionCache(self.env, self.author)
1670        req.locale = None
1671        req.args = args
1672        req.session = Session(env, req)
1673
1674        ## Create and return context.
1675        #
1676        context = Context.from_request(req)
1677        context.realm = 'discussion-email2trac'
1678        context.db = self.env.get_db_cnx()
1679        context.content = content
1680        context.subject = subject
1681
1682        ## Read content parts from content.
1683        #
1684        context.content_parts = self.get_message_parts(content)
1685        context.content_parts = self.unique_attachment_names(
1686          context.content_parts)
1687
1688        api._prepare_context(context)
1689
1690        return context
1691
1692    def _add_topic(self, api, context, topic):
1693        context.req.perm.assert_permission('DISCUSSION_APPEND')
1694
1695        ## Filter topic.
1696        #
1697        for discussion_filter in api.discussion_filters:
1698            accept, topic_or_error = discussion_filter.filter_topic(
1699              context, topic)
1700            if accept:
1701                topic = topic_or_error
1702            else:
1703                raise TracError(topic_or_error)
1704
1705        ## Add a new topic.
1706        #
1707        new_topic_id = api.add_topic(context, topic)
1708
1709        ## Get inserted topic with new ID.
1710        #
1711        topic = api.get_topic(context, new_topic_id)
1712
1713        ## Attach attachments.
1714        #
1715        self.id = topic['id']
1716        self.attach_attachments(context.content_parts, True)
1717
1718        ## Notify change listeners.
1719        #
1720        for listener in api.topic_change_listeners:
1721            listener.topic_created(context, topic)
1722
1723    def _add_message(self, api, context, message):
1724        context.req.perm.assert_permission('DISCUSSION_APPEND')
1725
1726        ## Filter message.
1727        #
1728        for discussion_filter in api.discussion_filters:
1729            accept, message_or_error = discussion_filter.filter_message(
1730              context, message)
1731            if accept:
1732                message = message_or_error
1733            else:
1734                raise TracError(message_or_error)
1735
1736        ## Add message.
1737        #
1738        new_msg_id = api.add_message(context, message)
1739
1740        ## Get inserted message with new ID.
1741        #
1742        message = api.get_message(context, new_msg_id)
1743
1744        ## Attach attachments.
1745        #
1746
1747        self.attach_attachments(context.content_parts, True)
1748
1749        ## Notify change listeners.
1750        #
1751        for listener in api.message_change_listeners:
1752            listener.message_created(context, message)
1753
1754########## MAIN function  ######################################################
1755
1756    def parse(self, fp):
1757        """
1758        """
1759        self.logger.debug('Main function parse')
1760        global m
1761
1762        m = email.message_from_file(fp)
1763       
1764        if not m:
1765            self.logger.debug('This is not a valid email message format')
1766            return
1767           
1768        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1769        #
1770        try:
1771            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1772        except AttributeError, detail:
1773            pass
1774
1775        if self.parameters.debug: # save email + try to decode message part
1776            self.save_email_for_debug(m, self.parameters.project_name)
1777        elif self.parameters.save_raw_message:     # save only the raw e-mail message text
1778            self.save_email_for_debug(m, self.parameters.project_name, True)
1779
1780        if self.VERSION <= 0.11:
1781            self.db = self.env.get_db_cnx()
1782        else:
1783            self.db = self.env.get_read_db()
1784
1785        self.get_sender_info(m)
1786
1787
1788        if self.parameters.white_list_file:
1789            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1790
1791        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1792
1793            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1794            return False
1795
1796        if self.email_header_acl('black_list', self.email_addr, False):
1797            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1798            return False
1799
1800        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1801            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1802            return False
1803
1804        ## If spam drop the message
1805        #
1806        if self.spam(m) == 'drop':
1807            return False
1808
1809        elif self.spam(m) == 'spam':
1810            spam_msg = True
1811        else:
1812            spam_msg = False
1813
1814        if not m['Subject']:
1815            subject  = 'No Subject'
1816        else:
1817            subject  = self.email_to_unicode(m['Subject'])
1818
1819        ## Bug in python logging module <2.6
1820        #
1821        self.logger.info('subject: %s' %repr(subject))
1822
1823        ## First try the new method and then fall back to old method
1824        #
1825        if not self.parse_delivered_to_field(m, subject, spam_msg):
1826            self.parse_subject_field(m, subject, spam_msg)
1827
1828    def parse_delivered_to_field(self, m, subject, spam_msg):
1829        """
1830        See if we have replied to an existing ticket
1831        """
1832        self.logger.debug('function parse_delivered_to_field')
1833
1834        ## Ticket id is in Delivered-To Field. Trac notify must send this
1835        #  eg: is+390@surfsara.nl
1836       
1837        ## Only run if this parameter is se
1838        #
1839        if not self.parameters.recipient_delimiter:
1840            return False
1841
1842        try:
1843
1844            #print self.parameters.project_name
1845            self.logger.debug('Delivered To: %s' %m['Delivered-To'])
1846
1847            id = m['Delivered-To']
1848            id = id.split(self.parameters.recipient_delimiter)[1]
1849            id = id.split('@')[0]
1850
1851            self.logger.debug('\t Found ticket id: %s' %id)
1852
1853            ## The ticket_update expects a # in front of the ticket id
1854            #
1855            id = "#%s" %(id)
1856
1857            ## true if ticket update
1858            #
1859            return self.ticket_update(m, id, spam_msg)
1860
1861        except KeyError, detail:
1862            pass
1863        except IndexError, detail:
1864            pass
1865
1866        return False
1867
1868
1869    def parse_subject_field(self, m, subject, spam_msg):
1870        """
1871        """
1872        self.logger.debug('function parse_subject_header')
1873
1874        ## [hic] #1529: Re: LRZ
1875        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1876        #
1877        ticket_regex = r'''
1878            (?P<new_fields>[#][?].*)
1879            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?[^:]*)?:)
1880            '''
1881
1882        ## Check if  FullBlogPlugin is installed
1883        #
1884        blog_enabled = None
1885        blog_regex = ''
1886        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1887            self.logger.debug('Trac BLOG support enabled')
1888            blog_enabled = True
1889            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
1890
1891
1892        ## Check if DiscussionPlugin is installed
1893        #
1894        discussion_enabled = None
1895        discussion_regex = ''
1896        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1897            self.logger.debug('Trac Discussion support enabled')
1898            discussion_enabled = True
1899            discussion_regex = r'''
1900            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1901            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1902            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1903            '''
1904
1905
1906        regex_str = ticket_regex + blog_regex + discussion_regex
1907        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1908
1909        ## Find out if this is a ticket, a blog or a discussion
1910        #
1911        result =  SYSTEM_RE.search(subject)
1912
1913        if result:
1914            ## update ticket + fields
1915            #
1916            if result.group('reply'):
1917                self.system = 'ticket'
1918
1919                ## Skip the last ':' character
1920                #
1921                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1922                    self.new_ticket(m, subject, spam_msg)
1923
1924            ## New ticket + fields
1925            #
1926            elif result.group('new_fields'):
1927                self.system = 'ticket'
1928                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1929
1930            if blog_enabled:
1931                if result.group('blog'):
1932                    self.system = 'blog'
1933                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1934
1935            if discussion_enabled:
1936                ## New topic.
1937                #
1938                if result.group('forum'):
1939                    self.system = 'discussion'
1940                    self.id = int(result.group('forum_id'))
1941                    self.discussion_topic(m, subject[result.end('forum'):])
1942
1943                ## Reply to topic.
1944                #
1945                elif result.group('topic'):
1946                    self.system = 'discussion'
1947                    self.id = int(result.group('topic_id'))
1948                    self.discussion_topic_reply(m, subject[result.end('topic'):])
1949
1950                ## Reply to topic message.
1951                #
1952                elif result.group('message'):
1953                    self.system = 'discussion'
1954                    self.id = int(result.group('message_id'))
1955                    self.discussion_message_reply(m, subject[result.end('message'):])
1956
1957        else:
1958
1959            self.system = 'ticket'
1960            (matched_id, subject) = self.ticket_update_by_subject(subject)
1961
1962            if matched_id:
1963
1964                if not self.ticket_update(m, matched_id, spam_msg):
1965                    self.new_ticket(m, subject, spam_msg)
1966
1967            else:
1968                ## No update by subject, so just create a new ticket
1969                #
1970                self.new_ticket(m, subject, spam_msg)
1971
1972
1973########## BODY TEXT functions  ###########################################################
1974
1975    def strip_signature(self, text):
1976        """
1977        Strip signature from message, inspired by Mailman software
1978        """
1979        self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
1980
1981        body = []
1982
1983        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1984        for line in text.splitlines():
1985
1986            match = STRIP_RE.match(line)
1987            if match:
1988                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
1989                break
1990
1991            body.append(line)
1992
1993        return ('\n'.join(body))
1994
1995    def reflow(self, text, delsp = 0):
1996        """
1997        Reflow the message based on the format="flowed" specification (RFC 3676)
1998        """
1999        flowedlines = []
2000        quotelevel = 0
2001        prevflowed = 0
2002
2003        for line in text.splitlines():
2004            from re import match
2005           
2006            ## Figure out the quote level and the content of the current line
2007            #
2008            m = match('(>*)( ?)(.*)', line)
2009            linequotelevel = len(m.group(1))
2010            line = m.group(3)
2011
2012            ## Determine whether this line is flowed
2013            #
2014            if line and line != '-- ' and line[-1] == ' ':
2015                flowed = 1
2016            else:
2017                flowed = 0
2018
2019            if flowed and delsp and line and line[-1] == ' ':
2020                line = line[:-1]
2021
2022            ## If the previous line is flowed, append this line to it
2023            #
2024            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
2025                flowedlines[-1] += line
2026
2027            ## Otherwise, start a new line
2028            #
2029            else:
2030                flowedlines.append('>' * linequotelevel + line)
2031
2032            prevflowed = flowed
2033           
2034
2035        return '\n'.join(flowedlines)
2036
2037    def strip_quotes(self, text):
2038        """
2039        Strip quotes from message by Nicolas Mendoza
2040        """
2041        self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
2042
2043        body = []
2044
2045        STRIP_RE = re.compile( self.parameters.email_quote )
2046
2047        for line in text.splitlines():
2048
2049            try:
2050
2051                match = STRIP_RE.match(line)
2052                if match:
2053                    self.logger.debug('\t"%s "  matched, skipping rest of message' %line)
2054                    continue
2055
2056            except UnicodeDecodeError:
2057
2058                tmp_line = self.email_to_unicode(line)
2059
2060                match = STRIP_RE.match(tmp_line)
2061                if match:
2062                    self.logger.debug('\t"%s "  matched, skipping rest of message' %line)
2063                    continue
2064               
2065            body.append(line)
2066
2067        return ('\n'.join(body))
2068
2069    def inline_properties(self, text):
2070        """
2071        Parse text if we use inline keywords to set ticket fields
2072        """
2073        self.logger.debug('function inline_properties')
2074
2075        properties = dict()
2076        body = list()
2077
2078        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
2079
2080        for line in text.splitlines():
2081            match = INLINE_EXP.match(line)
2082            if match:
2083                keyword, value = match.groups()
2084
2085                if self.parameters.inline_properties_first_wins:
2086                    if keyword in self.properties.keys():
2087                        continue
2088
2089                self.properties[keyword] = value.strip()
2090                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
2091
2092            else:
2093                body.append(line)
2094               
2095        return '\n'.join(body)
2096
2097
2098    def wrap_text(self, text, replace_whitespace = False):
2099        """
2100        Will break a lines longer then given length into several small
2101        lines of size given length
2102        """
2103        import textwrap
2104
2105        LINESEPARATOR = '\n'
2106        reformat = ''
2107
2108        for s in text.split(LINESEPARATOR):
2109            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2110            if tmp:
2111                reformat = '%s\n%s' %(reformat,tmp)
2112            else:
2113                reformat = '%s\n' %reformat
2114
2115        return reformat
2116
2117        # Python2.4 and higher
2118        #
2119        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2120        #
2121
2122########## EMAIL attachements functions ###########################################################
2123
2124    def inline_part(self, part):
2125        """
2126        """
2127        self.logger.debug('function inline_part()')
2128
2129        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2130
2131    def get_message_parts(self, msg, new_email=False):
2132        """
2133        parses the email message and returns a list of body parts and attachments
2134        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2135        """
2136        self.logger.debug('function get_message_parts()')
2137
2138        message_parts = list()
2139   
2140        ALTERNATIVE_MULTIPART = False
2141
2142        for part in msg.walk():
2143            content_maintype = part.get_content_maintype()
2144            content_type =  part.get_content_type()
2145
2146            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2147            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2148
2149            ## Check content type
2150            #
2151            if content_type in self.STRIP_CONTENT_TYPES:
2152                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2153                continue
2154
2155            ## Catch some mulitpart execptions
2156            #
2157            if content_type == 'multipart/alternative':
2158                ALTERNATIVE_MULTIPART = True
2159                continue
2160
2161            ## Skip multipart containers
2162            #
2163            if content_maintype == 'multipart':
2164                self.logger.debug("\t Skipping multipart container")
2165                continue
2166           
2167            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2168            #  or if there is one and it says "inline"
2169            #
2170            inline = self.inline_part(part)
2171
2172            ## Drop HTML message
2173            #
2174            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2175
2176                if content_type == 'text/html':
2177                    self.logger.debug('\t Skipping alternative HTML message')
2178                    ALTERNATIVE_MULTIPART = False
2179                    continue
2180
2181
2182            #if self.VERSION < 1.0:
2183            #    filename = part.get_filename()
2184
2185            ## convert 7 bit filename to 8 bit unicode
2186            #
2187            raw_filename = part.get_filename()
2188            filename = self.email_to_unicode(raw_filename);
2189
2190            s = '\t unicode filename: %s' %(filename)
2191            self.print_unicode(s)
2192            self.logger.debug('\t raw filename: %s' %repr(raw_filename))
2193
2194            if self.VERSION < 1.0:
2195                filename = self.check_filename_length(filename)
2196
2197            ## Save all non plain text message as attachment
2198            #
2199            if not content_type in ['text/plain']:
2200
2201                message_parts.append( (filename, part) )
2202
2203                ## We only convert html messages
2204                #
2205                if not content_type == 'text/html':
2206                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2207                    continue
2208
2209
2210            ## We have an text or html message
2211            #
2212            if not inline:
2213                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2214                    message_parts.append( (filename, part) )
2215                    continue
2216               
2217            ## Try to decode message part. We have a html or plain text messafe
2218            #
2219            body_text = part.get_payload(decode=1)
2220            if not body_text:           
2221                body_text = part.get_payload(decode=0)
2222
2223            ## Try to convert html message
2224            #
2225            if content_type == 'text/html':
2226
2227                body_text = self.html_2_txt(body_text)
2228                if not body_text:
2229                    continue
2230
2231            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2232            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2233
2234            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2235                body_text = self.reflow(body_text, delsp == 'yes')
2236
2237            if new_email and self.parameters.only_strip_on_update:
2238                self.logger.debug('Skip signature/quote stripping for new messages')
2239            else:
2240                if self.parameters.strip_signature:
2241                    body_text = self.strip_signature(body_text)
2242
2243                if self.parameters.strip_quotes:
2244                    body_text = self.strip_quotes(body_text)
2245
2246            if self.parameters.inline_properties:
2247                body_text = self.inline_properties(body_text)
2248
2249            if self.parameters.use_textwrap:
2250                body_text = self.wrap_text(body_text)
2251
2252            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2253            #
2254            charset = part.get_content_charset()
2255            if not charset:
2256                charset = 'iso-8859-15'
2257
2258            try:
2259                ubody_text = unicode(body_text, charset)
2260
2261            except UnicodeError, detail:
2262                ubody_text = unicode(body_text, 'iso-8859-15')
2263
2264            except LookupError, detail:
2265                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2266
2267            if self.parameters.verbatim_format:
2268                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2269            else:
2270                message_parts.append('%s' %ubody_text)
2271
2272        return message_parts
2273       
2274    def unique_attachment_names(self, message_parts):
2275        """
2276        Make sure we have unique names attachments:
2277          - check if it contains illegal characters
2278          - Rename "None" filenames to "untitled-part"
2279        """
2280        self.logger.debug('function unique_attachment_names()')
2281        renamed_parts = []
2282        attachment_names = set()
2283
2284        for item in message_parts:
2285           
2286            ## If not an attachment, leave it alone
2287            #
2288            if not isinstance(item, tuple):
2289                renamed_parts.append(item)
2290                continue
2291               
2292            (filename, part) = item
2293
2294            ## If filename = None, use a default one
2295            #
2296            if filename in [ 'None']:
2297                filename = 'untitled-part'
2298                self.logger.info('\t Rename filename "None" to: %s' %filename)
2299
2300                ## Guess the extension from the content type, use non strict mode
2301                #  some additional non-standard but commonly used MIME types
2302                #  are also recognized
2303                #
2304                ext = mimetypes.guess_extension(part.get_content_type(), False)
2305                if not ext:
2306                    ext = '.bin'
2307
2308                filename = '%s%s' % (filename, ext)
2309
2310            ## Discard relative paths for windows/unix in attachment names
2311            #
2312            filename = filename.replace('\\', '_')
2313            filename = filename.replace('/', '_')
2314
2315            ## remove linefeed char
2316            #
2317            for forbidden_char in ['\r', '\n']:
2318                filename = filename.replace(forbidden_char,'')
2319
2320            ## We try to normalize the filename to utf-8 NFC if we can.
2321            #  Files uploaded from OS X might be in NFD.
2322            #  Check python version and then try it
2323            #
2324            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2325            #   try:
2326            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2327            #   except TypeError:
2328            #       pass
2329
2330            ## Make the filename unique for this ticket
2331            #
2332            num = 0
2333            unique_filename = filename
2334            dummy_filename, ext = os.path.splitext(filename)
2335
2336            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2337                num += 1
2338                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2339               
2340            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2341            self.print_unicode(s)
2342
2343            attachment_names.add(unique_filename)
2344
2345            renamed_parts.append((filename, unique_filename, part))
2346   
2347        return renamed_parts
2348           
2349           
2350    def attachment_exists(self, filename):
2351
2352        self.logger.debug("function attachment_exists")
2353
2354        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2355        self.print_unicode(s)
2356
2357        ## Do we have a valid ticket id
2358        #
2359        if not self.id:
2360            return False
2361
2362        try:
2363            if self.system == 'discussion':
2364
2365                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2366
2367            elif self.system == 'blog':
2368
2369                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2370
2371            else:
2372
2373                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2374
2375            return True
2376
2377        except attachment.ResourceNotFound:
2378
2379            return False
2380
2381########## TRAC Ticket Text ###########################################################
2382           
2383    def get_body_text(self, message_parts):
2384        """
2385        """
2386        self.logger.debug('function get_body_text()')
2387
2388        body_text = []
2389       
2390        for part in message_parts:
2391       
2392            ## Plain text part, append it
2393            #
2394            if not isinstance(part, tuple):
2395                body_text.extend(part.strip().splitlines())
2396                body_text.append("")
2397                continue
2398
2399            (original, filename, part) = part
2400            inline = self.inline_part(part)
2401
2402            ## Skip generation of attachment link if html is converted to text
2403            #
2404            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2405                s = 'Skipping attachment link for html part: %s' %(filename)
2406                self.print_unicode(s)
2407                continue
2408           
2409            if part.get_content_maintype() == 'image' and inline:
2410
2411                if self.system != 'discussion':
2412                    s = 'wiki image link for: %s' %(filename)
2413                    self.print_unicode(s)
2414                    body_text.append('[[Image(%s)]]' % filename)
2415
2416                body_text.append("")
2417
2418            else:
2419
2420                if self.system != 'discussion':
2421
2422                    s = 'wiki attachment link for: %s' %(filename)
2423                    self.print_unicode(s)
2424                    body_text.append('[attachment:"%s"]' % filename)
2425
2426                body_text.append("")
2427
2428        ## Convert list body_texts to string
2429        #
2430        body_text = '\r\n'.join(body_text)
2431        return body_text
2432
2433    def html_mailto_link(self, subject):
2434        """
2435        This function returns a HTML mailto tag with the ticket id and author email address
2436        """
2437        self.logger.debug("function html_mailto_link")
2438
2439        if not self.author:
2440            author = self.email_addr
2441        else:   
2442            author = self.author
2443
2444        if not self.parameters.mailto_cc:
2445            self.parameters.mailto_cc = ''
2446           
2447        ## Bug in urllib.quote function
2448        #
2449        if isinstance(subject, unicode):
2450            subject = subject.encode('utf-8')
2451
2452        ## use urllib to escape the chars
2453        #
2454        s = '%s?Subject=%s&cc=%s' %(
2455               urllib.quote(self.email_addr),
2456               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2457               urllib.quote(self.parameters.mailto_cc)
2458               )
2459
2460        if self.VERSION in [ 0.10 ]:
2461            s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2462        else:
2463            s = '[mailto:"%s" Reply to: %s]' %(s, author)
2464
2465        self.logger.debug("\tmailto link %s" %s)
2466        return s
2467
2468########## TRAC notify section ###########################################################
2469
2470    def notify(self, tkt, new=True, modtime=0):
2471        """
2472        A wrapper for the TRAC notify function. So we can use templates
2473        """
2474        self.logger.debug('function notify()')
2475
2476
2477        class Email2TracNotifyEmail(TicketNotifyEmail):
2478
2479            def __init__(self, env):
2480                TicketNotifyEmail.__init__(self, env)
2481                self.email2trac_notify_reporter = None
2482                self.email2trac_replyto = None
2483
2484            def send(self, torcpts, ccrcpts):
2485                #print 'Yes this works'
2486                dest = self.reporter or 'anonymous'
2487                hdrs = {}
2488                hdrs['Message-ID'] = self.get_message_id(dest, self.modtime)
2489                hdrs['X-Trac-Ticket-ID'] = str(self.ticket.id)
2490                hdrs['X-Trac-Ticket-URL'] = self.data['ticket']['link']
2491                if not self.newticket:
2492                    msgid = self.get_message_id(dest)
2493                    hdrs['In-Reply-To'] = msgid
2494                    hdrs['References'] = msgid
2495
2496
2497                if self.email2trac_notify_reporter:
2498                    if not self.email2trac_notify_reporter in torcpts:
2499                        torcpts.append(self.email2trac_notify_reporter)
2500
2501                if self.email2trac_replyto:
2502                    # use to rewrite reply to
2503                    # hdrs does not work, multiple reply addresses
2504                    #hdrs['Reply-To'] = 'bas.van.der.vlies@gmail.com'
2505                    self.replyto_email = self.email2trac_replyto
2506       
2507                NotifyEmail.send(self, torcpts, ccrcpts, hdrs)
2508
2509        if self.parameters.dry_run  :
2510                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2511                return
2512
2513        try:
2514
2515
2516            ## create false {abs_}href properties, to trick Notify()
2517            #
2518            if self.VERSION in [0.10]:
2519                self.env.abs_href = Href(self.get_config('project', 'url'))
2520                self.env.href = Href(self.get_config('project', 'url'))
2521
2522            tn = Email2TracNotifyEmail(self.env)
2523
2524            ## additionally append sender (regardeless of settings in trac.ini)
2525            #
2526            if self.parameters.notify_reporter:
2527
2528                self.logger.debug('\t Notify reporter set')
2529
2530                if not self.email_header_acl('notify_reporter_black_list', self.email_addr, False):
2531                    tn.email2trac_notify_reporter = self.email_addr
2532
2533            if self.parameters.notify_replyto_rewrite:
2534
2535                self.logger.debug('\t Notify replyto rewrite set to:%s' %self.parameters.notify_replyto_rewrite)
2536
2537                if self.parameters.notify_replyto_rewrite in ['use_mail_domain']:
2538                    self.logger.debug('\t\t use_mail_domain:%s' %self.smtp_default_domain)
2539                    tn.email2trac_replyto = '%s@%s' %(self.id, self.smtp_default_domain )
2540
2541                elif self.parameters.notify_replyto_rewrite in ['use_trac_smtp_replyto']:
2542                    self.logger.debug('\t\t use_trac_smtp_replyto delimiter:%s' %self.parameters.recipient_delimiter)
2543
2544                    ## handle addres with @ and without
2545                    #
2546                    dummy = self.smtp_replyto.split('@')
2547                    if len(dummy) > 1:
2548                        tn.email2trac_replyto = '%s%s%s@%s' %(dummy[0], self.parameters.recipient_delimiter, self.id, dummy[1])
2549                    else:
2550                        tn.email2trac_replyto = '%s%s%s' %(dummy[0], self.parameters.recipient_delimiter, self.id)
2551
2552            if self.parameters.alternate_notify_template:
2553
2554                if self.VERSION >= 0.11:
2555
2556                    from trac.web.chrome import Chrome
2557
2558                    if  self.parameters.alternate_notify_template_update and not new:
2559                        tn.template_name = self.parameters.alternate_notify_template_update
2560                    else:
2561                        tn.template_name = self.parameters.alternate_notify_template
2562
2563                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2564                       
2565                else:
2566
2567                    tn.template_name = self.parameters.alternate_notify_template
2568
2569            tn.notify(tkt, new, modtime)
2570
2571        except Exception, e:
2572            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2573
2574########## END Class Definition  ########################################################
2575
2576
2577########## Parse Config File  ###########################################################
2578
2579def ReadConfig(file, name):
2580    """
2581    Parse the config file
2582    """
2583    if not os.path.isfile(file):
2584        print 'File %s does not exist' %file
2585        sys.exit(1)
2586
2587    config = trac_config.Configuration(file)
2588   
2589    parentdir = config.get('DEFAULT', 'parentdir')
2590    sections = config.sections()
2591
2592    ## use some trac internals to get the defaults
2593    #
2594    tmp = config.parser.defaults()
2595    project =  SaraDict()
2596
2597    for option, value in tmp.items():
2598        try:
2599            project[option] = int(value)
2600        except ValueError:
2601            project[option] = value
2602
2603    if name:
2604        if name in sections:
2605            project =  SaraDict()
2606            for option, value in  config.options(name):
2607                try:
2608                    project[option] = int(value)
2609                except ValueError:
2610                    project[option] = value
2611
2612        elif not parentdir:
2613            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2614            print "or set parentdir in the [DEFAULT] section"
2615            sys.exit(1)
2616
2617    ## If parentdir then set project dir to parentdir + name
2618    #
2619    if not project.has_key('project'):
2620        if not parentdir:
2621            print "You must set project or parentdir in your configuration file"
2622            sys.exit(1)
2623        elif not name:
2624            print "You must configure a  project section in your configuration file"
2625        else:
2626            project['project'] = os.path.join(parentdir, name)
2627
2628    ##
2629    # Save the project name
2630    #
2631    project['project_name'] = os.path.basename(project['project'])
2632
2633    return project
2634
2635########## Setup Logging ###############################################################
2636
2637def setup_log(parameters, project_name, interactive=None):
2638    """
2639    Setup logging
2640
2641    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2642    would be interpreted by the ConfigParser itself.
2643    """
2644    logger = logging.getLogger('email2trac %s' %project_name)
2645
2646    if interactive:
2647        parameters.log_type = 'stderr'
2648
2649    if not parameters.log_type:
2650        if sys.platform in ['win32', 'cygwin']:
2651            parameters.log_type = 'eventlog'
2652        else:
2653            parameters.log_type = 'syslog'
2654
2655    if parameters.log_type == 'file':
2656
2657        if not parameters.log_file:
2658            parameters.log_file = 'email2trac.log'
2659
2660        if not os.path.isabs(parameters.log_file):
2661            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2662
2663        log_handler = logging.FileHandler(parameters.log_file)
2664
2665    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2666        ## Requires win32 extensions
2667        #
2668        logid = "email2trac"
2669        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2670
2671    elif parameters.log_type in ('syslog', 'unix'):
2672        log_handler = logging.handlers.SysLogHandler('/dev/log')
2673
2674    elif parameters.log_type in ('stderr'):
2675        log_handler = logging.StreamHandler(sys.stderr)
2676
2677    else:
2678        log_handler = logging.handlers.BufferingHandler(0)
2679
2680    if parameters.log_format:
2681        parameters.log_format = parameters.log_format.replace('$(', '%(')
2682    else:
2683        parameters.log_format = '%(name)s: %(message)s'
2684        if parameters.log_type in ('file', 'stderr'):
2685            parameters.log_format = '%(asctime)s ' + parameters.log_format
2686
2687    log_formatter = logging.Formatter(parameters.log_format)
2688    log_handler.setFormatter(log_formatter)
2689    logger.addHandler(log_handler)
2690
2691    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2692        logger.setLevel(logging.DEBUG)
2693        parameters.debug = 1
2694
2695    elif parameters.log_level in ['INFO'] or parameters.verbose:
2696        logger.setLevel(logging.INFO)
2697
2698    elif parameters.log_level in ['WARNING']:
2699        logger.setLevel(logging.WARNING)
2700
2701    elif parameters.log_level in ['ERROR']:
2702        logger.setLevel(logging.ERROR)
2703
2704    elif parameters.log_level in ['CRITICAL']:
2705        logger.setLevel(logging.CRITICAL)
2706
2707    else:
2708        logger.setLevel(logging.INFO)
2709
2710    return logger
2711
2712########## Own TicketNotifyEmail class ###############################################################
2713
2714if __name__ == '__main__':
2715    ## Default config file
2716    #
2717    agilo = False
2718    bh_product = None
2719    configfile = '@email2trac_conf@'
2720    project = ''
2721    component = ''
2722    ticket_prefix = 'default'
2723    dry_run = None
2724    verbose = None
2725    debug_interactive = None
2726    virtualenv = '@virtualenv@'
2727
2728    SHORT_OPT = 'AB:cdE:hf:np:t:v'
2729    LONG_OPT  =  ['agilo', 'bh_product=', 'component=', 'debug',
2730                  'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=',
2731                  'virtualenv=', 'verbose']
2732
2733    try:
2734        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2735    except getopt.error,detail:
2736        print __doc__
2737        print detail
2738        sys.exit(1)
2739   
2740    project_name = None
2741    for opt,value in opts:
2742        if opt in [ '-h', '--help']:
2743            print __doc__
2744            sys.exit(0)
2745        elif opt in ['-A', '--agilo']:
2746            agilo = True
2747        elif opt in ['-B', '--bh_product']:
2748            bh_product = value
2749        elif opt in ['-c', '--component']:
2750            component = value
2751        elif opt in ['-d', '--debug']:
2752            debug_interactive = 1
2753        elif opt in ['-E', '--virtualenv']:
2754            virtualenv = value
2755        elif opt in ['-f', '--file']:
2756            configfile = value
2757        elif opt in ['-n', '--dry-run']:
2758            dry_run = True
2759        elif opt in ['-p', '--project']:
2760            project_name = value
2761        elif opt in ['-t', '--ticket_prefix']:
2762            ticket_prefix = value
2763        elif opt in ['-v', '--verbose']:
2764            verbose = True
2765
2766    if virtualenv and os.path.exists(virtualenv):
2767        activate_this = os.path.join(virtualenv, 'bin/activate_this.py')
2768        if os.path.exists(activate_this):
2769            execfile(activate_this, dict(__file__=activate_this))
2770
2771    try:
2772        from trac import __version__ as trac_version
2773        from trac import config as trac_config
2774
2775    except ImportError, detail:
2776        print detail
2777        print "Can not find a a valid trac installation, solutions could be:"
2778        print "\tset PYTHONPATH"
2779        print "\tuse the --virtualenv <dir> option"
2780        sys.exit(1)
2781   
2782    settings = ReadConfig(configfile, project_name)
2783
2784    ## The default prefix for ticket values in email2trac.conf
2785    #
2786    settings.ticket_prefix = ticket_prefix
2787    settings.dry_run = dry_run
2788    settings.verbose = verbose
2789
2790    if bh_product:
2791        settings.bh_product = bh_product
2792
2793    if not settings.debug and debug_interactive:
2794        settings.debug = debug_interactive
2795
2796    if not settings.project:
2797        print __doc__
2798        print 'No Trac project is defined in the email2trac config file.'
2799        sys.exit(1)
2800
2801    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2802   
2803    if component:
2804        settings['component'] = component
2805
2806    ## We are only interested in the major versions
2807    # 0.12.3 --> 0.12
2808    # 1.0.2  --> 1.0
2809    #
2810    l = trac_version.split('.')
2811    version = '.'.join(l[0:2])
2812
2813    logger.debug("Found trac version: %s" %(version))
2814   
2815    try:
2816        if version == '0.10':
2817            from trac import attachment
2818            from trac.env import Environment
2819            from trac.ticket import Ticket
2820            from trac.web.href import Href
2821            from trac import util
2822            from trac.ticket.web_ui import TicketModule
2823
2824            #
2825            # return  util.text.to_unicode(str)
2826            #
2827            # see http://projects.edgewall.com/trac/changeset/2799
2828            from trac.ticket.notification import TicketNotifyEmail
2829            from trac import config as trac_config
2830            from trac.core import TracError
2831
2832        elif version in ['0.11', '0.12', '0.13', '1.0', '1.1']:
2833            from trac import attachment
2834            from trac import config as trac_config
2835            from trac import util
2836            from trac.core import TracError
2837            from trac.env import Environment
2838            from trac.perm import PermissionSystem
2839            from trac.perm import PermissionCache
2840            from trac.test import Mock, MockPerm
2841            from trac.ticket.api import TicketSystem
2842            from trac.ticket.web_ui import TicketModule
2843            from trac.web.href import Href
2844
2845            try:
2846                import pkg_resources
2847                pkg = pkg_resources.get_distribution('BloodhoundMultiProduct')
2848                bloodhound = pkg.version.split()[:2]
2849            except pkg_resources.DistributionNotFound:
2850                # assume no bloodhound
2851                bloodhound = None
2852
2853            if bloodhound:
2854                from multiproduct.env import Environment, ProductEnvironment
2855                from multiproduct.ticket.web_ui import (ProductTicketModule
2856                                                        as TicketModule)
2857                logger.debug("Found Bloodhound Distribution")
2858
2859                if not settings.bh_product:
2860                    print __doc__
2861                    print 'No Bloodhound project defined (bh_project) in section:%s email2trac config file.' %(settings.project)
2862                    sys.exit(1)
2863
2864            if agilo:
2865
2866                try:
2867
2868                    from agilo.ticket.model import AgiloTicket as Ticket
2869
2870                except ImportError, detail:
2871
2872                    try:
2873                        from agilo.ticket.model import Ticket
2874                    except ImportError, detail:
2875                        logger.error('Could not find Trac  Agilo environemnt')
2876                        sys.exit(0)
2877
2878            else:
2879
2880                from trac.ticket import Ticket
2881
2882            #
2883            # return  util.text.to_unicode(str)
2884            #
2885            # see http://projects.edgewall.com/trac/changeset/2799
2886            from trac.ticket.notification import TicketNotifyEmail
2887            from trac.notification import NotifyEmail
2888
2889        else:
2890            logger.error('TRAC version %s is not supported' %version)
2891            sys.exit(0)
2892
2893        ## Must be set before environment is created
2894        #
2895        if settings.has_key('python_egg_cache'):
2896            python_egg_cache = str(settings['python_egg_cache'])
2897            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2898
2899        if settings.debug > 0:
2900            logger.debug('Loading environment %s', settings.project)
2901
2902        try:
2903            env = Environment(settings['project'], create=0)
2904
2905            if bloodhound:
2906                ## possibly overkill testing if the multiproduct schema is a
2907                # new enough version
2908                #
2909                from multiproduct.env import MultiProductSystem
2910                mps = MultiProductSystem(env)
2911                if mps.get_version() > 4:
2912                    try:
2913                        env = ProductEnvironment(env, settings.bh_product, create=0)
2914                    except LookupError:
2915                        logger.error('%s is not a valid Bloodhound' %settings.bh_product)
2916                        sys.exit(0)
2917               
2918        except IOError, detail:
2919            logger.error("trac error: %s" %detail)
2920            sys.exit(0)
2921        except TracError, detail:
2922            logger.error("trac error: %s" %detail)
2923            sys.exit(0)
2924
2925        tktparser = TicketEmailParser(env, settings, logger, float(version))
2926        tktparser.parse(sys.stdin)
2927
2928    ## Catch all errors and use the logging module
2929    #
2930    except Exception, error:
2931
2932        etype, evalue, etb = sys.exc_info()
2933        for e in traceback.format_exception(etype, evalue, etb):
2934            logger.critical(e)
2935
2936        if m:
2937            tktparser.save_email_for_debug(m, settings.project_name)
2938
2939        sys.exit(1)
2940
2941# EOB
Note: See TracBrowser for help on using the repository browser.