source: trunk/email2trac.py.in @ 614

Last change on this file since 614 was 614, checked in by bas, 11 years ago

fixed a typo in detecting which trac version is installed, see #303

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 85.9 KB
Line 
1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
5#       Copyright 2002 SARA
6#
7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
10#
11#       http://www.apache.org/licenses/LICENSE-2.0
12#
13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
21"""
22email2trac.py -- Email -> TRAC tickets
23
24A MTA filter to create Trac tickets from inbound emails.
25
26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
29
30Authors:
31  Bas van der Vlies <basv@sara.nl>
32  Walter de Jong <walter@sara.nl>
33
34How to use
35----------
36 * See https://subtrac.sara.nl/oss/email2trac/
37
38 * Commandline opions:
39                -h,--help
40                -d, --debug
41                -f,--file  <configuration file>
42                -n,--dry-run
43                -p, --project <project name>
44                -t, --ticket_prefix <name>
45
46SVN Info:
47        $Id: email2trac.py.in 614 2012-10-01 08:04:23Z bas $
48"""
49import os
50import sys
51import string
52import getopt
53import time
54import email
55import email.Iterators
56import email.Header
57import re
58import urllib
59import unicodedata
60import mimetypes
61import traceback
62import logging
63import logging.handlers
64import UserDict
65import tempfile
66
67from datetime import timedelta, datetime
68from stat import *
69
70try:
71    from trac import __version__ as trac_version
72    from trac import config as trac_config
73
74except ImportError, detail:
75    print "Can not find a a valid trac installation, try setting PYTHONPATH"
76    sys.exit(1)
77
78## Some global variables
79#
80m = None
81
82# This is to for the function AlwaysNotifyReporter
83sender_email = None
84
85class SaraDict(UserDict.UserDict):
86    def __init__(self, dictin = None):
87        UserDict.UserDict.__init__(self)
88        self.name = None
89       
90        if dictin:
91            if dictin.has_key('name'):
92                self.name = dictin['name']
93                del dictin['name']
94            self.data = dictin
95           
96    def get_value(self, name):
97        if self.has_key(name):
98            return self[name]
99        else:
100            return None
101               
102    def __repr__(self):
103        return repr(self.data)
104
105    def __str__(self):
106        return str(self.data)
107           
108    def __getattr__(self, name):
109        """
110        override the class attribute get method. Return the value
111        from the dictionary
112        """
113        if self.data.has_key(name):
114            return self.data[name]
115        else:
116            return None
117           
118    def __setattr__(self, name, value):
119        """
120        override the class attribute set method only when the UserDict
121        has set its class attribute
122        """
123        if self.__dict__.has_key('data'):
124            self.data[name] = value
125        else:
126            self.__dict__[name] = value
127
128    def __iter__(self):
129        return iter(self.data.keys())
130
131class TicketEmailParser(object):
132    env = None
133    comment = '> '
134
135    def __init__(self, env, parameters, logger, version):
136        self.env = env
137
138        # Database connection
139        #
140        self.db = None
141
142        # Save parameters
143        #
144        self.parameters = parameters
145        self.logger = logger
146
147        # Some useful mail constants
148        #
149        self.email_name = None
150        self.email_addr = None
151        self.email_from = None
152        self.author     = None
153        self.id         = None
154
155        ## Will be set to True if the user has ben registered
156        #
157        self.allow_registered_user = False
158       
159        self.STRIP_CONTENT_TYPES = list()
160
161        ## fields properties via body_text
162        #
163        self.properties = dict()
164
165        self.VERSION = version
166
167        self.get_config = self.env.config.get
168
169        ## init function ##
170        #
171        self.setup_parameters()
172
173    def setup_parameters(self):
174        if self.parameters.umask:
175            os.umask(self.parameters.umask)
176
177        if not self.parameters.spam_level:
178            self.parameters.spam_level = 0
179
180        if not self.parameters.spam_header:
181            self.parameters.spam_header = 'X-Spam-Score'
182
183        if not self.parameters.email_quote:
184            self.parameters.email_quote = '^> .*'
185            self.parameters.skip_line_regex = '^> .*'
186
187        if not self.parameters.ticket_update_by_subject_lookback:
188            self.parameters.ticket_update_by_subject_lookback = 30
189
190        if self.parameters.verbatim_format == None:
191            self.parameters.verbatim_format = 1
192
193        if self.parameters.reflow == None:
194            self.parameters.reflow = 1
195
196        if self.parameters.binhex:
197            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
198
199        if self.parameters.applesingle:
200            self.STRIP_CONTENT_TYPES.append('application/applefile')
201
202        if self.parameters.appledouble:
203            self.STRIP_CONTENT_TYPES.append('application/applefile')
204
205        if self.parameters.strip_content_types:
206            items = self.parameters.strip_content_types.split(',')
207            for item in items:
208                self.STRIP_CONTENT_TYPES.append(item.strip())
209
210        if self.parameters.tmpdir:
211            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
212        else:
213            self.parameters.tmpdir = os.path.normcase('/tmp')
214
215        if self.parameters.email_triggers_workflow == None:
216            self.parameters.email_triggers_workflow = 1
217
218        if not self.parameters.subject_field_separator:
219            self.parameters.subject_field_separator = '&'
220        else:
221            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
222
223        if not self.parameters.strip_signature_regex:
224            self.parameters.strip_signature_regex = '^-- $'
225            self.parameters.cut_line_regex = '^-- $'
226
227        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
228        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
229
230
231        self.system = None
232
233########## Email Header Functions ###########################################################
234
235    def spam(self, message):
236        """
237        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
238        # Note if Spam_level then '*' are included
239        """
240        self.logger.debug('function spam')
241
242        spam = False
243        if message.has_key(self.parameters.spam_header):
244            spam_l = string.split(message[self.parameters.spam_header])
245            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
246
247            try:
248                number = spam_l[0].count('*')
249            except IndexError, detail:
250                number = 0
251               
252            if number >= self.parameters.spam_level:
253                spam = True
254               
255        ## treat virus mails as spam
256        #
257        elif message.has_key('X-Virus-found'):         
258            spam = True
259
260        ## Astaro firewall spam handling
261        #
262        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
263            spam = True
264
265        ## How to handle SPAM messages
266        #
267        if self.parameters.drop_spam and spam:
268
269            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
270            return 'drop'   
271
272        elif spam:
273
274            return 'Spam'   
275        else:
276
277            return False
278
279    def email_header_acl(self, keyword, message_field, default):
280        """
281        This function wil check if the email address is allowed or denied
282        to send mail to the ticket list
283        """
284        self.logger.debug('function email_header_acl: %s' %keyword)
285
286        try:
287            mail_addresses = self.parameters[keyword]
288
289            # Check if we have an empty string
290            #
291            if not mail_addresses:
292                return default
293
294        except KeyError, detail:
295            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
296
297            return default
298
299        mail_addresses = string.split(mail_addresses, ',')
300        message_addresses = string.split(message_field, ',')
301
302        for entry in mail_addresses:
303            entry = entry.strip()
304            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
305
306            for addr in message_addresses:
307                addr = addr.strip()
308
309                if self.parameters.compare_function_list in [ 'matches', 'match']:
310                    s = '\t%s matches %s' %(addr, entry)
311                    result =  TO_RE.match(addr)
312                else:
313                    s = '\t%s contains %s' %(addr, entry)
314                    result =  TO_RE.search(addr)
315
316                if result:
317                    self.logger.debug(s)
318                    return True
319
320        return False
321
322    def email_header_txt(self, m):
323        """
324        Display To and CC addresses in description field
325        """
326        s = ''
327
328        if m['To'] and len(m['To']) > 0:
329            s = "'''To:''' %s\r\n" %(m['To'])
330        if m['Cc'] and len(m['Cc']) > 0:
331            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
332
333        return  self.email_to_unicode(s)
334
335
336    def get_sender_info(self, message):
337        """
338        Get the default author name and email address from the message
339        """
340        self.logger.debug('function get_sender_info')
341
342        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
343        self.email_to_addrs = list()
344
345        for n,e in to_addrs:
346            self.email_to_addrs.append(e)
347
348        self.email_to_addrs = ','.join(self.email_to_addrs)
349
350
351        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
352
353        ## decode email name can contain charset
354        #
355        self.email_name = self.email_to_unicode(self.email_name)
356
357        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
358        self.logger.debug(dstr)
359
360        ## Trac can not handle author's name that contains spaces
361        #
362        if self.email_addr.lower() == self.trac_smtp_from.lower():
363            if self.email_name:
364                self.author = self.email_name
365            else:
366                self.author = "email2trac"
367        else:
368            self.author = self.email_addr
369
370        if self.parameters.ignore_trac_user_settings:
371            return
372
373        # Is this a registered user, use email address as search key:
374        # result:
375        #   u : login name
376        #   n : Name that the user has set in the settings tab
377        #   e : email address that the user has set in the settings tab
378        #
379        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
380            if (
381                (e and (e.lower() == self.email_addr.lower())) or
382                (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
383            )
384            ]
385
386        if len(users) >= 1:
387            self.email_from = users[0][2]
388            self.author = users[0][0]
389
390            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
391
392            if self.parameters.white_list_registered_users:
393                self.logger.debug('\tget_sender_info: added registered user to white list')
394                self.allow_registered_user = True
395                   
396
397    def set_cc_fields(self, ticket, message, update=False):
398        """
399        Set all the right fields for a new ticket
400        """
401        self.logger.debug('function set_cc_fields')
402
403
404        # Put all CC-addresses in ticket CC field
405        #
406        if self.parameters.reply_all:
407
408            email_cc = ''
409
410            cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
411
412            if not cc_addrs:
413                return
414
415            ## Build a list of forbidden CC addresses
416            #
417            #to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
418            #to_list = list()
419            #for n,e in to_addrs:
420            #   to_list.append(e)
421
422            if update:
423                self.logger.debug("\tupdate cc-fields")
424
425            for name,addr in cc_addrs:
426       
427                ## Prevent mail loop
428                #
429                if addr == self.trac_smtp_from:
430                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
431                    continue
432
433                ## Alwyas remove reporter email address
434                #
435                elif addr == self.email_addr:
436                    self.logger.debug("\tSkipping reporter email address for CC-field")
437                    continue
438
439                else:
440                    if email_cc:
441                        email_cc = '%s, %s' %(email_cc, addr)
442                    else:
443                        email_cc = addr
444
445            if email_cc:
446                self.logger.debug('\tCC fields: %s' %email_cc)
447                ticket['cc'] = self.email_to_unicode(email_cc)
448
449
450    def acl_list_from_file(self, f, keyword):
451        """
452        Read the email address from a file
453        """
454        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
455       
456        if  not os.path.isfile(f):
457            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
458        else:
459            ## read whole file and replace '\n' with ''
460            #
461            addr_l = open(f, 'r').readlines()
462            s = ','.join(addr_l).replace('\n','')
463
464            try:
465                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
466            except KeyError, detail:
467                self.parameters[keyword] = s
468
469########## DEBUG functions  ###########################################################
470
471    def debug_body(self, message_body, temp_file):
472        """
473        """
474        self.logger.debug('function debug_body:')
475
476        body_file = "%s.body" %(temp_file)
477        fx = open(body_file, 'wb')
478
479        if self.parameters.dry_run:
480            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
481            return
482
483        self.logger.debug('writing body to %s' %(body_file))
484        if not message_body:
485                message_body = '(None)'
486
487        message_body = message_body.encode('utf-8')
488
489        fx.write(message_body)
490        fx.close()
491        try:
492            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
493        except OSError:
494            pass
495
496    def debug_attachments(self, message_parts, temp_file):
497        """
498        """
499        self.logger.debug('function debug_attachments')
500       
501        n = 0
502        for item in message_parts:
503
504            ## Skip inline text parts
505            #
506            if not isinstance(item, tuple):
507                continue
508               
509            (original, filename, part) = item
510            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
511
512            dummy_filename, ext = os.path.splitext(filename)
513
514            n = n + 1
515            part_name = 'part%d%s' %(n, ext)
516            part_file = "%s.%s" %(temp_file, part_name)
517
518            s = 'writing %s: filename: %s' %(part_file, filename)
519            self.print_unicode(s)
520
521            ## Forbidden chars, just write part files instead of names
522            #
523            #filename = filename.replace('\\', '_')
524            #filename = filename.replace('/', '_')
525            #filename = filename + '.att_email2trac'
526            # part_file = os.path.join(self.parameters.tmpdir, filename)
527            #part_file = util.text.unicode_quote(part_file)
528            #self.print_unicode(part_file)
529
530            if self.parameters.dry_run:
531                self.logger.info('DRY_RUN: NOT saving attachments')
532                continue
533
534            fx = open(part_file, 'wb')
535            text = part.get_payload(decode=1)
536
537            if not text:
538                text = '(None)'
539
540            fx.write(text)
541            fx.close()
542
543            try:
544                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
545            except OSError:
546                pass
547
548    def save_email_for_debug(self, message, project_name, create_tempfile=False):
549
550        if self.parameters.dry_run:
551            self.logger.debug('DRY_RUN: NOT saving email message')
552            return
553
554        if create_tempfile:
555            (fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
556            fx = os.fdopen(fd, 'wb')
557        else:
558            tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
559            fx = open(tmp_file, 'wb')
560
561
562        self.logger.debug('saving email to %s' %(tmp_file))
563        fx.write('%s' % message)
564        fx.close()
565       
566        try:
567            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
568        except OSError:
569            pass
570
571        message_parts = self.get_message_parts(message)
572        message_parts = self.unique_attachment_names(message_parts)
573        body_text = self.get_body_text(message_parts)
574        self.debug_body(body_text, tmp_file)
575        self.debug_attachments(message_parts, tmp_file)
576
577########## Conversion functions  ###########################################################
578
579    def email_to_unicode(self, message_str):
580        """
581        Email has 7 bit ASCII code, convert it to unicode with the charset
582        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
583        understands it.
584        """
585        self.logger.debug("function email_to_unicode")
586
587        results =  email.Header.decode_header(message_str)
588
589        s = None
590        for text,format in results:
591            if format:
592                try:
593                    temp = unicode(text, format)
594                except UnicodeError, detail:
595                    # This always works
596                    #
597                    msg = 'ERROR: Could not find charset: %s, please install' %format
598                    self.logger.error(msg)
599                    temp = unicode(text, 'iso-8859-15')
600                except LookupError, detail:
601                    msg = 'ERROR: Could not find charset: %s, please install' %format
602                    self.logger.error(msg)
603                    #temp = unicode(text, 'iso-8859-15')
604                    temp = message_str
605                       
606            else:
607                temp = string.strip(text)
608                temp = unicode(text, 'iso-8859-15')
609
610            if s:
611                s = '%s %s' %(s, temp)
612            else:
613                s = '%s' %temp
614
615        return s
616
617    def str_to_dict(self, s):
618        """
619        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
620        """
621        self.logger.debug("function str_to_dict")
622
623        fields = string.split(s, self.parameters.subject_field_separator)
624
625        result = dict()
626        for field in fields:
627            try:
628                index, value = string.split(field, '=')
629
630                # We can not change the description of a ticket via the subject
631                # line. The description is the body of the email
632                #
633                if index.lower() in ['description']:
634                    continue
635
636                if value:
637                    result[index.lower()] = value
638
639            except ValueError:
640                pass
641        return result
642
643    def print_unicode(self,s):
644        """
645        This function prints unicode strings if possible else it will quote it
646        """
647        try:
648            self.logger.debug(s)
649        except UnicodeEncodeError, detail:
650            self.logger.debug(util.text.unicode_quote(s))
651
652
653    def html_2_txt(self, data):
654        """
655        Various routines to convert html syntax to valid trac wiki syntax
656        """
657        self.logger.debug('function html_2_txt')
658
659        ## This routine make an safe html that can be include
660        #  in trac, but no further text processing can be done
661        #
662#       try:
663#           from lxml.html.clean import Cleaner
664#           tags_rm = list()
665#           tags_rm.append('body')
666#
667#           cleaner = Cleaner(remove_tags=tags_rm )
668#           parsed_data = cleaner.clean_html(data)
669#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
670#
671#           return parsed_data
672#           
673#       except ImportError::
674#           pass
675
676        parsed_data = None
677        if self.parameters.html2text_cmd:
678            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
679            f = os.fdopen(fd, 'w')
680
681            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
682            self.logger.debug('\t html2text conversion %s'%(cmd))
683   
684            if self.parameters.dry_run:
685                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
686
687            else:
688                f.write(data)
689                f.close()
690
691                lines = os.popen(cmd).readlines()
692                parsed_data =  ''.join(lines)
693
694                os.unlink(tmp_file)
695
696        else:
697            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
698
699        return parsed_data
700
701    def check_filename_length(self, name):
702        """
703        To bypass a bug in Trac
704        check if the filename lenght is not larger then OS limit.
705          yes : return truncated filename
706          no  : return unmodified filename
707        """
708        self.logger.debug('function check_filename_length: ')
709
710        if not name:
711            return 'None'
712       
713        dummy_filename, ext = os.path.splitext(name)
714
715        ## Trac uses this format
716        #
717        quote_format = util.text.unicode_quote(dummy_filename)
718
719        ## Determine max filename lenght
720        #
721        try:
722            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
723        except AttributeError, detail:
724            filemax_lenght = 240
725
726        if len(quote_format) <= filemax_length:
727            return name
728
729        else:
730            ## Truncate file to filemax_length and reserve room for extension
731            #  We must break on a boundry
732            #
733            length = filemax_length - 6
734
735            for i in range(0,10):
736
737                truncated = quote_format[ : (length - i)]
738
739                try:
740                    unqoute_truncated = util.text.unicode_unquote(truncated)
741                    unqoute_truncated = unqoute_truncated + ext
742
743                    self.print_unicode('\t ' + unqoute_truncated)
744
745                    break
746
747                except UnicodeDecodeError, detail:
748                    continue
749
750
751            return unqoute_truncated
752
753########## TRAC ticket functions  ###########################################################
754
755    def mail_workflow(self, tkt):
756        """
757        """
758        self.logger.debug('function mail_workflow: ')
759       
760        req = Mock(authname=self.author, perm=MockPerm(), args={})
761        ticket_system = TicketSystem(self.env)
762       
763        try:
764            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
765
766        except KeyError:
767            ## fallback for compability (Will be deprecated)
768            #  workflow can be none.
769            #
770            workflow = None
771            if tkt['status'] in ['closed']:
772                workflow = self.parameters.workflow
773
774        if workflow:
775
776            ## process all workflow implementations
777            #
778            tkt_module = TicketModule(self.env)
779            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
780
781            for field in field_changes.keys():
782
783                ## We have already processed these fields
784                #
785                if not field in ['summary', 'description']:
786                    s = 'workflow : %s, field %s : %s, by %s' \
787                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
788                    self.logger.debug(s)
789
790                    tkt[field] = field_changes[field]['new']
791
792            return True
793
794        else:
795            return False
796
797    def check_permission_participants(self, tkt, action):
798        """
799        Check if the mailer is allowed to update the ticket
800        """
801        self.logger.debug('function check_permission_participants %s')
802
803        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
804            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
805
806            return True
807
808        perm = PermissionSystem(self.env)
809        if perm.check_permission(action, self.author):
810            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
811
812            return True
813       
814        # Is the updater in the CC?
815        try:
816            cc_list = tkt['cc'].split(',')
817            for cc in cc_list:
818                if self.email_addr.lower() in cc.lower.strip():
819                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
820
821                    return True
822
823        except KeyError:
824            pass
825
826        return False
827
828    def check_permission(self, tkt, action):
829        """
830        check if the reporter has the right permission for the action:
831          - TICKET_CREATE
832          - TICKET_MODIFY
833          - TICKET_APPEND
834          - TICKET_CHGPROP
835
836        There are three models:
837            - None      : no checking at all
838            - trac      : check the permission via trac permission model
839            - email2trac: ....
840        """
841        self.logger.debug("function check_permission: %s" %(action))
842
843        if self.parameters.ticket_permission_system in ['trac']:
844
845            perm = PermissionCache(self.env, self.author)
846            if perm.has_permission(action):
847                return True
848            else:
849                return False
850
851        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
852            return (self.check_permission_participants(tkt, action))   
853
854        ## Default is to allow everybody ticket updates and ticket creation
855        #
856        else:
857                return True
858
859
860    def update_ticket_fields(self, ticket, user_dict, new=None):
861        """
862        This will update the ticket fields. It will check if the
863        given fields are known and if the right values are specified
864        It will only update the ticket field value:
865            - If the field is known
866            - If the value supplied is valid for the ticket field.
867              If not then there are two options:
868               1) Skip the value (new=None)
869               2) Set default value for field (new=1)
870        """
871        self.logger.debug("function update_ticket_fields")
872
873        ## Check only permission model on ticket updates
874        #
875        if not new:
876            if self.parameters.ticket_permission_system:
877                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
878                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
879                    return False
880
881        ## Build a system dictionary from the ticket fields
882        #  with field as index and option as value
883        #
884        sys_dict = dict()
885
886        for field in ticket.fields:
887
888            try:
889                sys_dict[field['name']] = field['options']
890
891            except KeyError:
892                sys_dict[field['name']] = None
893                pass
894
895        ## Check user supplied fields an compare them with the
896        #  system one's
897        #
898        for field,value in user_dict.items():
899       
900            s = 'user_field\t %s = %s' %(field,value)
901            self.print_unicode(s)
902
903            if not field in sys_dict.keys(): 
904                self.logger.debug('%s is not a valid field for tickets' %(field))
905                continue
906
907            ## To prevent mail loop
908            #
909            if field == 'cc':
910
911                cc_list = user_dict['cc'].split(',')
912
913                if self.trac_smtp_from in cc_list:
914                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
915
916                    cc_list.remove(self.trac_smtp_from)
917
918                value = ','.join(cc_list)
919               
920
921            ## Check if every value is allowed for this field
922            #
923            if sys_dict[field]:
924
925                if value in sys_dict[field]:
926                    ticket[field] = value
927                else:
928                    ## Must we set a default if value is not allowed
929                    #
930                    if new:
931                        value = self.get_config('ticket', 'default_%s' %(field) )
932
933            else:
934
935                ticket[field] = value
936
937            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
938            self.print_unicode(s)
939
940    def ticket_update(self, m, id, spam):
941        """
942        If the current email is a reply to an existing ticket, this function
943        will append the contents of this email to that ticket, instead of
944        creating a new one.
945        """
946        self.logger.debug("function ticket_update")
947
948        if not self.parameters.ticket_update:
949            self.logger.debug("ticket_update disabled")
950            return False
951
952        ## Must we update ticket fields
953        #
954        update_fields = dict()
955        try:
956            id, keywords = string.split(id, '?')
957
958            update_fields = self.str_to_dict(keywords)
959
960            ## Strip '#'
961            #
962            self.id = int(id[1:])
963
964        except ValueError:
965
966            ## Strip '#'
967            #
968            self.id = int(id[1:])
969
970        self.logger.debug("\tticket id: %s" %id)
971
972        ## When is the change committed
973        #
974        if self.VERSION < 0.11:
975            when = int(time.time())
976        else:
977            when = datetime.now(util.datefmt.utc)
978
979        try:
980            tkt = Ticket(self.env, self.id, self.db)
981
982        except util.TracError, detail:
983
984            ## Not a valid ticket
985            #
986            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
987            self.id = None
988            return False
989
990        ## Check the permission of the reporter
991        #
992        if self.parameters.ticket_permission_system:
993            if not self.check_permission(tkt, 'TICKET_APPEND'):
994                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
995                return False
996
997        ## How many changes has this ticket
998        #
999        grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1000        cnum = sum(1 for e in grouped) + 1
1001
1002
1003        ## reopen the ticket if it is was closed
1004        #  We must use the ticket workflow framework
1005        #
1006        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1007
1008            if not self.mail_workflow(tkt):
1009
1010                if tkt['status'] in ['closed']:
1011                    tkt['status'] = 'reopened'
1012                    tkt['resolution'] = ''
1013
1014        ## Old pre 0.11 situation
1015        #
1016        elif self.parameters.email_triggers_workflow:
1017
1018            self.logger.debug('\temail triggers workflow pre trac 0.11')
1019
1020            if tkt['status'] in ['closed']:
1021                tkt['status'] = 'reopened'
1022                tkt['resolution'] = ''
1023
1024        else:
1025            self.logger.debug('\temail triggers workflow disabled')
1026
1027        ## Must we update some ticket fields properties via subject line
1028        #
1029        if update_fields:
1030            self.update_ticket_fields(tkt, update_fields)
1031
1032
1033        message_parts = self.get_message_parts(m)
1034        message_parts = self.unique_attachment_names(message_parts)
1035
1036        ## Must we update some ticket fields properties via inline comments
1037        # in body_text
1038        #
1039        if self.properties:
1040                self.update_ticket_fields(tkt, self.properties)
1041
1042        ## Must we update the CC ticket field
1043        #
1044        self.set_cc_fields(tkt, m, update=True)
1045
1046        if self.parameters.email_header:
1047            message_parts.insert(0, self.email_header_txt(m))
1048
1049        body_text = self.get_body_text(message_parts)
1050
1051        error_with_attachments = self.attach_attachments(message_parts)
1052
1053        if body_text.strip() or update_fields or self.properties:
1054
1055            if self.parameters.dry_run:
1056                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1057                self.logger.info(s)
1058
1059            else:
1060                if error_with_attachments:
1061                    body_text = '%s\\%s' %(error_with_attachments, body_text)
1062
1063                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1064                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1065           
1066        if not spam:
1067            self.notify(tkt, False, when)
1068
1069        return True
1070
1071    def set_ticket_fields(self, ticket):
1072        """
1073        set the ticket fields to value specified
1074            - /etc/email2trac.conf with <prefix>_<field>
1075            - trac default values, trac.ini
1076        """
1077        self.logger.debug('function set_ticket_fields')
1078
1079        user_dict = dict()
1080
1081        for field in ticket.fields:
1082
1083            name = field['name']
1084
1085            ## default trac value
1086            #
1087            if not field.get('custom'):
1088                value = self.get_config('ticket', 'default_%s' %(name) )
1089
1090                ## skip this field can only be set by email2trac.conf
1091                #
1092                if name in ['resolution']:
1093                    value = None
1094
1095            else:
1096                ##  Else get the default value for reporter
1097                #
1098                value = field.get('value')
1099                options = field.get('options')
1100
1101                if value and options and (value not in options):
1102                     value = options[int(value)]
1103   
1104            s = 'trac[%s] = %s' %(name, value)
1105            self.print_unicode(s)
1106
1107            ## email2trac.conf settings
1108            #
1109            prefix = self.parameters.ticket_prefix
1110            try:
1111                value = self.parameters['%s_%s' %(prefix, name)]
1112
1113                s = 'email2trac[%s] = %s ' %(name, value)
1114                self.print_unicode(s)
1115
1116            except KeyError, detail:
1117                pass
1118       
1119            if value:
1120                user_dict[name] = value
1121
1122                s = 'used %s = %s' %(name, value)
1123                self.print_unicode(s)
1124
1125        self.update_ticket_fields(ticket, user_dict, new=1)
1126
1127        if 'status' not in user_dict.keys():
1128            ticket['status'] = 'new'
1129
1130
1131    def ticket_update_by_subject(self, subject):
1132        """
1133        This list of Re: prefixes is probably incomplete. Taken from
1134        wikipedia. Here is how the subject is matched
1135          - Re: <subject>
1136          - Re: (<Mail list label>:)+ <subject>
1137
1138        So we must have the last column
1139        """
1140        self.logger.debug('function ticket_update_by_subject')
1141
1142        found_id = None
1143        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1144               
1145            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1146            result = SUBJECT_RE.search(subject)
1147
1148            if result:
1149                ## This is a reply
1150                #
1151                orig_subject = result.group(1)
1152
1153                self.logger.debug('subject search string: %s' %(orig_subject))
1154
1155                cursor = self.db.cursor()
1156                summaries = [orig_subject, '%%: %s' % orig_subject]
1157
1158                ## Time resolution is in micoseconds
1159                #
1160                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1161
1162                if self.VERSION < 0.12:
1163                    lookback = util.datefmt.to_timestamp(search_date)
1164                else:
1165                    lookback = util.datefmt.to_utimestamp(search_date)
1166
1167                for summary in summaries:
1168                    self.logger.debug('Looking for summary matching: "%s"' % summary)
1169
1170                    sql = """SELECT id, reporter FROM ticket
1171                            WHERE changetime >= %s AND summary LIKE %s
1172                            ORDER BY changetime DESC"""
1173
1174                    cursor.execute(sql, [lookback, summary.strip()])
1175
1176                    for row in cursor:
1177
1178                        (matched_id, sql_reporter) = row
1179
1180                        ## Save first entry.
1181                        #
1182                        if not found_id:
1183                            found_id = matched_id
1184                           
1185                        ## If subject and reporter are the same. The we certainly have found the right ticket
1186                        #
1187                        if sql_reporter == self.author:
1188                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1189                            found_id = matched_id
1190                            break
1191
1192                    if found_id:
1193                        self.logger.debug('Found matching ticket id: %d' % found_id)
1194                        found_id = '#%d' % found_id
1195                        return (found_id, orig_subject)
1196                   
1197                    # obsolete !?? 12 Aug 2011
1198                    #subject = orig_subject
1199
1200        return (found_id, subject)
1201
1202
1203    def new_ticket(self, msg, subject, spam, set_fields = None):
1204        """
1205        Create a new ticket
1206        """
1207        self.logger.debug('function new_ticket')
1208
1209        tkt = Ticket(self.env)
1210
1211        ## self.author can be email address of an username
1212        #
1213        tkt['reporter'] = self.author
1214
1215        self.set_cc_fields(tkt, msg)
1216
1217        self.set_ticket_fields(tkt)
1218
1219        ## Check the permission of the reporter
1220        #
1221        if self.parameters.ticket_permission_system:
1222            if not self.check_permission(tkt, 'TICKET_CREATE'):
1223                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1224                return False
1225
1226        ## Old style setting for component, will be removed
1227        #
1228        if spam:
1229            tkt['component'] = 'Spam'
1230
1231        elif self.parameters.has_key('component'):
1232            tkt['component'] = self.parameters['component']
1233
1234        if not msg['Subject']:
1235            tkt['summary'] = u'(No subject)'
1236        else:
1237            tkt['summary'] = subject
1238
1239
1240        if set_fields:
1241            rest, keywords = string.split(set_fields, '?')
1242
1243            if keywords:
1244                update_fields = self.str_to_dict(keywords)
1245                self.update_ticket_fields(tkt, update_fields)
1246
1247
1248        message_parts = self.get_message_parts(msg, True)
1249
1250        ## Must we update some ticket fields properties via body_text
1251        #
1252        if self.properties:
1253                self.update_ticket_fields(tkt, self.properties)
1254
1255        message_parts = self.unique_attachment_names(message_parts)
1256       
1257        ## produce e-mail like header
1258        #
1259        head = ''
1260        if self.parameters.email_header:
1261            head = self.email_header_txt(msg)
1262            message_parts.insert(0, head)
1263           
1264        body_text = self.get_body_text(message_parts)
1265
1266        tkt['description'] = body_text
1267
1268        ## When is the change committed
1269        #
1270        if self.VERSION < 0.11:
1271            when = int(time.time())
1272        else:
1273            when = datetime.now(util.datefmt.utc)
1274
1275        if self.parameters.dry_run:
1276            self.logger.info('DRY_RUN: tkt.insert()')
1277        else:
1278            self.id = tkt.insert()
1279   
1280        changed = False
1281        comment = ''
1282
1283        ## some routines in trac are dependend on ticket id
1284        #  like alternate notify template
1285        #
1286        if self.parameters.alternate_notify_template:
1287            tkt['id'] = self.id
1288            changed = True
1289
1290        ## Rewrite the description if we have mailto enabled
1291        #
1292        if self.parameters.mailto_link:
1293            changed = True
1294            comment = u'\nadded mailto line\n'
1295
1296            #mailto = self.html_mailto_link( m['Subject'])
1297            mailto = self.html_mailto_link(subject)
1298
1299            tkt['description'] = u'%s\r\n%s%s\r\n' \
1300                %(head, mailto, body_text)
1301   
1302        ## Save the attachments to the ticket   
1303        #
1304        error_with_attachments =  self.attach_attachments(message_parts)
1305
1306        if error_with_attachments:
1307            changed = True
1308            comment = '%s\n%s\n' %(comment, error_with_attachments)
1309
1310        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1311            if self.mail_workflow(tkt):
1312                changed = True
1313
1314        if changed:
1315            if self.parameters.dry_run:
1316                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1317                self.logger.info(s)
1318
1319            else:
1320                tkt.save_changes(tkt['reporter'], comment)
1321
1322        if not spam:
1323            self.notify(tkt, True)
1324
1325
1326    def attach_attachments(self, message_parts, update=False):
1327        '''
1328        save any attachments as files in the ticket's directory
1329        '''
1330        self.logger.debug('function attach_attachments()')
1331
1332        if self.parameters.dry_run:
1333            self.logger.debug("DRY_RUN: no attachments attached to tickets")
1334            return ''
1335
1336        count = 0
1337
1338        ## Get Maxium attachment size
1339        #
1340        max_size = int(self.get_config('attachment', 'max_size'))
1341        status   = None
1342       
1343        for item in message_parts:
1344            ## Skip body parts
1345            #
1346            if not isinstance(item, tuple):
1347                continue
1348               
1349            (original, filename, part) = item
1350
1351            ## We have to determine the size so we use this temporary solution.
1352            #
1353            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1354            text = part.get_payload(decode=1)
1355            if not text:
1356                text = '(None)'
1357            fd.write(text)
1358            fd.close()
1359
1360            ## get the file_size
1361            #
1362            stats = os.lstat(path)
1363            file_size = stats[ST_SIZE]
1364
1365            ## Check if the attachment size is allowed
1366            #
1367            if (max_size != -1) and (file_size > max_size):
1368                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1369                    %(status, original, file_size, max_size)
1370
1371                os.unlink(path)
1372                continue
1373            else:
1374                count = count + 1
1375                   
1376            ## Insert the attachment
1377            #
1378            fd = open(path, 'rb')
1379            if self.system == 'discussion':
1380                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1381            elif self.system == 'blog':
1382                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1383            else:
1384                s = 'Attach %s to ticket %d' %(filename, self.id)
1385                self.print_unicode(s)
1386                att = attachment.Attachment(self.env, 'ticket', self.id)
1387 
1388            ## This will break the ticket_update system, the body_text is vaporized
1389            #  ;-(
1390            #
1391            if not update:
1392                att.author = self.author
1393                att.description = self.email_to_unicode('Added by email2trac')
1394
1395            try:
1396
1397                self.logger.debug('Insert atachment')
1398                att.insert(filename, fd, file_size)
1399
1400            except OSError, detail:
1401
1402                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1403                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1404
1405            ## Remove the created temporary filename
1406            #
1407            fd.close()
1408            os.unlink(path)
1409
1410        ## return error
1411        #
1412        return status
1413
1414########## Fullblog functions  #################################################
1415
1416    def blog(self, msg, subject, id, params):
1417        """
1418        The blog create/update function
1419        """
1420        ## import the modules
1421        #
1422        from tracfullblog.core import FullBlogCore
1423        from tracfullblog.model import BlogPost, BlogComment
1424
1425        ## instantiate blog core
1426        #
1427        blog = FullBlogCore(self.env)
1428        req = Mock(authname='anonymous', perm=MockPerm(), args={})
1429
1430        ## parameters from the subject
1431        #
1432        params = self.str_to_dict((params or '').lstrip('?'))
1433
1434        ## preferably get the time from the email date header, or else
1435        #  use current date and time
1436        date = email.Utils.parsedate_tz(msg.get('date'))
1437        if date:
1438            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1439        else:
1440            self.logger.warn("No valid date header found")
1441            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1442
1443        ## blog entry affected
1444        #
1445        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1446
1447        ## check wether a blog post exists
1448        #
1449        post = BlogPost(self.env, self.id)
1450        force_update = self.properties.get('update', params.get('update'))
1451
1452        ## message parts
1453        #
1454        message_parts = self.get_message_parts(msg)
1455        message_parts = self.unique_attachment_names(message_parts)
1456
1457        if post.get_versions() and not force_update:
1458
1459            ## add comment to blog entry
1460            #
1461            comment = BlogComment(self.env, self.id)
1462            comment.author = self.properties.get('author', params.get('author', self.author))
1463            comment.comment = self.get_body_text(message_parts)
1464            comment.time = dt
1465
1466            if self.parameters.dry_run:
1467                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1468                return
1469            warnings = blog.create_comment(req, comment)
1470
1471        else:
1472            ## create or update blog entry
1473            #
1474            post.author = self.properties.get('author', params.get('author', self.author))
1475            post.categories = self.properties.get('categories', params.get('categories', ''))
1476            post.title = subject.strip()
1477            post.publish_time = dt
1478            post.body = self.get_body_text(message_parts)
1479           
1480            if self.parameters.dry_run:
1481                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1482                return
1483            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1484
1485        ## check for problems
1486        #
1487        if warnings:
1488            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1489       
1490        ## all seems well, attach attachments
1491        #
1492        self.attach_attachments(message_parts)
1493
1494
1495########## Discussion functions  ##############################################
1496
1497    def discussion_topic(self, content, subject):
1498
1499        ## Import modules.
1500        #
1501        from tracdiscussion.api import DiscussionApi
1502        from trac.util.datefmt import to_timestamp, utc
1503
1504        self.logger.debug('Creating a new topic in forum:', self.id)
1505
1506        ## Get dissussion API component.
1507        #
1508        api = self.env[DiscussionApi]
1509        context = self._create_context(content, subject)
1510
1511        ## Get forum for new topic.
1512        #
1513        forum = api.get_forum(context, self.id)
1514
1515        if not forum:
1516            self.logger.error("ERROR: Replied forum doesn't exist")
1517
1518        ## Prepare topic.
1519        #
1520        topic = {'forum' : forum['id'],
1521                 'subject' : context.subject,
1522                 'time': to_timestamp(datetime.now(utc)),
1523                 'author' : self.author,
1524                 'subscribers' : [self.email_addr],
1525                 'body' : self.get_body_text(context.content_parts)}
1526
1527        ## Add topic to DB and commit it.
1528        #
1529        self._add_topic(api, context, topic)
1530        self.db.commit()
1531
1532    def discussion_topic_reply(self, content, subject):
1533
1534        ## Import modules.
1535        #
1536        from tracdiscussion.api import DiscussionApi
1537        from trac.util.datefmt import to_timestamp, utc
1538
1539        self.logger.debug('Replying to discussion topic', self.id)
1540
1541        ## Get dissussion API component.
1542        #
1543        api = self.env[DiscussionApi]
1544        context = self._create_context(content, subject)
1545
1546        ## Get replied topic.
1547        #
1548        topic = api.get_topic(context, self.id)
1549
1550        if not topic:
1551            self.logger.error("ERROR: Replied topic doesn't exist")
1552
1553        ## Prepare message.
1554        #
1555        message = {'forum' : topic['forum'],
1556                   'topic' : topic['id'],
1557                   'replyto' : -1,
1558                   'time' : to_timestamp(datetime.now(utc)),
1559                   'author' : self.author,
1560                   'body' : self.get_body_text(context.content_parts)}
1561
1562        ## Add message to DB and commit it.
1563        #
1564        self._add_message(api, context, message)
1565        self.db.commit()
1566
1567    def discussion_message_reply(self, content, subject):
1568
1569        ## Import modules.
1570        #
1571        from tracdiscussion.api import DiscussionApi
1572        from trac.util.datefmt import to_timestamp, utc
1573
1574        self.logger.debug('Replying to discussion message', self.id)
1575
1576        ## Get dissussion API component.
1577        #
1578        api = self.env[DiscussionApi]
1579        context = self._create_context(content, subject)
1580
1581        ## Get replied message.
1582        #
1583        message = api.get_message(context, self.id)
1584
1585        if not message:
1586            self.logger.error("ERROR: Replied message doesn't exist")
1587
1588        ## Prepare message.
1589        #
1590        message = {'forum' : message['forum'],
1591                   'topic' : message['topic'],
1592                   'replyto' : message['id'],
1593                   'time' : to_timestamp(datetime.now(utc)),
1594                   'author' : self.author,
1595                   'body' : self.get_body_text(context.content_parts)}
1596
1597        ## Add message to DB and commit it.
1598        #
1599        self._add_message(api, context, message)
1600        self.db.commit()
1601
1602    def _create_context(self, content, subject):
1603
1604        ## Import modules.
1605        #
1606        from trac.mimeview import Context
1607        from trac.web.api import Request
1608        from trac.perm import PermissionCache
1609
1610        ## TODO: Read server base URL from config.
1611        #  Create request object to mockup context creation.
1612        #
1613        environ = {'SERVER_PORT' : 80,
1614                   'SERVER_NAME' : 'test',
1615                   'REQUEST_METHOD' : 'POST',
1616                   'wsgi.url_scheme' : 'http',
1617                   'wsgi.input' : sys.stdin}
1618        chrome =  {'links': {},
1619                   'scripts': [],
1620                   'ctxtnav': [],
1621                   'warnings': [],
1622                   'notices': []}
1623
1624        if self.env.base_url_for_redirect:
1625            environ['trac.base_url'] = self.env.base_url
1626
1627        req = Request(environ, None)
1628        req.chrome = chrome
1629        req.tz = 'missing'
1630        req.authname = self.author
1631        req.perm = PermissionCache(self.env, self.author)
1632        req.locale = None
1633
1634        ## Create and return context.
1635        #
1636        context = Context.from_request(req)
1637        context.realm = 'discussion-email2trac'
1638        context.cursor = self.db.cursor()
1639        context.content = content
1640        context.subject = subject
1641
1642        ## Read content parts from content.
1643        #
1644        context.content_parts = self.get_message_parts(content)
1645        context.content_parts = self.unique_attachment_names(
1646          context.content_parts)
1647
1648        return context
1649
1650    def _add_topic(self, api, context, topic):
1651        context.req.perm.assert_permission('DISCUSSION_APPEND')
1652
1653        ## Filter topic.
1654        #
1655        for discussion_filter in api.discussion_filters:
1656            accept, topic_or_error = discussion_filter.filter_topic(
1657              context, topic)
1658            if accept:
1659                topic = topic_or_error
1660            else:
1661                raise TracError(topic_or_error)
1662
1663        ## Add a new topic.
1664        #
1665        api.add_topic(context, topic)
1666
1667        ## Get inserted topic with new ID.
1668        #
1669        topic = api.get_topic_by_time(context, topic['time'])
1670
1671        ## Attach attachments.
1672        #
1673        self.id = topic['id']
1674        self.attach_attachments(context.content_parts, True)
1675
1676        ## Notify change listeners.
1677        #
1678        for listener in api.topic_change_listeners:
1679            listener.topic_created(context, topic)
1680
1681    def _add_message(self, api, context, message):
1682        context.req.perm.assert_permission('DISCUSSION_APPEND')
1683
1684        ## Filter message.
1685        #
1686        for discussion_filter in api.discussion_filters:
1687            accept, message_or_error = discussion_filter.filter_message(
1688              context, message)
1689            if accept:
1690                message = message_or_error
1691            else:
1692                raise TracError(message_or_error)
1693
1694        ## Add message.
1695        #
1696        api.add_message(context, message)
1697
1698        ## Get inserted message with new ID.
1699        #
1700        message = api.get_message_by_time(context, message['time'])
1701
1702        ## Attach attachments.
1703        #
1704
1705        self.attach_attachments(context.content_parts, True)
1706
1707        ## Notify change listeners.
1708        #
1709        for listener in api.message_change_listeners:
1710            listener.message_created(context, message)
1711
1712########## MAIN function  ######################################################
1713
1714    def parse(self, fp):
1715        """
1716        """
1717        self.logger.debug('Main function parse')
1718        global m
1719
1720        m = email.message_from_file(fp)
1721       
1722        if not m:
1723            self.logger.debug('This is not a valid email message format')
1724            return
1725           
1726        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1727        #
1728        try:
1729            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1730        except AttributeError, detail:
1731            pass
1732
1733        if self.parameters.debug:     # save the entire e-mail message text
1734            self.save_email_for_debug(m, self.parameters.project_name, True)
1735
1736        self.db = self.env.get_db_cnx()
1737        self.get_sender_info(m)
1738
1739
1740        if self.parameters.white_list_file:
1741            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1742
1743        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1744
1745            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1746            return False
1747
1748        if self.email_header_acl('black_list', self.email_addr, False):
1749            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1750            return False
1751
1752        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1753            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1754            return False
1755
1756        ## If spam drop the message
1757        #
1758        if self.spam(m) == 'drop':
1759            return False
1760
1761        elif self.spam(m) == 'spam':
1762            spam_msg = True
1763        else:
1764            spam_msg = False
1765
1766        if not m['Subject']:
1767            subject  = 'No Subject'
1768        else:
1769            subject  = self.email_to_unicode(m['Subject'])
1770
1771        ## Bug in python logging module <2.6
1772        #
1773        self.logger.info('subject: %s' %repr(subject))
1774
1775        ## [hic] #1529: Re: LRZ
1776        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1777        #
1778        ticket_regex = r'''
1779            (?P<new_fields>[#][?].*)
1780            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1781            '''
1782        ## Check if  FullBlogPlugin is installed
1783        #
1784        blog_enabled = None
1785        blog_regex = ''
1786        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1787            self.logger.debug('Trac BLOG support enabled')
1788            blog_enabled = True
1789            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
1790
1791
1792        ## Check if DiscussionPlugin is installed
1793        #
1794        discussion_enabled = None
1795        discussion_regex = ''
1796        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1797            self.logger.debug('Trac Discussion support enabled')
1798            discussion_enabled = True
1799            discussion_regex = r'''
1800            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1801            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1802            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1803            '''
1804
1805
1806        regex_str = ticket_regex + blog_regex + discussion_regex
1807        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1808
1809        ## Find out if this is a ticket, a blog or a discussion
1810        #
1811        result =  SYSTEM_RE.search(subject)
1812
1813        if result:
1814            ## update ticket + fields
1815            #
1816            if result.group('reply'):
1817                self.system = 'ticket'
1818
1819                ## Skip the last ':' character
1820                #
1821                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1822                    self.new_ticket(m, subject, spam_msg)
1823
1824            ## New ticket + fields
1825            #
1826            elif result.group('new_fields'):
1827                self.system = 'ticket'
1828                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1829
1830            if blog_enabled:
1831                if result.group('blog'):
1832                    self.system = 'blog'
1833                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1834
1835            if discussion_enabled:
1836                ## New topic.
1837                #
1838                if result.group('forum'):
1839                    self.system = 'discussion'
1840                    self.id = int(result.group('forum_id'))
1841                    self.discussion_topic(m, subject[result.end('forum'):])
1842
1843                ## Reply to topic.
1844                #
1845                elif result.group('topic'):
1846                    self.system = 'discussion'
1847                    self.id = int(result.group('topic_id'))
1848                    self.discussion_topic_reply(m, subject[result.end('topic'):])
1849
1850                ## Reply to topic message.
1851                #
1852                elif result.group('message'):
1853                    self.system = 'discussion'
1854                    self.id = int(result.group('message_id'))
1855                    self.discussion_message_reply(m, subject[result.end('message'):])
1856
1857        else:
1858
1859            self.system = 'ticket'
1860            (matched_id, subject) = self.ticket_update_by_subject(subject)
1861
1862            if matched_id:
1863
1864                if not self.ticket_update(m, matched_id, spam_msg):
1865                    self.new_ticket(m, subject, spam_msg)
1866
1867            else:
1868                ## No update by subject, so just create a new ticket
1869                #
1870                self.new_ticket(m, subject, spam_msg)
1871
1872
1873########## BODY TEXT functions  ###########################################################
1874
1875    def strip_signature(self, text):
1876        """
1877        Strip signature from message, inspired by Mailman software
1878        """
1879        self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
1880
1881        body = []
1882
1883        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1884        for line in text.splitlines():
1885
1886            match = STRIP_RE.match(line)
1887            if match:
1888                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
1889                break
1890
1891            body.append(line)
1892
1893        return ('\n'.join(body))
1894
1895    def reflow(self, text, delsp = 0):
1896        """
1897        Reflow the message based on the format="flowed" specification (RFC 3676)
1898        """
1899        flowedlines = []
1900        quotelevel = 0
1901        prevflowed = 0
1902
1903        for line in text.splitlines():
1904            from re import match
1905           
1906            ## Figure out the quote level and the content of the current line
1907            #
1908            m = match('(>*)( ?)(.*)', line)
1909            linequotelevel = len(m.group(1))
1910            line = m.group(3)
1911
1912            ## Determine whether this line is flowed
1913            #
1914            if line and line != '-- ' and line[-1] == ' ':
1915                flowed = 1
1916            else:
1917                flowed = 0
1918
1919            if flowed and delsp and line and line[-1] == ' ':
1920                line = line[:-1]
1921
1922            ## If the previous line is flowed, append this line to it
1923            #
1924            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1925                flowedlines[-1] += line
1926
1927            ## Otherwise, start a new line
1928            #
1929            else:
1930                flowedlines.append('>' * linequotelevel + line)
1931
1932            prevflowed = flowed
1933           
1934
1935        return '\n'.join(flowedlines)
1936
1937    def strip_quotes(self, text):
1938        """
1939        Strip quotes from message by Nicolas Mendoza
1940        """
1941        self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
1942
1943        body = []
1944
1945        STRIP_RE = re.compile( self.parameters.email_quote )
1946
1947        for line in text.splitlines():
1948
1949            try:
1950
1951                match = STRIP_RE.match(line)
1952                if match:
1953                    print line
1954                    continue
1955
1956            except UnicodeDecodeError:
1957
1958                tmp_line = self.email_to_unicode(line)
1959
1960                match = STRIP_RE.match(tmp_line)
1961                if match:
1962                    print line
1963                    continue
1964               
1965            body.append(line)
1966
1967        return ('\n'.join(body))
1968
1969    def inline_properties(self, text):
1970        """
1971        Parse text if we use inline keywords to set ticket fields
1972        """
1973        self.logger.debug('function inline_properties')
1974
1975        properties = dict()
1976        body = list()
1977
1978        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
1979
1980        for line in text.splitlines():
1981            match = INLINE_EXP.match(line)
1982            if match:
1983                keyword, value = match.groups()
1984
1985                if self.parameters.inline_properties_first_wins:
1986                    if keyword in self.properties.keys():
1987                        continue
1988
1989                self.properties[keyword] = value.strip()
1990                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
1991
1992            else:
1993                body.append(line)
1994               
1995        return '\n'.join(body)
1996
1997
1998    def wrap_text(self, text, replace_whitespace = False):
1999        """
2000        Will break a lines longer then given length into several small
2001        lines of size given length
2002        """
2003        import textwrap
2004
2005        LINESEPARATOR = '\n'
2006        reformat = ''
2007
2008        for s in text.split(LINESEPARATOR):
2009            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2010            if tmp:
2011                reformat = '%s\n%s' %(reformat,tmp)
2012            else:
2013                reformat = '%s\n' %reformat
2014
2015        return reformat
2016
2017        # Python2.4 and higher
2018        #
2019        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2020        #
2021
2022########## EMAIL attachements functions ###########################################################
2023
2024    def inline_part(self, part):
2025        """
2026        """
2027        self.logger.debug('function inline_part()')
2028
2029        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2030
2031    def get_message_parts(self, msg, new_email=False):
2032        """
2033        parses the email message and returns a list of body parts and attachments
2034        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2035        """
2036        self.logger.debug('function get_message_parts()')
2037
2038        message_parts = list()
2039   
2040        ALTERNATIVE_MULTIPART = False
2041
2042        for part in msg.walk():
2043            content_maintype = part.get_content_maintype()
2044            content_type =  part.get_content_type()
2045
2046            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2047            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2048
2049            ## Check content type
2050            #
2051            if content_type in self.STRIP_CONTENT_TYPES:
2052                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2053                continue
2054
2055            ## Catch some mulitpart execptions
2056            #
2057            if content_type == 'multipart/alternative':
2058                ALTERNATIVE_MULTIPART = True
2059                continue
2060
2061            ## Skip multipart containers
2062            #
2063            if content_maintype == 'multipart':
2064                self.logger.debug("\t Skipping multipart container")
2065                continue
2066           
2067            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2068            #  or if there is one and it says "inline"
2069            #
2070            inline = self.inline_part(part)
2071
2072            ## Drop HTML message
2073            #
2074            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2075
2076                if content_type == 'text/html':
2077                    self.logger.debug('\t Skipping alternative HTML message')
2078                    ALTERNATIVE_MULTIPART = False
2079                    continue
2080
2081            filename = part.get_filename()
2082            s = '\t unicode filename: %s' %(filename)
2083            self.print_unicode(s)
2084            self.logger.debug('\t raw filename: %s' %repr(filename))
2085
2086            filename = self.check_filename_length(filename)
2087
2088            ## Save all non plain text message as attachment
2089            #
2090            if not content_type in ['text/plain']:
2091
2092                message_parts.append( (filename, part) )
2093
2094                ## We only convert html messages
2095                #
2096                if not content_type == 'text/html':
2097                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2098                    continue
2099
2100
2101            ## We have an text or html message
2102            #
2103            if not inline:
2104                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2105                    message_parts.append( (filename, part) )
2106                    continue
2107               
2108            ## Try to decode message part. We have a html or plain text messafe
2109            #
2110            body_text = part.get_payload(decode=1)
2111            if not body_text:           
2112                body_text = part.get_payload(decode=0)
2113
2114            ## Try to convert html message
2115            #
2116            if content_type == 'text/html':
2117
2118                body_text = self.html_2_txt(body_text)
2119                if not body_text:
2120                    continue
2121
2122            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2123            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2124
2125            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2126                body_text = self.reflow(body_text, delsp == 'yes')
2127
2128            if new_email and self.parameters.only_strip_on_update:
2129                self.logger.debug('Skip signature/quote stripping for new messages')
2130            else:
2131                if self.parameters.strip_signature:
2132                    body_text = self.strip_signature(body_text)
2133
2134                if self.parameters.strip_quotes:
2135                    body_text = self.strip_quotes(body_text)
2136
2137            if self.parameters.inline_properties:
2138                body_text = self.inline_properties(body_text)
2139
2140            if self.parameters.use_textwrap:
2141                body_text = self.wrap_text(body_text)
2142
2143            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2144            #
2145            charset = part.get_content_charset()
2146            if not charset:
2147                charset = 'iso-8859-15'
2148
2149            try:
2150                ubody_text = unicode(body_text, charset)
2151
2152            except UnicodeError, detail:
2153                ubody_text = unicode(body_text, 'iso-8859-15')
2154
2155            except LookupError, detail:
2156                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2157
2158            if self.parameters.verbatim_format:
2159                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2160            else:
2161                message_parts.append('%s' %ubody_text)
2162
2163        return message_parts
2164       
2165    def unique_attachment_names(self, message_parts):
2166        """
2167        Make sure we have unique names attachments:
2168          - check if it contains illegal characters
2169          - Rename "None" filenames to "untitled-part"
2170        """
2171        self.logger.debug('function unique_attachment_names()')
2172        renamed_parts = []
2173        attachment_names = set()
2174
2175        for item in message_parts:
2176           
2177            ## If not an attachment, leave it alone
2178            #
2179            if not isinstance(item, tuple):
2180                renamed_parts.append(item)
2181                continue
2182               
2183            (filename, part) = item
2184
2185            ## If filename = None, use a default one
2186            #
2187            if filename in [ 'None']:
2188                filename = 'untitled-part'
2189                self.logger.info('\t Rename filename "None" to: %s' %filename)
2190
2191                ## Guess the extension from the content type, use non strict mode
2192                #  some additional non-standard but commonly used MIME types
2193                #  are also recognized
2194                #
2195                ext = mimetypes.guess_extension(part.get_content_type(), False)
2196                if not ext:
2197                    ext = '.bin'
2198
2199                filename = '%s%s' % (filename, ext)
2200
2201            ## Discard relative paths for windows/unix in attachment names
2202            #
2203            filename = filename.replace('\\', '_')
2204            filename = filename.replace('/', '_')
2205
2206            ## remove linefeed char
2207            #
2208            for forbidden_char in ['\r', '\n']:
2209                filename = filename.replace(forbidden_char,'')
2210
2211            ## We try to normalize the filename to utf-8 NFC if we can.
2212            #  Files uploaded from OS X might be in NFD.
2213            #  Check python version and then try it
2214            #
2215            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2216            #   try:
2217            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2218            #   except TypeError:
2219            #       pass
2220
2221            ## Make the filename unique for this ticket
2222            #
2223            num = 0
2224            unique_filename = filename
2225            dummy_filename, ext = os.path.splitext(filename)
2226
2227            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2228                num += 1
2229                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2230               
2231            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2232            self.print_unicode(s)
2233
2234            attachment_names.add(unique_filename)
2235
2236            renamed_parts.append((filename, unique_filename, part))
2237   
2238        return renamed_parts
2239           
2240           
2241    def attachment_exists(self, filename):
2242
2243        self.logger.debug("function attachment_exists")
2244
2245        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2246        self.print_unicode(s)
2247
2248        ## Do we have a valid ticket id
2249        #
2250        if not self.id:
2251            return False
2252
2253        try:
2254            if self.system == 'discussion':
2255
2256                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2257
2258            elif self.system == 'blog':
2259
2260                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2261
2262            else:
2263
2264                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2265
2266            return True
2267
2268        except attachment.ResourceNotFound:
2269
2270            return False
2271
2272########## TRAC Ticket Text ###########################################################
2273           
2274    def get_body_text(self, message_parts):
2275        """
2276        """
2277        self.logger.debug('function get_body_text()')
2278
2279        body_text = []
2280       
2281        for part in message_parts:
2282       
2283            ## Plain text part, append it
2284            #
2285            if not isinstance(part, tuple):
2286                body_text.extend(part.strip().splitlines())
2287                body_text.append("")
2288                continue
2289
2290            (original, filename, part) = part
2291            inline = self.inline_part(part)
2292
2293            ## Skip generation of attachment link if html is converted to text
2294            #
2295            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2296                s = 'Skipping attachment link for html part: %s' %(filename)
2297                self.print_unicode(s)
2298                continue
2299           
2300            if part.get_content_maintype() == 'image' and inline:
2301
2302                if self.system != 'discussion':
2303                    s = 'wiki image link for: %s' %(filename)
2304                    self.print_unicode(s)
2305                    body_text.append('[[Image(%s)]]' % filename)
2306
2307                body_text.append("")
2308
2309            else:
2310
2311                if self.system != 'discussion':
2312
2313                    s = 'wiki attachment link for: %s' %(filename)
2314                    self.print_unicode(s)
2315                    body_text.append('[attachment:"%s"]' % filename)
2316
2317                body_text.append("")
2318
2319        ## Convert list body_texts to string
2320        #
2321        body_text = '\r\n'.join(body_text)
2322        return body_text
2323
2324    def html_mailto_link(self, subject):
2325        """
2326        This function returns a HTML mailto tag with the ticket id and author email address
2327        """
2328        self.logger.debug("function html_mailto_link")
2329
2330        if not self.author:
2331            author = self.email_addr
2332        else:   
2333            author = self.author
2334
2335        if not self.parameters.mailto_cc:
2336            self.parameters.mailto_cc = ''
2337           
2338        ## Bug in urllib.quote function
2339        #
2340        if isinstance(subject, unicode):
2341            subject = subject.encode('utf-8')
2342
2343        ## use urllib to escape the chars
2344        #
2345        s = '%s?Subject=%s&cc=%s' %(
2346               urllib.quote(self.email_addr),
2347               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2348               urllib.quote(self.parameters.mailto_cc)
2349               )
2350
2351        if self.VERSION in [ 0.10 ]:
2352            s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2353        else:
2354            s = '[mailto:"%s" Reply to: %s]' %(s, author)
2355
2356        self.logger.debug("\tmailto link %s" %s)
2357        return s
2358
2359########## TRAC notify section ###########################################################
2360
2361    def notify(self, tkt, new=True, modtime=0):
2362        """
2363        A wrapper for the TRAC notify function. So we can use templates
2364        """
2365        self.logger.debug('function notify()')
2366
2367        if self.parameters.notify_reporter:
2368            self.logger.debug('\t Notify reporter set')
2369            global sender_email
2370            sender_email = self.email_addr
2371 
2372            self.logger.debug('\t Using Email2TracNotification function AlwaysNotifyReporter')
2373            import trac.notification as Email2TracNotification
2374            Email2TracNotification.Notify.notify = AlwaysNotifyReporter
2375
2376        if self.parameters.dry_run  :
2377                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2378                return
2379        try:
2380
2381            #from trac.ticket.notification import TicketNotificationSystem
2382            #tn_sys = TicketNotificationSystem(self.env)
2383            #print tn_sys
2384            #print tn_sys.__dict__
2385            #sys.exit(0)
2386
2387            ## create false {abs_}href properties, to trick Notify()
2388            #
2389            if self.VERSION in [0.10]:
2390                self.env.abs_href = Href(self.get_config('project', 'url'))
2391                self.env.href = Href(self.get_config('project', 'url'))
2392
2393            tn = TicketNotifyEmail(self.env)
2394
2395            if self.parameters.alternate_notify_template:
2396
2397                if self.VERSION >= 0.11:
2398
2399                    from trac.web.chrome import Chrome
2400
2401                    if  self.parameters.alternate_notify_template_update and not new:
2402                        tn.template_name = self.parameters.alternate_notify_template_update
2403                    else:
2404                        tn.template_name = self.parameters.alternate_notify_template
2405
2406                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2407                       
2408                else:
2409
2410                    tn.template_name = self.parameters.alternate_notify_template
2411
2412            tn.notify(tkt, new, modtime)
2413
2414        except Exception, e:
2415            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2416
2417########## END Class Definition  ########################################################
2418
2419########## Global Notificaition Function ################################################
2420def AlwaysNotifyReporter(self, resid):
2421    """
2422    Copy of def notify() to manipulate recipents to always include reporter for the
2423    notification.
2424    """
2425    #print sender_email, resid
2426    (torcpts, ccrcpts) = self.get_recipients(resid)
2427    #print "get_recipients finished"
2428
2429    if not tktparser.email_header_acl('notify_reporter_black_list', sender_email, False):
2430        ## additionally append sender (regardeless of settings in trac.ini)
2431        #
2432        if not sender_email in torcpts:
2433            torcpts.append(sender_email)
2434
2435    self.begin_send()
2436    self.send(torcpts, ccrcpts)
2437    self.finish_send()
2438
2439########## Parse Config File  ###########################################################
2440
2441def ReadConfig(file, name):
2442    """
2443    Parse the config file
2444    """
2445    if not os.path.isfile(file):
2446        print 'File %s does not exist' %file
2447        sys.exit(1)
2448
2449    config = trac_config.Configuration(file)
2450   
2451    parentdir = config.get('DEFAULT', 'parentdir')
2452    sections = config.sections()
2453
2454    ## use some trac internals to get the defaults
2455    #
2456    tmp = config.parser.defaults()
2457    project =  SaraDict()
2458
2459    for option, value in tmp.items():
2460        try:
2461            project[option] = int(value)
2462        except ValueError:
2463            project[option] = value
2464
2465    if name:
2466        if name in sections:
2467            project =  SaraDict()
2468            for option, value in  config.options(name):
2469                try:
2470                    project[option] = int(value)
2471                except ValueError:
2472                    project[option] = value
2473
2474        elif not parentdir:
2475            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2476            print "or set parentdir in the [DEFAULT] section"
2477            sys.exit(1)
2478
2479    ## If parentdir then set project dir to parentdir + name
2480    #
2481    if not project.has_key('project'):
2482        if not parentdir:
2483            print "You must set project or parentdir in your configuration file"
2484            sys.exit(1)
2485        elif not name:
2486            print "You must configure a  project section in your configuration file"
2487        else:
2488            project['project'] = os.path.join(parentdir, name)
2489
2490    ##
2491    # Save the project name
2492    #
2493    project['project_name'] = os.path.basename(project['project'])
2494
2495    return project
2496
2497########## Setup Logging ###############################################################
2498
2499def setup_log(parameters, project_name, interactive=None):
2500    """
2501    Setup logging
2502
2503    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2504    would be interpreted by the ConfigParser itself.
2505    """
2506    logger = logging.getLogger('email2trac %s' %project_name)
2507
2508    if interactive:
2509        parameters.log_type = 'stderr'
2510
2511    if not parameters.log_type:
2512        if sys.platform in ['win32', 'cygwin']:
2513            parameters.log_type = 'eventlog'
2514        else:
2515            parameters.log_type = 'syslog'
2516
2517    if parameters.log_type == 'file':
2518
2519        if not parameters.log_file:
2520            parameters.log_file = 'email2trac.log'
2521
2522        if not os.path.isabs(parameters.log_file):
2523            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2524
2525        log_handler = logging.FileHandler(parameters.log_file)
2526
2527    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2528        ## Requires win32 extensions
2529        #
2530        logid = "email2trac"
2531        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2532
2533    elif parameters.log_type in ('syslog', 'unix'):
2534        log_handler = logging.handlers.SysLogHandler('/dev/log')
2535
2536    elif parameters.log_type in ('stderr'):
2537        log_handler = logging.StreamHandler(sys.stderr)
2538
2539    else:
2540        log_handler = logging.handlers.BufferingHandler(0)
2541
2542    if parameters.log_format:
2543        parameters.log_format = parameters.log_format.replace('$(', '%(')
2544    else:
2545        parameters.log_format = '%(name)s: %(message)s'
2546        if parameters.log_type in ('file', 'stderr'):
2547            parameters.log_format = '%(asctime)s ' + parameters.log_format
2548
2549    log_formatter = logging.Formatter(parameters.log_format)
2550    log_handler.setFormatter(log_formatter)
2551    logger.addHandler(log_handler)
2552
2553    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2554        logger.setLevel(logging.DEBUG)
2555        parameters.debug = 1
2556
2557    elif parameters.log_level in ['INFO'] or parameters.verbose:
2558        logger.setLevel(logging.INFO)
2559
2560    elif parameters.log_level in ['WARNING']:
2561        logger.setLevel(logging.WARNING)
2562
2563    elif parameters.log_level in ['ERROR']:
2564        logger.setLevel(logging.ERROR)
2565
2566    elif parameters.log_level in ['CRITICAL']:
2567        logger.setLevel(logging.CRITICAL)
2568
2569    else:
2570        logger.setLevel(logging.INFO)
2571
2572    return logger
2573
2574
2575if __name__ == '__main__':
2576    ## Default config file
2577    #
2578    configfile = '@email2trac_conf@'
2579    project = ''
2580    component = ''
2581    ticket_prefix = 'default'
2582    dry_run = None
2583    verbose = None
2584    debug_interactive = None
2585
2586    SHORT_OPT = 'cdhf:np:t:v'
2587    LONG_OPT  =  ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2588
2589    try:
2590        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2591    except getopt.error,detail:
2592        print __doc__
2593        print detail
2594        sys.exit(1)
2595   
2596    project_name = None
2597    for opt,value in opts:
2598        if opt in [ '-h', '--help']:
2599            print __doc__
2600            sys.exit(0)
2601        elif opt in ['-c', '--component']:
2602            component = value
2603        elif opt in ['-d', '--debug']:
2604            debug_interactive = 1
2605        elif opt in ['-f', '--file']:
2606            configfile = value
2607        elif opt in ['-n', '--dry-run']:
2608            dry_run = True
2609        elif opt in ['-p', '--project']:
2610            project_name = value
2611        elif opt in ['-t', '--ticket_prefix']:
2612            ticket_prefix = value
2613        elif opt in ['-v', '--verbose']:
2614            verbose = True
2615   
2616    settings = ReadConfig(configfile, project_name)
2617
2618    ## The default prefix for ticket values in email2trac.conf
2619    #
2620    settings.ticket_prefix = ticket_prefix
2621    settings.dry_run = dry_run
2622    settings.verbose = verbose
2623
2624    if not settings.debug and debug_interactive:
2625        settings.debug = debug_interactive
2626
2627    if not settings.project:
2628        print __doc__
2629        print 'No Trac project is defined in the email2trac config file.'
2630        sys.exit(1)
2631
2632    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2633   
2634    if component:
2635        settings['component'] = component
2636
2637    ## Determine major trac version used to be in email2trac.conf
2638    # Quick hack for 0.12
2639    #
2640    version = '0.%s' %(trac_version.split('.')[1])
2641    if version.startswith('0.12'):
2642        version = '0.12'
2643    elif version.startswith('0.13'):
2644        version = '0.13'
2645    elif trac_version.startswith("1.0"):
2646        version = '1.0'
2647
2648    logger.debug("Found trac version: %s" %(version))
2649   
2650    try:
2651        if version == '0.10':
2652            from trac import attachment
2653            from trac.env import Environment
2654            from trac.ticket import Ticket
2655            from trac.web.href import Href
2656            from trac import util
2657            from trac.ticket.web_ui import TicketModule
2658
2659            #
2660            # return  util.text.to_unicode(str)
2661            #
2662            # see http://projects.edgewall.com/trac/changeset/2799
2663            from trac.ticket.notification import TicketNotifyEmail
2664            from trac import config as trac_config
2665            from trac.core import TracError
2666
2667        elif version in ['0.11', '0.12', '0.13', '1.0']:
2668            from trac import attachment
2669            from trac import config as trac_config
2670            from trac import util
2671            from trac.core import TracError
2672            from trac.env import Environment
2673            from trac.perm import PermissionSystem
2674            from trac.perm import PermissionCache
2675            from trac.test import Mock, MockPerm
2676            from trac.ticket import Ticket
2677            from trac.ticket.api import TicketSystem
2678            from trac.ticket.web_ui import TicketModule
2679            from trac.web.href import Href
2680
2681            #
2682            # return  util.text.to_unicode(str)
2683            #
2684            # see http://projects.edgewall.com/trac/changeset/2799
2685            from trac.ticket.notification import TicketNotifyEmail
2686
2687        else:
2688            logger.error('TRAC version %s is not supported' %version)
2689            sys.exit(0)
2690
2691        ## Must be set before environment is created
2692        #
2693        if settings.has_key('python_egg_cache'):
2694            python_egg_cache = str(settings['python_egg_cache'])
2695            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2696
2697        if settings.debug > 0:
2698            logger.debug('Loading environment %s', settings.project)
2699
2700        try:
2701            env = Environment(settings['project'], create=0)
2702        except IOError, detail:
2703            logger.error("trac error: %s" %detail)
2704            sys.exit(0)
2705
2706        tktparser = TicketEmailParser(env, settings, logger, float(version))
2707        tktparser.parse(sys.stdin)
2708
2709    ## Catch all errors and use the logging module
2710    #
2711    except Exception, error:
2712
2713        etype, evalue, etb = sys.exc_info()
2714        for e in traceback.format_exception(etype, evalue, etb):
2715            logger.critical(e)
2716
2717        if m:
2718            tktparser.save_email_for_debug(m, settings.project_name, True)
2719
2720        sys.exit(1)
2721# EOB
Note: See TracBrowser for help on using the repository browser.