source: trunk/email2trac.py.in @ 634

Last change on this file since 634 was 634, checked in by bas, 11 years ago

removed some leftovers/typo's

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 90.2 KB
Line 
1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
5#       Copyright 2002 SURFsara
6#
7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
10#
11#       http://www.apache.org/licenses/LICENSE-2.0
12#
13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
21"""
22email2trac.py -- Email -> TRAC tickets
23
24A MTA filter to create Trac tickets from inbound emails.
25
26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
29
30Authors:
31  Bas van der Vlies <bas.vandervlies@surfsara.nl>
32  Walter de Jong <walter.dejong@surfsara.nl>
33
34How to use
35----------
36 * See https://oss.trac.surfsara.nl/email2trac/
37
38 * Commandline opions:
39                -h,--help
40                -A, --agilo
41                -d, --debug
42                -E, --virtualenv <path>
43                -f,--file  <configuration file>
44                -n,--dry-run
45                -p, --project <project name>
46                -t, --ticket_prefix <name>
47                -v, --verbose
48
49SVN Info:
50        $Id: email2trac.py.in 634 2013-08-06 14:04:32Z bas $
51"""
52import os
53import sys
54import string
55import getopt
56import time
57import email
58import email.Iterators
59import email.Header
60import re
61import urllib
62import unicodedata
63import mimetypes
64import traceback
65import logging
66import logging.handlers
67import UserDict
68import tempfile
69
70from datetime import timedelta, datetime
71from stat import *
72
73## Some global variables
74#
75m = None
76
77class SaraDict(UserDict.UserDict):
78    def __init__(self, dictin = None):
79        UserDict.UserDict.__init__(self)
80        self.name = None
81       
82        if dictin:
83            if dictin.has_key('name'):
84                self.name = dictin['name']
85                del dictin['name']
86            self.data = dictin
87           
88    def get_value(self, name):
89        if self.has_key(name):
90            return self[name]
91        else:
92            return None
93               
94    def __repr__(self):
95        return repr(self.data)
96
97    def __str__(self):
98        return str(self.data)
99           
100    def __getattr__(self, name):
101        """
102        override the class attribute get method. Return the value
103        from the dictionary
104        """
105        if self.data.has_key(name):
106            return self.data[name]
107        else:
108            return None
109           
110    def __setattr__(self, name, value):
111        """
112        override the class attribute set method only when the UserDict
113        has set its class attribute
114        """
115        if self.__dict__.has_key('data'):
116            self.data[name] = value
117        else:
118            self.__dict__[name] = value
119
120    def __iter__(self):
121        return iter(self.data.keys())
122
123
124
125class TicketEmailParser(object):
126    env = None
127    comment = '> '
128
129    def __init__(self, env, parameters, logger, version):
130        self.env = env
131
132        # Database connection
133        #
134        self.db = None
135
136        # Save parameters
137        #
138        self.parameters = parameters
139        self.logger = logger
140
141        # Some useful mail constants
142        #
143        self.email_name = None
144        self.email_addr = None
145        self.email_from = None
146        self.author     = None
147        self.id         = None
148
149        ## Will be set to True if the user has ben registered
150        #
151        self.allow_registered_user = False
152       
153        self.STRIP_CONTENT_TYPES = list()
154
155        ## fields properties via body_text
156        #
157        self.properties = dict()
158
159        self.VERSION = version
160
161        self.get_config = self.env.config.get
162
163        ## init function ##
164        #
165        self.setup_parameters()
166
167    def setup_parameters(self):
168        if self.parameters.umask:
169            os.umask(self.parameters.umask)
170
171        if not self.parameters.spam_level:
172            self.parameters.spam_level = 0
173
174        if not self.parameters.spam_header:
175            self.parameters.spam_header = 'X-Spam-Score'
176
177        if not self.parameters.email_quote:
178            self.parameters.email_quote = '^> .*'
179            self.parameters.skip_line_regex = '^> .*'
180
181        if not self.parameters.ticket_update_by_subject_lookback:
182            self.parameters.ticket_update_by_subject_lookback = 30
183
184        if self.parameters.verbatim_format == None:
185            self.parameters.verbatim_format = 1
186
187        if self.parameters.reflow == None:
188            self.parameters.reflow = 1
189
190        if self.parameters.binhex:
191            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
192
193        if self.parameters.applesingle:
194            self.STRIP_CONTENT_TYPES.append('application/applefile')
195
196        if self.parameters.appledouble:
197            self.STRIP_CONTENT_TYPES.append('application/applefile')
198
199        if self.parameters.strip_content_types:
200            items = self.parameters.strip_content_types.split(',')
201            for item in items:
202                self.STRIP_CONTENT_TYPES.append(item.strip())
203
204        if self.parameters.tmpdir:
205            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
206        else:
207            self.parameters.tmpdir = os.path.normcase('/tmp')
208
209        if self.parameters.email_triggers_workflow == None:
210            self.parameters.email_triggers_workflow = 1
211
212        if not self.parameters.subject_field_separator:
213            self.parameters.subject_field_separator = '&'
214        else:
215            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
216
217        if not self.parameters.strip_signature_regex:
218            self.parameters.strip_signature_regex = '^-- $'
219            self.parameters.cut_line_regex = '^-- $'
220
221        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
222        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
223        self.smtp_replyto = self.get_config('notification', 'smtp_replyto')
224
225
226        self.system = None
227
228########## Email Header Functions ###########################################################
229
230    def spam(self, message):
231        """
232        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
233        # Note if Spam_level then '*' are included
234        """
235        self.logger.debug('function spam')
236
237        spam = False
238        if message.has_key(self.parameters.spam_header):
239            spam_l = string.split(message[self.parameters.spam_header])
240            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
241
242            try:
243                number = spam_l[0].count('*')
244            except IndexError, detail:
245                number = 0
246               
247            if number >= self.parameters.spam_level:
248                spam = True
249               
250        ## treat virus mails as spam
251        #
252        elif message.has_key('X-Virus-found'):         
253            spam = True
254
255        ## Astaro firewall spam handling
256        #
257        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
258            spam = True
259
260        ## How to handle SPAM messages
261        #
262        if self.parameters.drop_spam and spam:
263
264            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
265            return 'drop'   
266
267        elif spam:
268
269            return 'Spam'   
270        else:
271
272            return False
273
274    def email_header_acl(self, keyword, message_field, default):
275        """
276        This function wil check if the email address is allowed or denied
277        to send mail to the ticket list
278        """
279        self.logger.debug('function email_header_acl: %s' %keyword)
280
281        try:
282            mail_addresses = self.parameters[keyword]
283
284            # Check if we have an empty string
285            #
286            if not mail_addresses:
287                return default
288
289        except KeyError, detail:
290            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
291
292            return default
293
294        mail_addresses = string.split(mail_addresses, ',')
295        message_addresses = string.split(message_field, ',')
296
297        for entry in mail_addresses:
298            entry = entry.strip()
299            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
300
301            for addr in message_addresses:
302                addr = addr.strip()
303
304                if self.parameters.compare_function_list in [ 'matches', 'match']:
305                    s = '\t%s matches %s' %(addr, entry)
306                    result =  TO_RE.match(addr)
307                else:
308                    s = '\t%s contains %s' %(addr, entry)
309                    result =  TO_RE.search(addr)
310
311                if result:
312                    self.logger.debug(s)
313                    return True
314
315        return False
316
317    def email_header_txt(self, m):
318        """
319        Display To and CC addresses in description field
320        """
321        s = ''
322
323        if m['To'] and len(m['To']) > 0:
324            s = "'''To:''' %s\r\n" %(m['To'])
325        if m['Cc'] and len(m['Cc']) > 0:
326            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
327
328        return  self.email_to_unicode(s)
329
330
331    def get_sender_info(self, message):
332        """
333        Get the default author name and email address from the message
334        """
335        self.logger.debug('function get_sender_info')
336
337        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
338        self.email_to_addrs = list()
339
340        for n,e in to_addrs:
341            self.email_to_addrs.append(e)
342
343        self.email_to_addrs = ','.join(self.email_to_addrs)
344
345
346        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
347
348        ## decode email name can contain charset
349        #
350        self.email_name = self.email_to_unicode(self.email_name)
351
352        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
353        self.logger.debug(dstr)
354
355        ## Trac can not handle author's name that contains spaces
356        #
357        if self.email_addr.lower() == self.trac_smtp_from.lower():
358            if self.email_name:
359                self.author = self.email_name
360            else:
361                self.author = "email2trac"
362        else:
363            self.author = self.email_addr
364
365        if self.parameters.ignore_trac_user_settings:
366            return
367
368        # Is this a registered user, use email address as search key:
369        # result:
370        #   u : login name
371        #   n : Name that the user has set in the settings tab
372        #   e : email address that the user has set in the settings tab
373        #
374        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
375            if (
376                (e and (e.lower() == self.email_addr.lower())) or
377                (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
378            )
379            ]
380
381        if len(users) >= 1:
382            self.email_from = users[0][2]
383            self.author = users[0][0]
384
385            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
386
387            if self.parameters.white_list_registered_users:
388                self.logger.debug('\tget_sender_info: added registered user to white list')
389                self.allow_registered_user = True
390                   
391
392    def set_cc_fields(self, ticket, message, update=False):
393        """
394        Set all the right fields for a new ticket
395        """
396        self.logger.debug('function set_cc_fields')
397
398
399        # Put all CC-addresses in ticket CC field
400        #
401        if self.parameters.reply_all:
402
403            email_cc = ''
404
405            cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
406
407            if not cc_addrs:
408                return
409
410            ## Build a list of forbidden CC addresses
411            #
412            #to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
413            #to_list = list()
414            #for n,e in to_addrs:
415            #   to_list.append(e)
416
417            if update:
418                self.logger.debug("\tupdate cc-fields")
419
420            for name,addr in cc_addrs:
421       
422                ## Prevent mail loop
423                #
424                if addr == self.trac_smtp_from:
425                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
426                    continue
427
428                ## Alwyas remove reporter email address
429                #
430                elif addr == self.email_addr:
431                    self.logger.debug("\tSkipping reporter email address for CC-field")
432                    continue
433
434                else:
435                    if email_cc:
436                        email_cc = '%s, %s' %(email_cc, addr)
437                    else:
438                        email_cc = addr
439
440            if email_cc:
441                self.logger.debug('\tCC fields: %s' %email_cc)
442                ticket['cc'] = self.email_to_unicode(email_cc)
443
444
445    def acl_list_from_file(self, f, keyword):
446        """
447        Read the email address from a file
448        """
449        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
450       
451        if  not os.path.isfile(f):
452            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
453        else:
454            ## read whole file and replace '\n' with ''
455            #
456            addr_l = open(f, 'r').readlines()
457            s = ','.join(addr_l).replace('\n','')
458
459            try:
460                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
461            except KeyError, detail:
462                self.parameters[keyword] = s
463
464########## DEBUG functions  ###########################################################
465
466    def debug_body(self, message_body, temp_file):
467        """
468        """
469        self.logger.debug('function debug_body:')
470
471        body_file = "%s.body" %(temp_file)
472        fx = open(body_file, 'wb')
473
474        if self.parameters.dry_run:
475            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
476            return
477
478        self.logger.debug('writing body to %s' %(body_file))
479        if not message_body:
480                message_body = '(None)'
481
482        message_body = message_body.encode('utf-8')
483
484        fx.write(message_body)
485        fx.close()
486        try:
487            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
488        except OSError:
489            pass
490
491    def debug_attachments(self, message_parts, temp_file):
492        """
493        """
494        self.logger.debug('function debug_attachments')
495       
496        n = 0
497        for item in message_parts:
498
499            ## Skip inline text parts
500            #
501            if not isinstance(item, tuple):
502                continue
503               
504            (original, filename, part) = item
505            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
506
507            dummy_filename, ext = os.path.splitext(filename)
508
509            n = n + 1
510            part_name = 'part%d%s' %(n, ext)
511            part_file = "%s.%s" %(temp_file, part_name)
512
513            s = 'writing %s: filename: %s' %(part_file, filename)
514            self.print_unicode(s)
515
516            ## Forbidden chars, just write part files instead of names
517            #
518            #filename = filename.replace('\\', '_')
519            #filename = filename.replace('/', '_')
520            #filename = filename + '.att_email2trac'
521            # part_file = os.path.join(self.parameters.tmpdir, filename)
522            #part_file = util.text.unicode_quote(part_file)
523            #self.print_unicode(part_file)
524
525            if self.parameters.dry_run:
526                self.logger.info('DRY_RUN: NOT saving attachments')
527                continue
528
529            fx = open(part_file, 'wb')
530            text = part.get_payload(decode=1)
531
532            if not text:
533                text = '(None)'
534
535            fx.write(text)
536            fx.close()
537
538            try:
539                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
540            except OSError:
541                pass
542
543    def save_email_for_debug(self, message, project_name, create_tempfile=False):
544
545        if self.parameters.dry_run:
546            self.logger.debug('DRY_RUN: NOT saving email message')
547            return
548
549        if create_tempfile:
550            (fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
551            fx = os.fdopen(fd, 'wb')
552        else:
553            tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
554            fx = open(tmp_file, 'wb')
555
556
557        self.logger.debug('saving email to %s' %(tmp_file))
558        fx.write('%s' % message)
559        fx.close()
560       
561        try:
562            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
563        except OSError:
564            pass
565
566        message_parts = self.get_message_parts(message)
567        message_parts = self.unique_attachment_names(message_parts)
568        body_text = self.get_body_text(message_parts)
569        self.debug_body(body_text, tmp_file)
570        self.debug_attachments(message_parts, tmp_file)
571
572########## Conversion functions  ###########################################################
573
574    def email_to_unicode(self, message_str):
575        """
576        Email has 7 bit ASCII code, convert it to unicode with the charset
577        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
578        understands it.
579        """
580        self.logger.debug("function email_to_unicode")
581
582        results =  email.Header.decode_header(message_str)
583
584        s = None
585        for text,format in results:
586            if format:
587                try:
588                    temp = unicode(text, format)
589                except UnicodeError, detail:
590                    # This always works
591                    #
592                    msg = 'ERROR: Could not find charset: %s, please install' %format
593                    self.logger.error(msg)
594                    temp = unicode(text, 'iso-8859-15')
595                except LookupError, detail:
596                    msg = 'ERROR: Could not find charset: %s, please install' %format
597                    self.logger.error(msg)
598                    #temp = unicode(text, 'iso-8859-15')
599                    temp = message_str
600                       
601            else:
602                temp = string.strip(text)
603                temp = unicode(text, 'iso-8859-15')
604
605            if s:
606                s = '%s %s' %(s, temp)
607            else:
608                s = '%s' %temp
609
610        return s
611
612    def str_to_dict(self, s):
613        """
614        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
615        """
616        self.logger.debug("function str_to_dict")
617
618        fields = string.split(s, self.parameters.subject_field_separator)
619
620        result = dict()
621        for field in fields:
622            try:
623                index, value = string.split(field, '=')
624
625                # We can not change the description of a ticket via the subject
626                # line. The description is the body of the email
627                #
628                if index.lower() in ['description']:
629                    continue
630
631                if value:
632                    result[index.lower()] = value
633
634            except ValueError:
635                pass
636        return result
637
638    def print_unicode(self,s):
639        """
640        This function prints unicode strings if possible else it will quote it
641        """
642        try:
643            self.logger.debug(s)
644        except UnicodeEncodeError, detail:
645            self.logger.debug(util.text.unicode_quote(s))
646
647
648    def html_2_txt(self, data):
649        """
650        Various routines to convert html syntax to valid trac wiki syntax
651        """
652        self.logger.debug('function html_2_txt')
653
654        ## This routine make an safe html that can be include
655        #  in trac, but no further text processing can be done
656        #
657#       try:
658#           from lxml.html.clean import Cleaner
659#           tags_rm = list()
660#           tags_rm.append('body')
661#
662#           cleaner = Cleaner(remove_tags=tags_rm )
663#           parsed_data = cleaner.clean_html(data)
664#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
665#
666#           return parsed_data
667#           
668#       except ImportError::
669#           pass
670
671        parsed_data = None
672        if self.parameters.html2text_cmd:
673            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
674            f = os.fdopen(fd, 'w')
675
676            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
677            self.logger.debug('\t html2text conversion %s'%(cmd))
678   
679            if self.parameters.dry_run:
680                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
681
682            else:
683                f.write(data)
684                f.close()
685
686                lines = os.popen(cmd).readlines()
687                parsed_data =  ''.join(lines)
688
689                os.unlink(tmp_file)
690
691        else:
692            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
693
694        return parsed_data
695
696    def check_filename_length(self, name):
697        """
698        To bypass a bug in Trac
699        check if the filename lenght is not larger then OS limit.
700          yes : return truncated filename
701          no  : return unmodified filename
702        """
703        self.logger.debug('function check_filename_length: ')
704
705        if not name:
706            return 'None'
707       
708        dummy_filename, ext = os.path.splitext(name)
709
710        ## Trac uses this format
711        #
712        try:
713            quote_format = util.text.unicode_quote(dummy_filename)
714
715        except UnicodeDecodeError, detail:
716            ## last resort convert to unicode
717            #
718            dummy_filename = util.text.to_unicode(dummy_filename)
719            quote_format = util.text.unicode_quote(dummy_filename)
720
721        ## Determine max filename lenght
722        #
723        try:
724            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
725        except AttributeError, detail:
726            filemax_lenght = 240
727
728        if len(quote_format) <= filemax_length:
729            return name
730
731        else:
732            ## Truncate file to filemax_length and reserve room for extension
733            #  We must break on a boundry
734            #
735            length = filemax_length - 6
736
737            for i in range(0,10):
738
739                truncated = quote_format[ : (length - i)]
740
741                try:
742                    unqoute_truncated = util.text.unicode_unquote(truncated)
743                    unqoute_truncated = unqoute_truncated + ext
744
745                    self.print_unicode('\t ' + unqoute_truncated)
746
747                    break
748
749                except UnicodeDecodeError, detail:
750                    continue
751
752
753            return unqoute_truncated
754
755########## TRAC ticket functions  ###########################################################
756
757    def mail_workflow(self, tkt):
758        """
759        """
760        self.logger.debug('function mail_workflow: ')
761       
762        req = Mock(authname=self.author, perm=MockPerm(), args={})
763        ticket_system = TicketSystem(self.env)
764       
765        try:
766            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
767
768        except KeyError:
769            ## fallback for compability (Will be deprecated)
770            #  workflow can be none.
771            #
772            workflow = None
773            if tkt['status'] in ['closed']:
774                workflow = self.parameters.workflow
775
776        if workflow:
777
778            ## process all workflow implementations
779            #
780            tkt_module = TicketModule(self.env)
781            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
782
783            for field in field_changes.keys():
784
785                ## We have already processed these fields
786                #
787                if not field in ['summary', 'description']:
788                    s = 'workflow : %s, field %s : %s, by %s' \
789                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
790                    self.logger.debug(s)
791
792                    tkt[field] = field_changes[field]['new']
793
794            return True
795
796        else:
797            return False
798
799    def check_permission_participants(self, tkt, action):
800        """
801        Check if the mailer is allowed to update the ticket
802        """
803        self.logger.debug('function check_permission_participants %s')
804
805        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
806            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
807
808            return True
809
810        perm = PermissionSystem(self.env)
811        if perm.check_permission(action, self.author):
812            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
813
814            return True
815       
816        # Is the updater in the CC?
817        try:
818            cc_list = tkt['cc'].split(',')
819            for cc in cc_list:
820                if self.email_addr.lower() in cc.lower.strip():
821                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
822
823                    return True
824
825        except KeyError:
826            pass
827
828        return False
829
830    def check_permission(self, tkt, action):
831        """
832        check if the reporter has the right permission for the action:
833          - TICKET_CREATE
834          - TICKET_MODIFY
835          - TICKET_APPEND
836          - TICKET_CHGPROP
837
838        There are three models:
839            - None      : no checking at all
840            - trac      : check the permission via trac permission model
841            - email2trac: ....
842        """
843        self.logger.debug("function check_permission: %s" %(action))
844
845        if self.parameters.ticket_permission_system in ['trac']:
846
847            perm = PermissionCache(self.env, self.author)
848            if perm.has_permission(action):
849                return True
850            else:
851                return False
852
853        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
854            return (self.check_permission_participants(tkt, action))   
855
856        ## Default is to allow everybody ticket updates and ticket creation
857        #
858        else:
859                return True
860
861
862    def update_ticket_fields(self, ticket, user_dict, new=None):
863        """
864        This will update the ticket fields. It will check if the
865        given fields are known and if the right values are specified
866        It will only update the ticket field value:
867            - If the field is known
868            - If the value supplied is valid for the ticket field.
869              If not then there are two options:
870               1) Skip the value (new=None)
871               2) Set default value for field (new=1)
872        """
873        self.logger.debug("function update_ticket_fields")
874
875        ## Check only permission model on ticket updates
876        #
877        if not new:
878            if self.parameters.ticket_permission_system:
879                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
880                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
881                    return False
882
883        ## Build a system dictionary from the ticket fields
884        #  with field as index and option as value
885        #
886        sys_dict = dict()
887
888        for field in ticket.fields:
889
890            try:
891                sys_dict[field['name']] = field['options']
892
893            except KeyError:
894                sys_dict[field['name']] = None
895                pass
896
897        ## Check user supplied fields an compare them with the
898        #  system one's
899        #
900        for field,value in user_dict.items():
901       
902            s = 'user_field\t %s = %s' %(field,value)
903            self.print_unicode(s)
904
905            if not field in sys_dict.keys(): 
906                self.logger.debug('%s is not a valid field for tickets' %(field))
907                continue
908
909            ## To prevent mail loop
910            #
911            if field == 'cc':
912
913                cc_list = user_dict['cc'].split(',')
914
915                if self.trac_smtp_from in cc_list:
916                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
917
918                    cc_list.remove(self.trac_smtp_from)
919
920                value = ','.join(cc_list)
921               
922
923            ## Check if every value is allowed for this field
924            #
925            if sys_dict[field]:
926
927                if value in sys_dict[field]:
928                    ticket[field] = value
929                else:
930                    ## Must we set a default if value is not allowed
931                    #
932                    if new:
933                        value = self.get_config('ticket', 'default_%s' %(field) )
934
935            else:
936
937                ticket[field] = value
938
939            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
940            self.print_unicode(s)
941
942    def ticket_update(self, m, id, spam):
943        """
944        If the current email is a reply to an existing ticket, this function
945        will append the contents of this email to that ticket, instead of
946        creating a new one.
947        """
948        self.logger.debug("function ticket_update")
949
950        if not self.parameters.ticket_update:
951            self.logger.debug("ticket_update disabled")
952            return False
953
954        ## Must we update ticket fields
955        #
956        update_fields = dict()
957        try:
958            id, keywords = string.split(id, '?')
959
960            update_fields = self.str_to_dict(keywords)
961
962            ## Strip '#'
963            #
964            self.id = int(id[1:])
965
966        except ValueError:
967
968            ## Strip '#'
969            #
970            self.id = int(id[1:])
971
972        self.logger.debug("\tticket id: %s" %id)
973
974        ## When is the change committed
975        #
976        if self.VERSION < 0.11:
977            when = int(time.time())
978        else:
979            when = datetime.now(util.datefmt.utc)
980
981        try:
982            tkt = Ticket(self.env, self.id, self.db)
983
984        except util.TracError, detail:
985
986            ## Not a valid ticket
987            #
988            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
989            self.id = None
990            return False
991
992        ## Check the permission of the reporter
993        #
994        if self.parameters.ticket_permission_system:
995            if not self.check_permission(tkt, 'TICKET_APPEND'):
996                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
997                return False
998
999        ## How many changes has this ticket
1000        #
1001        grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1002        cnum = sum(1 for e in grouped) + 1
1003
1004
1005        ## reopen the ticket if it is was closed
1006        #  We must use the ticket workflow framework
1007        #
1008        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1009
1010            if not self.mail_workflow(tkt):
1011
1012                if tkt['status'] in ['closed']:
1013                    tkt['status'] = 'reopened'
1014                    tkt['resolution'] = ''
1015
1016        ## Old pre 0.11 situation
1017        #
1018        elif self.parameters.email_triggers_workflow:
1019
1020            self.logger.debug('\temail triggers workflow pre trac 0.11')
1021
1022            if tkt['status'] in ['closed']:
1023                tkt['status'] = 'reopened'
1024                tkt['resolution'] = ''
1025
1026        else:
1027            self.logger.debug('\temail triggers workflow disabled')
1028
1029        ## Must we update some ticket fields properties via subject line
1030        #
1031        if update_fields:
1032            self.update_ticket_fields(tkt, update_fields)
1033
1034
1035        message_parts = self.get_message_parts(m)
1036        message_parts = self.unique_attachment_names(message_parts)
1037
1038        ## Must we update some ticket fields properties via inline comments
1039        # in body_text
1040        #
1041        if self.properties:
1042                self.update_ticket_fields(tkt, self.properties)
1043
1044        ## Must we update the CC ticket field
1045        #
1046        self.set_cc_fields(tkt, m, update=True)
1047
1048        if self.parameters.email_header:
1049            message_parts.insert(0, self.email_header_txt(m))
1050
1051        body_text = self.get_body_text(message_parts)
1052
1053        error_with_attachments = self.attach_attachments(message_parts)
1054
1055        if body_text.strip() or update_fields or self.properties:
1056
1057            if self.parameters.dry_run:
1058                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1059                self.logger.info(s)
1060
1061            else:
1062                if error_with_attachments:
1063                    body_text = '%s\\%s' %(error_with_attachments, body_text)
1064
1065                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1066                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1067           
1068        if not spam:
1069            self.notify(tkt, False, when)
1070
1071        return True
1072
1073    def set_ticket_fields(self, ticket):
1074        """
1075        set the ticket fields to value specified
1076            - /etc/email2trac.conf with <prefix>_<field>
1077            - trac default values, trac.ini
1078        """
1079        self.logger.debug('function set_ticket_fields')
1080
1081        user_dict = dict()
1082
1083        for field in ticket.fields:
1084
1085            name = field['name']
1086
1087            ## default trac value
1088            #
1089            if not field.get('custom'):
1090                value = self.get_config('ticket', 'default_%s' %(name) )
1091
1092                ## skip this field can only be set by email2trac.conf
1093                #
1094                if name in ['resolution']:
1095                    value = None
1096
1097            else:
1098                ##  Else get the default value for reporter
1099                #
1100                value = field.get('value')
1101                options = field.get('options')
1102
1103                if value and options and (value not in options):
1104                     value = options[int(value)]
1105   
1106            s = 'trac[%s] = %s' %(name, value)
1107            self.print_unicode(s)
1108
1109            ## email2trac.conf settings
1110            #
1111            prefix = self.parameters.ticket_prefix
1112            try:
1113                value = self.parameters['%s_%s' %(prefix, name)]
1114
1115                s = 'email2trac[%s] = %s ' %(name, value)
1116                self.print_unicode(s)
1117
1118            except KeyError, detail:
1119                pass
1120       
1121            if value:
1122                user_dict[name] = value
1123
1124                s = 'used %s = %s' %(name, value)
1125                self.print_unicode(s)
1126
1127        self.update_ticket_fields(ticket, user_dict, new=1)
1128
1129        if 'status' not in user_dict.keys():
1130            ticket['status'] = 'new'
1131
1132
1133    def ticket_update_by_subject(self, subject):
1134        """
1135        This list of Re: prefixes is probably incomplete. Taken from
1136        wikipedia. Here is how the subject is matched
1137          - Re: <subject>
1138          - Re: (<Mail list label>:)+ <subject>
1139
1140        So we must have the last column
1141        """
1142        self.logger.debug('function ticket_update_by_subject')
1143
1144        found_id = None
1145        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1146               
1147            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1148            result = SUBJECT_RE.search(subject)
1149
1150            if result:
1151                ## This is a reply
1152                #
1153                orig_subject = result.group(1)
1154
1155                self.logger.debug('subject search string: %s' %(orig_subject))
1156
1157                cursor = self.db.cursor()
1158                summaries = [orig_subject, '%%: %s' % orig_subject]
1159
1160                ## Time resolution is in micoseconds
1161                #
1162                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1163
1164                if self.VERSION < 0.12:
1165                    lookback = util.datefmt.to_timestamp(search_date)
1166                else:
1167                    lookback = util.datefmt.to_utimestamp(search_date)
1168
1169                for summary in summaries:
1170                    self.logger.debug('Looking for summary matching: "%s"' % summary)
1171
1172                    sql = """SELECT id, reporter FROM ticket
1173                            WHERE changetime >= %s AND summary LIKE %s
1174                            ORDER BY changetime DESC"""
1175
1176                    cursor.execute(sql, [lookback, summary.strip()])
1177
1178                    for row in cursor:
1179
1180                        (matched_id, sql_reporter) = row
1181
1182                        ## Save first entry.
1183                        #
1184                        if not found_id:
1185                            found_id = matched_id
1186                           
1187                        ## If subject and reporter are the same. The we certainly have found the right ticket
1188                        #
1189                        if sql_reporter == self.author:
1190                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1191                            found_id = matched_id
1192                            break
1193
1194                    if found_id:
1195                        self.logger.debug('Found matching ticket id: %d' % found_id)
1196                        found_id = '#%d' % found_id
1197                        return (found_id, orig_subject)
1198                   
1199                    # obsolete !?? 12 Aug 2011
1200                    #subject = orig_subject
1201
1202        return (found_id, subject)
1203
1204
1205    def new_ticket(self, msg, subject, spam, set_fields = None):
1206        """
1207        Create a new ticket
1208        """
1209        self.logger.debug('function new_ticket')
1210
1211        tkt = Ticket(self.env)
1212
1213        ## self.author can be email address of an username
1214        #
1215        tkt['reporter'] = self.author
1216
1217        self.set_cc_fields(tkt, msg)
1218
1219        self.set_ticket_fields(tkt)
1220
1221        ## Check the permission of the reporter
1222        #
1223        if self.parameters.ticket_permission_system:
1224            if not self.check_permission(tkt, 'TICKET_CREATE'):
1225                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1226                return False
1227
1228        ## Old style setting for component, will be removed
1229        #
1230        if spam:
1231            tkt['component'] = 'Spam'
1232
1233        elif self.parameters.has_key('component'):
1234            tkt['component'] = self.parameters['component']
1235
1236        if not msg['Subject']:
1237            tkt['summary'] = u'(No subject)'
1238        else:
1239            tkt['summary'] = subject
1240
1241
1242        if set_fields:
1243            rest, keywords = string.split(set_fields, '?')
1244
1245            if keywords:
1246                update_fields = self.str_to_dict(keywords)
1247                self.update_ticket_fields(tkt, update_fields)
1248
1249
1250        message_parts = self.get_message_parts(msg, True)
1251
1252        ## Must we update some ticket fields properties via body_text
1253        #
1254        if self.properties:
1255                self.update_ticket_fields(tkt, self.properties)
1256
1257        message_parts = self.unique_attachment_names(message_parts)
1258       
1259        ## produce e-mail like header
1260        #
1261        head = ''
1262        if self.parameters.email_header:
1263            head = self.email_header_txt(msg)
1264            message_parts.insert(0, head)
1265           
1266        body_text = self.get_body_text(message_parts)
1267
1268        tkt['description'] = body_text
1269
1270        ## When is the change committed
1271        #
1272        if self.VERSION < 0.11:
1273            when = int(time.time())
1274        else:
1275            when = datetime.now(util.datefmt.utc)
1276
1277        if self.parameters.dry_run:
1278            self.logger.info('DRY_RUN: tkt.insert()')
1279        else:
1280            self.id = tkt.insert()
1281   
1282        changed = False
1283        comment = ''
1284
1285        ## some routines in trac are dependend on ticket id
1286        #  like alternate notify template
1287        #
1288        if self.parameters.alternate_notify_template:
1289            tkt['id'] = self.id
1290            changed = True
1291
1292        ## Rewrite the description if we have mailto enabled
1293        #
1294        if self.parameters.mailto_link:
1295            changed = True
1296            comment = u'\nadded mailto line\n'
1297
1298            #mailto = self.html_mailto_link( m['Subject'])
1299            mailto = self.html_mailto_link(subject)
1300
1301            tkt['description'] = u'%s\r\n%s%s\r\n' \
1302                %(head, mailto, body_text)
1303   
1304        ## Save the attachments to the ticket   
1305        #
1306        error_with_attachments =  self.attach_attachments(message_parts)
1307
1308        if error_with_attachments:
1309            changed = True
1310            comment = '%s\n%s\n' %(comment, error_with_attachments)
1311
1312        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1313            if self.mail_workflow(tkt):
1314                changed = True
1315
1316        if changed:
1317            if self.parameters.dry_run:
1318                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1319                self.logger.info(s)
1320
1321            else:
1322                tkt.save_changes(tkt['reporter'], comment)
1323
1324        if not spam:
1325            self.notify(tkt, True)
1326
1327
1328    def attach_attachments(self, message_parts, update=False):
1329        '''
1330        save any attachments as files in the ticket's directory
1331        '''
1332        self.logger.debug('function attach_attachments()')
1333
1334        if self.parameters.dry_run:
1335            self.logger.debug("DRY_RUN: no attachments attached to tickets")
1336            return ''
1337
1338        count = 0
1339
1340        ## Get Maxium attachment size
1341        #
1342        max_size = int(self.get_config('attachment', 'max_size'))
1343        status   = None
1344       
1345        for item in message_parts:
1346            ## Skip body parts
1347            #
1348            if not isinstance(item, tuple):
1349                continue
1350               
1351            (original, filename, part) = item
1352
1353            ## We have to determine the size so we use this temporary solution.
1354            #
1355            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1356            text = part.get_payload(decode=1)
1357            if not text:
1358                text = '(None)'
1359            fd.write(text)
1360            fd.close()
1361
1362            ## get the file_size
1363            #
1364            stats = os.lstat(path)
1365            file_size = stats[ST_SIZE]
1366
1367            ## Check if the attachment size is allowed
1368            #
1369            if (max_size != -1) and (file_size > max_size):
1370                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1371                    %(status, original, file_size, max_size)
1372
1373                os.unlink(path)
1374                continue
1375            else:
1376                count = count + 1
1377                   
1378            ## Insert the attachment
1379            #
1380            fd = open(path, 'rb')
1381            if self.system == 'discussion':
1382                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1383            elif self.system == 'blog':
1384                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1385            else:
1386                s = 'Attach %s to ticket %d' %(filename, self.id)
1387                self.print_unicode(s)
1388                att = attachment.Attachment(self.env, 'ticket', self.id)
1389 
1390            ## This will break the ticket_update system, the body_text is vaporized
1391            #  ;-(
1392            #
1393            if not update:
1394                att.author = self.author
1395                att.description = self.email_to_unicode('Added by email2trac')
1396
1397            try:
1398
1399                self.logger.debug('Insert atachment')
1400                att.insert(filename, fd, file_size)
1401
1402            except OSError, detail:
1403
1404                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1405                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1406
1407            ## Remove the created temporary filename
1408            #
1409            fd.close()
1410            os.unlink(path)
1411
1412        ## return error
1413        #
1414        return status
1415
1416########## Fullblog functions  #################################################
1417
1418    def blog(self, msg, subject, id, params):
1419        """
1420        The blog create/update function
1421        """
1422        ## import the modules
1423        #
1424        from tracfullblog.core import FullBlogCore
1425        from tracfullblog.model import BlogPost, BlogComment
1426
1427        ## instantiate blog core
1428        #
1429        blog = FullBlogCore(self.env)
1430        req = Mock(authname='anonymous', perm=MockPerm(), args={})
1431
1432        ## parameters from the subject
1433        #
1434        params = self.str_to_dict((params or '').lstrip('?'))
1435
1436        ## preferably get the time from the email date header, or else
1437        #  use current date and time
1438        date = email.Utils.parsedate_tz(msg.get('date'))
1439        if date:
1440            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1441        else:
1442            self.logger.warn("No valid date header found")
1443            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1444
1445        ## blog entry affected
1446        #
1447        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1448
1449        ## check wether a blog post exists
1450        #
1451        post = BlogPost(self.env, self.id)
1452        force_update = self.properties.get('update', params.get('update'))
1453
1454        ## message parts
1455        #
1456        message_parts = self.get_message_parts(msg)
1457        message_parts = self.unique_attachment_names(message_parts)
1458
1459        if post.get_versions() and not force_update:
1460
1461            ## add comment to blog entry
1462            #
1463            comment = BlogComment(self.env, self.id)
1464            comment.author = self.properties.get('author', params.get('author', self.author))
1465            comment.comment = self.get_body_text(message_parts)
1466            comment.time = dt
1467
1468            if self.parameters.dry_run:
1469                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1470                return
1471            warnings = blog.create_comment(req, comment)
1472
1473        else:
1474            ## create or update blog entry
1475            #
1476            post.author = self.properties.get('author', params.get('author', self.author))
1477            post.categories = self.properties.get('categories', params.get('categories', ''))
1478            post.title = subject.strip()
1479            post.publish_time = dt
1480            post.body = self.get_body_text(message_parts)
1481           
1482            if self.parameters.dry_run:
1483                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1484                return
1485            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1486
1487        ## check for problems
1488        #
1489        if warnings:
1490            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1491       
1492        ## all seems well, attach attachments
1493        #
1494        self.attach_attachments(message_parts)
1495
1496
1497########## Discussion functions  ##############################################
1498
1499    def discussion_topic(self, content, subject):
1500
1501        ## Import modules.
1502        #
1503        from tracdiscussion.api import DiscussionApi
1504        from trac.util.datefmt import to_timestamp, utc
1505
1506        self.logger.debug('Creating a new topic in forum:', self.id)
1507
1508        ## Get dissussion API component.
1509        #
1510        api = self.env[DiscussionApi]
1511        context = self._create_context(content, subject)
1512
1513        ## Get forum for new topic.
1514        #
1515        forum = api.get_forum(context, self.id)
1516
1517        if not forum:
1518            self.logger.error("ERROR: Replied forum doesn't exist")
1519
1520        ## Prepare topic.
1521        #
1522        topic = {'forum' : forum['id'],
1523                 'subject' : context.subject,
1524                 'time': to_timestamp(datetime.now(utc)),
1525                 'author' : self.author,
1526                 'subscribers' : [self.email_addr],
1527                 'body' : self.get_body_text(context.content_parts)}
1528
1529        ## Add topic to DB and commit it.
1530        #
1531        self._add_topic(api, context, topic)
1532        self.db.commit()
1533
1534    def discussion_topic_reply(self, content, subject):
1535
1536        ## Import modules.
1537        #
1538        from tracdiscussion.api import DiscussionApi
1539        from trac.util.datefmt import to_timestamp, utc
1540
1541        self.logger.debug('Replying to discussion topic', self.id)
1542
1543        ## Get dissussion API component.
1544        #
1545        api = self.env[DiscussionApi]
1546        context = self._create_context(content, subject)
1547
1548        ## Get replied topic.
1549        #
1550        topic = api.get_topic(context, self.id)
1551
1552        if not topic:
1553            self.logger.error("ERROR: Replied topic doesn't exist")
1554
1555        ## Prepare message.
1556        #
1557        message = {'forum' : topic['forum'],
1558                   'topic' : topic['id'],
1559                   'replyto' : -1,
1560                   'time' : to_timestamp(datetime.now(utc)),
1561                   'author' : self.author,
1562                   'body' : self.get_body_text(context.content_parts)}
1563
1564        ## Add message to DB and commit it.
1565        #
1566        self._add_message(api, context, message)
1567        self.db.commit()
1568
1569    def discussion_message_reply(self, content, subject):
1570
1571        ## Import modules.
1572        #
1573        from tracdiscussion.api import DiscussionApi
1574        from trac.util.datefmt import to_timestamp, utc
1575
1576        self.logger.debug('Replying to discussion message', self.id)
1577
1578        ## Get dissussion API component.
1579        #
1580        api = self.env[DiscussionApi]
1581        context = self._create_context(content, subject)
1582
1583        ## Get replied message.
1584        #
1585        message = api.get_message(context, self.id)
1586
1587        if not message:
1588            self.logger.error("ERROR: Replied message doesn't exist")
1589
1590        ## Prepare message.
1591        #
1592        message = {'forum' : message['forum'],
1593                   'topic' : message['topic'],
1594                   'replyto' : message['id'],
1595                   'time' : to_timestamp(datetime.now(utc)),
1596                   'author' : self.author,
1597                   'body' : self.get_body_text(context.content_parts)}
1598
1599        ## Add message to DB and commit it.
1600        #
1601        self._add_message(api, context, message)
1602        self.db.commit()
1603
1604    def _create_context(self, content, subject):
1605
1606        ## Import modules.
1607        #
1608        from trac.mimeview import Context
1609        from trac.web.api import Request
1610        from trac.perm import PermissionCache
1611
1612        ## TODO: Read server base URL from config.
1613        #  Create request object to mockup context creation.
1614        #
1615        environ = {'SERVER_PORT' : 80,
1616                   'SERVER_NAME' : 'test',
1617                   'REQUEST_METHOD' : 'POST',
1618                   'wsgi.url_scheme' : 'http',
1619                   'wsgi.input' : sys.stdin}
1620        chrome =  {'links': {},
1621                   'scripts': [],
1622                   'ctxtnav': [],
1623                   'warnings': [],
1624                   'notices': []}
1625
1626        if self.env.base_url_for_redirect:
1627            environ['trac.base_url'] = self.env.base_url
1628
1629        req = Request(environ, None)
1630        req.chrome = chrome
1631        req.tz = 'missing'
1632        req.authname = self.author
1633        req.perm = PermissionCache(self.env, self.author)
1634        req.locale = None
1635
1636        ## Create and return context.
1637        #
1638        context = Context.from_request(req)
1639        context.realm = 'discussion-email2trac'
1640        context.cursor = self.db.cursor()
1641        context.content = content
1642        context.subject = subject
1643
1644        ## Read content parts from content.
1645        #
1646        context.content_parts = self.get_message_parts(content)
1647        context.content_parts = self.unique_attachment_names(
1648          context.content_parts)
1649
1650        return context
1651
1652    def _add_topic(self, api, context, topic):
1653        context.req.perm.assert_permission('DISCUSSION_APPEND')
1654
1655        ## Filter topic.
1656        #
1657        for discussion_filter in api.discussion_filters:
1658            accept, topic_or_error = discussion_filter.filter_topic(
1659              context, topic)
1660            if accept:
1661                topic = topic_or_error
1662            else:
1663                raise TracError(topic_or_error)
1664
1665        ## Add a new topic.
1666        #
1667        api.add_topic(context, topic)
1668
1669        ## Get inserted topic with new ID.
1670        #
1671        topic = api.get_topic_by_time(context, topic['time'])
1672
1673        ## Attach attachments.
1674        #
1675        self.id = topic['id']
1676        self.attach_attachments(context.content_parts, True)
1677
1678        ## Notify change listeners.
1679        #
1680        for listener in api.topic_change_listeners:
1681            listener.topic_created(context, topic)
1682
1683    def _add_message(self, api, context, message):
1684        context.req.perm.assert_permission('DISCUSSION_APPEND')
1685
1686        ## Filter message.
1687        #
1688        for discussion_filter in api.discussion_filters:
1689            accept, message_or_error = discussion_filter.filter_message(
1690              context, message)
1691            if accept:
1692                message = message_or_error
1693            else:
1694                raise TracError(message_or_error)
1695
1696        ## Add message.
1697        #
1698        api.add_message(context, message)
1699
1700        ## Get inserted message with new ID.
1701        #
1702        message = api.get_message_by_time(context, message['time'])
1703
1704        ## Attach attachments.
1705        #
1706
1707        self.attach_attachments(context.content_parts, True)
1708
1709        ## Notify change listeners.
1710        #
1711        for listener in api.message_change_listeners:
1712            listener.message_created(context, message)
1713
1714########## MAIN function  ######################################################
1715
1716    def parse(self, fp):
1717        """
1718        """
1719        self.logger.debug('Main function parse')
1720        global m
1721
1722        m = email.message_from_file(fp)
1723       
1724        if not m:
1725            self.logger.debug('This is not a valid email message format')
1726            return
1727           
1728        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1729        #
1730        try:
1731            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1732        except AttributeError, detail:
1733            pass
1734
1735        if self.parameters.debug:     # save the entire e-mail message text
1736            self.save_email_for_debug(m, self.parameters.project_name, True)
1737
1738        if self.VERSION <= 0.11:
1739            self.db = self.env.get_db_cnx()
1740        else:
1741            self.db = self.env.get_read_db()
1742
1743        self.get_sender_info(m)
1744
1745
1746        if self.parameters.white_list_file:
1747            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1748
1749        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1750
1751            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1752            return False
1753
1754        if self.email_header_acl('black_list', self.email_addr, False):
1755            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1756            return False
1757
1758        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1759            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1760            return False
1761
1762        ## If spam drop the message
1763        #
1764        if self.spam(m) == 'drop':
1765            return False
1766
1767        elif self.spam(m) == 'spam':
1768            spam_msg = True
1769        else:
1770            spam_msg = False
1771
1772        if not m['Subject']:
1773            subject  = 'No Subject'
1774        else:
1775            subject  = self.email_to_unicode(m['Subject'])
1776
1777        ## Bug in python logging module <2.6
1778        #
1779        self.logger.info('subject: %s' %repr(subject))
1780
1781        ## First try the new method and then fall back to old method
1782        #
1783        if not self.parse_delivered_to_field(m, subject, spam_msg):
1784            self.parse_subject_field(m, subject, spam_msg)
1785
1786    def parse_delivered_to_field(self, m, subject, spam_msg):
1787        """
1788        """
1789        self.logger.debug('function parse_delivered_to_field')
1790
1791        ## Ticket id is in Delivered-To Field. Trac notify must send this
1792        #  eg: is+390@surfsara.nl
1793        try:
1794
1795            #print self.parameters.project_name
1796            self.logger.debug('Delivered To: %s' %m['Delivered-To'])
1797
1798            id = m['Delivered-To']
1799            id = id.split(self.parameters.recipient_delimiter)[1]
1800            id = id.split('@')[0]
1801
1802            self.logger.debug('\t Found ticket id: %s' %id)
1803
1804            if not self.ticket_update(m, id, spam_msg):
1805                return False
1806
1807        except KeyError, detail:
1808            pass
1809        except IndexError, detail:
1810            pass
1811
1812        return False
1813
1814    def parse_subject_field(self, m, subject, spam_msg):
1815        """
1816        """
1817        self.logger.debug('function parse_subject_header')
1818
1819        ## [hic] #1529: Re: LRZ
1820        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1821        #
1822        ticket_regex = r'''
1823            (?P<new_fields>[#][?].*)
1824            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1825            '''
1826
1827        ## Check if  FullBlogPlugin is installed
1828        #
1829        blog_enabled = None
1830        blog_regex = ''
1831        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1832            self.logger.debug('Trac BLOG support enabled')
1833            blog_enabled = True
1834            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
1835
1836
1837        ## Check if DiscussionPlugin is installed
1838        #
1839        discussion_enabled = None
1840        discussion_regex = ''
1841        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1842            self.logger.debug('Trac Discussion support enabled')
1843            discussion_enabled = True
1844            discussion_regex = r'''
1845            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1846            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1847            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1848            '''
1849
1850
1851        regex_str = ticket_regex + blog_regex + discussion_regex
1852        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1853
1854        ## Find out if this is a ticket, a blog or a discussion
1855        #
1856        result =  SYSTEM_RE.search(subject)
1857
1858        if result:
1859            ## update ticket + fields
1860            #
1861            if result.group('reply'):
1862                self.system = 'ticket'
1863
1864                ## Skip the last ':' character
1865                #
1866                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1867                    self.new_ticket(m, subject, spam_msg)
1868
1869            ## New ticket + fields
1870            #
1871            elif result.group('new_fields'):
1872                self.system = 'ticket'
1873                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1874
1875            if blog_enabled:
1876                if result.group('blog'):
1877                    self.system = 'blog'
1878                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1879
1880            if discussion_enabled:
1881                ## New topic.
1882                #
1883                if result.group('forum'):
1884                    self.system = 'discussion'
1885                    self.id = int(result.group('forum_id'))
1886                    self.discussion_topic(m, subject[result.end('forum'):])
1887
1888                ## Reply to topic.
1889                #
1890                elif result.group('topic'):
1891                    self.system = 'discussion'
1892                    self.id = int(result.group('topic_id'))
1893                    self.discussion_topic_reply(m, subject[result.end('topic'):])
1894
1895                ## Reply to topic message.
1896                #
1897                elif result.group('message'):
1898                    self.system = 'discussion'
1899                    self.id = int(result.group('message_id'))
1900                    self.discussion_message_reply(m, subject[result.end('message'):])
1901
1902        else:
1903
1904            self.system = 'ticket'
1905            (matched_id, subject) = self.ticket_update_by_subject(subject)
1906
1907            if matched_id:
1908
1909                if not self.ticket_update(m, matched_id, spam_msg):
1910                    self.new_ticket(m, subject, spam_msg)
1911
1912            else:
1913                ## No update by subject, so just create a new ticket
1914                #
1915                self.new_ticket(m, subject, spam_msg)
1916
1917
1918########## BODY TEXT functions  ###########################################################
1919
1920    def strip_signature(self, text):
1921        """
1922        Strip signature from message, inspired by Mailman software
1923        """
1924        self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
1925
1926        body = []
1927
1928        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1929        for line in text.splitlines():
1930
1931            match = STRIP_RE.match(line)
1932            if match:
1933                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
1934                break
1935
1936            body.append(line)
1937
1938        return ('\n'.join(body))
1939
1940    def reflow(self, text, delsp = 0):
1941        """
1942        Reflow the message based on the format="flowed" specification (RFC 3676)
1943        """
1944        flowedlines = []
1945        quotelevel = 0
1946        prevflowed = 0
1947
1948        for line in text.splitlines():
1949            from re import match
1950           
1951            ## Figure out the quote level and the content of the current line
1952            #
1953            m = match('(>*)( ?)(.*)', line)
1954            linequotelevel = len(m.group(1))
1955            line = m.group(3)
1956
1957            ## Determine whether this line is flowed
1958            #
1959            if line and line != '-- ' and line[-1] == ' ':
1960                flowed = 1
1961            else:
1962                flowed = 0
1963
1964            if flowed and delsp and line and line[-1] == ' ':
1965                line = line[:-1]
1966
1967            ## If the previous line is flowed, append this line to it
1968            #
1969            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1970                flowedlines[-1] += line
1971
1972            ## Otherwise, start a new line
1973            #
1974            else:
1975                flowedlines.append('>' * linequotelevel + line)
1976
1977            prevflowed = flowed
1978           
1979
1980        return '\n'.join(flowedlines)
1981
1982    def strip_quotes(self, text):
1983        """
1984        Strip quotes from message by Nicolas Mendoza
1985        """
1986        self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
1987
1988        body = []
1989
1990        STRIP_RE = re.compile( self.parameters.email_quote )
1991
1992        for line in text.splitlines():
1993
1994            try:
1995
1996                match = STRIP_RE.match(line)
1997                if match:
1998                    print line
1999                    continue
2000
2001            except UnicodeDecodeError:
2002
2003                tmp_line = self.email_to_unicode(line)
2004
2005                match = STRIP_RE.match(tmp_line)
2006                if match:
2007                    print line
2008                    continue
2009               
2010            body.append(line)
2011
2012        return ('\n'.join(body))
2013
2014    def inline_properties(self, text):
2015        """
2016        Parse text if we use inline keywords to set ticket fields
2017        """
2018        self.logger.debug('function inline_properties')
2019
2020        properties = dict()
2021        body = list()
2022
2023        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
2024
2025        for line in text.splitlines():
2026            match = INLINE_EXP.match(line)
2027            if match:
2028                keyword, value = match.groups()
2029
2030                if self.parameters.inline_properties_first_wins:
2031                    if keyword in self.properties.keys():
2032                        continue
2033
2034                self.properties[keyword] = value.strip()
2035                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
2036
2037            else:
2038                body.append(line)
2039               
2040        return '\n'.join(body)
2041
2042
2043    def wrap_text(self, text, replace_whitespace = False):
2044        """
2045        Will break a lines longer then given length into several small
2046        lines of size given length
2047        """
2048        import textwrap
2049
2050        LINESEPARATOR = '\n'
2051        reformat = ''
2052
2053        for s in text.split(LINESEPARATOR):
2054            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2055            if tmp:
2056                reformat = '%s\n%s' %(reformat,tmp)
2057            else:
2058                reformat = '%s\n' %reformat
2059
2060        return reformat
2061
2062        # Python2.4 and higher
2063        #
2064        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2065        #
2066
2067########## EMAIL attachements functions ###########################################################
2068
2069    def inline_part(self, part):
2070        """
2071        """
2072        self.logger.debug('function inline_part()')
2073
2074        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2075
2076    def get_message_parts(self, msg, new_email=False):
2077        """
2078        parses the email message and returns a list of body parts and attachments
2079        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2080        """
2081        self.logger.debug('function get_message_parts()')
2082
2083        message_parts = list()
2084   
2085        ALTERNATIVE_MULTIPART = False
2086
2087        for part in msg.walk():
2088            content_maintype = part.get_content_maintype()
2089            content_type =  part.get_content_type()
2090
2091            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2092            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2093
2094            ## Check content type
2095            #
2096            if content_type in self.STRIP_CONTENT_TYPES:
2097                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2098                continue
2099
2100            ## Catch some mulitpart execptions
2101            #
2102            if content_type == 'multipart/alternative':
2103                ALTERNATIVE_MULTIPART = True
2104                continue
2105
2106            ## Skip multipart containers
2107            #
2108            if content_maintype == 'multipart':
2109                self.logger.debug("\t Skipping multipart container")
2110                continue
2111           
2112            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2113            #  or if there is one and it says "inline"
2114            #
2115            inline = self.inline_part(part)
2116
2117            ## Drop HTML message
2118            #
2119            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2120
2121                if content_type == 'text/html':
2122                    self.logger.debug('\t Skipping alternative HTML message')
2123                    ALTERNATIVE_MULTIPART = False
2124                    continue
2125
2126            filename = part.get_filename()
2127            s = '\t unicode filename: %s' %(filename)
2128            self.print_unicode(s)
2129            self.logger.debug('\t raw filename: %s' %repr(filename))
2130
2131            filename = self.check_filename_length(filename)
2132
2133            ## Save all non plain text message as attachment
2134            #
2135            if not content_type in ['text/plain']:
2136
2137                message_parts.append( (filename, part) )
2138
2139                ## We only convert html messages
2140                #
2141                if not content_type == 'text/html':
2142                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2143                    continue
2144
2145
2146            ## We have an text or html message
2147            #
2148            if not inline:
2149                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2150                    message_parts.append( (filename, part) )
2151                    continue
2152               
2153            ## Try to decode message part. We have a html or plain text messafe
2154            #
2155            body_text = part.get_payload(decode=1)
2156            if not body_text:           
2157                body_text = part.get_payload(decode=0)
2158
2159            ## Try to convert html message
2160            #
2161            if content_type == 'text/html':
2162
2163                body_text = self.html_2_txt(body_text)
2164                if not body_text:
2165                    continue
2166
2167            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2168            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2169
2170            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2171                body_text = self.reflow(body_text, delsp == 'yes')
2172
2173            if new_email and self.parameters.only_strip_on_update:
2174                self.logger.debug('Skip signature/quote stripping for new messages')
2175            else:
2176                if self.parameters.strip_signature:
2177                    body_text = self.strip_signature(body_text)
2178
2179                if self.parameters.strip_quotes:
2180                    body_text = self.strip_quotes(body_text)
2181
2182            if self.parameters.inline_properties:
2183                body_text = self.inline_properties(body_text)
2184
2185            if self.parameters.use_textwrap:
2186                body_text = self.wrap_text(body_text)
2187
2188            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2189            #
2190            charset = part.get_content_charset()
2191            if not charset:
2192                charset = 'iso-8859-15'
2193
2194            try:
2195                ubody_text = unicode(body_text, charset)
2196
2197            except UnicodeError, detail:
2198                ubody_text = unicode(body_text, 'iso-8859-15')
2199
2200            except LookupError, detail:
2201                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2202
2203            if self.parameters.verbatim_format:
2204                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2205            else:
2206                message_parts.append('%s' %ubody_text)
2207
2208        return message_parts
2209       
2210    def unique_attachment_names(self, message_parts):
2211        """
2212        Make sure we have unique names attachments:
2213          - check if it contains illegal characters
2214          - Rename "None" filenames to "untitled-part"
2215        """
2216        self.logger.debug('function unique_attachment_names()')
2217        renamed_parts = []
2218        attachment_names = set()
2219
2220        for item in message_parts:
2221           
2222            ## If not an attachment, leave it alone
2223            #
2224            if not isinstance(item, tuple):
2225                renamed_parts.append(item)
2226                continue
2227               
2228            (filename, part) = item
2229
2230            ## If filename = None, use a default one
2231            #
2232            if filename in [ 'None']:
2233                filename = 'untitled-part'
2234                self.logger.info('\t Rename filename "None" to: %s' %filename)
2235
2236                ## Guess the extension from the content type, use non strict mode
2237                #  some additional non-standard but commonly used MIME types
2238                #  are also recognized
2239                #
2240                ext = mimetypes.guess_extension(part.get_content_type(), False)
2241                if not ext:
2242                    ext = '.bin'
2243
2244                filename = '%s%s' % (filename, ext)
2245
2246            ## Discard relative paths for windows/unix in attachment names
2247            #
2248            filename = filename.replace('\\', '_')
2249            filename = filename.replace('/', '_')
2250
2251            ## remove linefeed char
2252            #
2253            for forbidden_char in ['\r', '\n']:
2254                filename = filename.replace(forbidden_char,'')
2255
2256            ## We try to normalize the filename to utf-8 NFC if we can.
2257            #  Files uploaded from OS X might be in NFD.
2258            #  Check python version and then try it
2259            #
2260            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2261            #   try:
2262            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2263            #   except TypeError:
2264            #       pass
2265
2266            ## Make the filename unique for this ticket
2267            #
2268            num = 0
2269            unique_filename = filename
2270            dummy_filename, ext = os.path.splitext(filename)
2271
2272            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2273                num += 1
2274                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2275               
2276            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2277            self.print_unicode(s)
2278
2279            attachment_names.add(unique_filename)
2280
2281            renamed_parts.append((filename, unique_filename, part))
2282   
2283        return renamed_parts
2284           
2285           
2286    def attachment_exists(self, filename):
2287
2288        self.logger.debug("function attachment_exists")
2289
2290        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2291        self.print_unicode(s)
2292
2293        ## Do we have a valid ticket id
2294        #
2295        if not self.id:
2296            return False
2297
2298        try:
2299            if self.system == 'discussion':
2300
2301                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2302
2303            elif self.system == 'blog':
2304
2305                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2306
2307            else:
2308
2309                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2310
2311            return True
2312
2313        except attachment.ResourceNotFound:
2314
2315            return False
2316
2317########## TRAC Ticket Text ###########################################################
2318           
2319    def get_body_text(self, message_parts):
2320        """
2321        """
2322        self.logger.debug('function get_body_text()')
2323
2324        body_text = []
2325       
2326        for part in message_parts:
2327       
2328            ## Plain text part, append it
2329            #
2330            if not isinstance(part, tuple):
2331                body_text.extend(part.strip().splitlines())
2332                body_text.append("")
2333                continue
2334
2335            (original, filename, part) = part
2336            inline = self.inline_part(part)
2337
2338            ## Skip generation of attachment link if html is converted to text
2339            #
2340            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2341                s = 'Skipping attachment link for html part: %s' %(filename)
2342                self.print_unicode(s)
2343                continue
2344           
2345            if part.get_content_maintype() == 'image' and inline:
2346
2347                if self.system != 'discussion':
2348                    s = 'wiki image link for: %s' %(filename)
2349                    self.print_unicode(s)
2350                    body_text.append('[[Image(%s)]]' % filename)
2351
2352                body_text.append("")
2353
2354            else:
2355
2356                if self.system != 'discussion':
2357
2358                    s = 'wiki attachment link for: %s' %(filename)
2359                    self.print_unicode(s)
2360                    body_text.append('[attachment:"%s"]' % filename)
2361
2362                body_text.append("")
2363
2364        ## Convert list body_texts to string
2365        #
2366        body_text = '\r\n'.join(body_text)
2367        return body_text
2368
2369    def html_mailto_link(self, subject):
2370        """
2371        This function returns a HTML mailto tag with the ticket id and author email address
2372        """
2373        self.logger.debug("function html_mailto_link")
2374
2375        if not self.author:
2376            author = self.email_addr
2377        else:   
2378            author = self.author
2379
2380        if not self.parameters.mailto_cc:
2381            self.parameters.mailto_cc = ''
2382           
2383        ## Bug in urllib.quote function
2384        #
2385        if isinstance(subject, unicode):
2386            subject = subject.encode('utf-8')
2387
2388        ## use urllib to escape the chars
2389        #
2390        s = '%s?Subject=%s&cc=%s' %(
2391               urllib.quote(self.email_addr),
2392               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2393               urllib.quote(self.parameters.mailto_cc)
2394               )
2395
2396        if self.VERSION in [ 0.10 ]:
2397            s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2398        else:
2399            s = '[mailto:"%s" Reply to: %s]' %(s, author)
2400
2401        self.logger.debug("\tmailto link %s" %s)
2402        return s
2403
2404########## TRAC notify section ###########################################################
2405
2406    def notify(self, tkt, new=True, modtime=0):
2407        """
2408        A wrapper for the TRAC notify function. So we can use templates
2409        """
2410        self.logger.debug('function notify()')
2411
2412
2413        #if self.parameters.notification_replyto_rewrite:
2414        #    global REPLY_TO_ADDRESS
2415        #    dummy = self.smtp_replyto.split('@')
2416           
2417        #    if len(dummy) > 1:
2418        #        REPLY_TO_ADDRESS = '%s+%s@%s' %(dummy[0], self.id, dummy[1])
2419        #    else:
2420        #        REPLY_TO_ADDRESS = '%s+%s' %(dummy[0], self.id)
2421
2422        #    import trac.notification as Email2TracNotification
2423        #    Email2TracNotification.Notify.notify = set_reply_to_field
2424
2425        class Email2TracNotifyEmail(TicketNotifyEmail):
2426
2427            def __init__(self, env):
2428                TicketNotifyEmail.__init__(self, env)
2429                self.email2trac_notify_reporter = None
2430                self.email2trac_replyto = None
2431
2432            def send(self, torcpts, ccrcpts):
2433                #print 'Yes this works'
2434                dest = self.reporter or 'anonymous'
2435                hdrs = {}
2436                hdrs['Message-ID'] = self.get_message_id(dest, self.modtime)
2437                hdrs['X-Trac-Ticket-ID'] = str(self.ticket.id)
2438                hdrs['X-Trac-Ticket-URL'] = self.data['ticket']['link']
2439                if not self.newticket:
2440                    msgid = self.get_message_id(dest)
2441                    hdrs['In-Reply-To'] = msgid
2442                    hdrs['References'] = msgid
2443
2444
2445                if self.email2trac_notify_reporter:
2446                    if not self.email2trac_notify_reporter in torcpts:
2447                        torcpts.append(sender_email)
2448
2449                if self.email2trac_replyto:
2450                    # use to rewrite reply to
2451                    # hdrs does not work, multiple reply addresses
2452                    #hdrs['Reply-To'] = 'bas.van.der.vlies@gmail.com'
2453                    self.replyto_email = self.email2trac_replyto
2454       
2455                NotifyEmail.send(self, torcpts, ccrcpts, hdrs)
2456
2457        if self.parameters.dry_run  :
2458                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2459                return
2460        try:
2461
2462            ## create false {abs_}href properties, to trick Notify()
2463            #
2464            if self.VERSION in [0.10]:
2465                self.env.abs_href = Href(self.get_config('project', 'url'))
2466                self.env.href = Href(self.get_config('project', 'url'))
2467
2468            tn = Email2TracNotifyEmail(self.env)
2469
2470            ## additionally append sender (regardeless of settings in trac.ini)
2471            #
2472            if self.parameters.notify_reporter:
2473
2474                self.logger.debug('\t Notify reporter set')
2475
2476                if not self.email_header_acl('notify_reporter_black_list', self.email_addr, False):
2477                    tn.email2trac_notify_reporter = self.email_addr
2478
2479            if self.parameters.notify_replyto_rewrite:
2480
2481                self.logger.debug('\t Notify replyto rewrite set')
2482
2483                action, value = self.parameters.notify_replyto_rewrite.split(':')
2484
2485                if action in ['use_mail_domain']:
2486                    self.logger.debug('\t\t use_mail_domain:%s' %value)
2487                    tn.email2trac_replyto = '%s@%s' %(self.id, value)
2488
2489                elif action in ['use_trac_smtp_replyto']:
2490                    self.logger.debug('\t\t use_trac_smtp_replyto delimiter:%s' %value)
2491                    dummy = self.smtp_replyto.split('@')
2492                    if len(dummy) > 1:
2493                        tn.email2trac_replyto = '%s%s%s@%s' %(dummy[0], value, self.id, dummy[1])
2494                    else:
2495                        tn.email2trac_replyto = '%s%s%s' %(dummy[0], value, self.id)
2496
2497            if self.parameters.alternate_notify_template:
2498
2499                if self.VERSION >= 0.11:
2500
2501                    from trac.web.chrome import Chrome
2502
2503                    if  self.parameters.alternate_notify_template_update and not new:
2504                        tn.template_name = self.parameters.alternate_notify_template_update
2505                    else:
2506                        tn.template_name = self.parameters.alternate_notify_template
2507
2508                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2509                       
2510                else:
2511
2512                    tn.template_name = self.parameters.alternate_notify_template
2513
2514            tn.notify(tkt, new, modtime)
2515
2516        except Exception, e:
2517            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2518
2519########## END Class Definition  ########################################################
2520
2521
2522########## Parse Config File  ###########################################################
2523
2524def ReadConfig(file, name):
2525    """
2526    Parse the config file
2527    """
2528    if not os.path.isfile(file):
2529        print 'File %s does not exist' %file
2530        sys.exit(1)
2531
2532    config = trac_config.Configuration(file)
2533   
2534    parentdir = config.get('DEFAULT', 'parentdir')
2535    sections = config.sections()
2536
2537    ## use some trac internals to get the defaults
2538    #
2539    tmp = config.parser.defaults()
2540    project =  SaraDict()
2541
2542    for option, value in tmp.items():
2543        try:
2544            project[option] = int(value)
2545        except ValueError:
2546            project[option] = value
2547
2548    if name:
2549        if name in sections:
2550            project =  SaraDict()
2551            for option, value in  config.options(name):
2552                try:
2553                    project[option] = int(value)
2554                except ValueError:
2555                    project[option] = value
2556
2557        elif not parentdir:
2558            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2559            print "or set parentdir in the [DEFAULT] section"
2560            sys.exit(1)
2561
2562    ## If parentdir then set project dir to parentdir + name
2563    #
2564    if not project.has_key('project'):
2565        if not parentdir:
2566            print "You must set project or parentdir in your configuration file"
2567            sys.exit(1)
2568        elif not name:
2569            print "You must configure a  project section in your configuration file"
2570        else:
2571            project['project'] = os.path.join(parentdir, name)
2572
2573    ##
2574    # Save the project name
2575    #
2576    project['project_name'] = os.path.basename(project['project'])
2577
2578    return project
2579
2580########## Setup Logging ###############################################################
2581
2582def setup_log(parameters, project_name, interactive=None):
2583    """
2584    Setup logging
2585
2586    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2587    would be interpreted by the ConfigParser itself.
2588    """
2589    logger = logging.getLogger('email2trac %s' %project_name)
2590
2591    if interactive:
2592        parameters.log_type = 'stderr'
2593
2594    if not parameters.log_type:
2595        if sys.platform in ['win32', 'cygwin']:
2596            parameters.log_type = 'eventlog'
2597        else:
2598            parameters.log_type = 'syslog'
2599
2600    if parameters.log_type == 'file':
2601
2602        if not parameters.log_file:
2603            parameters.log_file = 'email2trac.log'
2604
2605        if not os.path.isabs(parameters.log_file):
2606            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2607
2608        log_handler = logging.FileHandler(parameters.log_file)
2609
2610    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2611        ## Requires win32 extensions
2612        #
2613        logid = "email2trac"
2614        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2615
2616    elif parameters.log_type in ('syslog', 'unix'):
2617        log_handler = logging.handlers.SysLogHandler('/dev/log')
2618
2619    elif parameters.log_type in ('stderr'):
2620        log_handler = logging.StreamHandler(sys.stderr)
2621
2622    else:
2623        log_handler = logging.handlers.BufferingHandler(0)
2624
2625    if parameters.log_format:
2626        parameters.log_format = parameters.log_format.replace('$(', '%(')
2627    else:
2628        parameters.log_format = '%(name)s: %(message)s'
2629        if parameters.log_type in ('file', 'stderr'):
2630            parameters.log_format = '%(asctime)s ' + parameters.log_format
2631
2632    log_formatter = logging.Formatter(parameters.log_format)
2633    log_handler.setFormatter(log_formatter)
2634    logger.addHandler(log_handler)
2635
2636    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2637        logger.setLevel(logging.DEBUG)
2638        parameters.debug = 1
2639
2640    elif parameters.log_level in ['INFO'] or parameters.verbose:
2641        logger.setLevel(logging.INFO)
2642
2643    elif parameters.log_level in ['WARNING']:
2644        logger.setLevel(logging.WARNING)
2645
2646    elif parameters.log_level in ['ERROR']:
2647        logger.setLevel(logging.ERROR)
2648
2649    elif parameters.log_level in ['CRITICAL']:
2650        logger.setLevel(logging.CRITICAL)
2651
2652    else:
2653        logger.setLevel(logging.INFO)
2654
2655    return logger
2656
2657########## Own TicketNotifyEmail class ###############################################################
2658
2659if __name__ == '__main__':
2660    ## Default config file
2661    #
2662    agilo = False
2663    configfile = '@email2trac_conf@'
2664    project = ''
2665    component = ''
2666    ticket_prefix = 'default'
2667    dry_run = None
2668    verbose = None
2669    debug_interactive = None
2670    virtualenv = '@virtualenv@'
2671
2672    SHORT_OPT = 'AcdE:hf:np:t:v'
2673    LONG_OPT  =  ['agilo', 'component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'virtualenv=', 'verbose']
2674
2675    try:
2676        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2677    except getopt.error,detail:
2678        print __doc__
2679        print detail
2680        sys.exit(1)
2681   
2682    project_name = None
2683    for opt,value in opts:
2684        if opt in [ '-h', '--help']:
2685            print __doc__
2686            sys.exit(0)
2687        elif opt in ['-A', '--agilo']:
2688            agilo = True
2689        elif opt in ['-c', '--component']:
2690            component = value
2691        elif opt in ['-d', '--debug']:
2692            debug_interactive = 1
2693        elif opt in ['-E', '--virtualenv']:
2694            virtualenv = value
2695        elif opt in ['-f', '--file']:
2696            configfile = value
2697        elif opt in ['-n', '--dry-run']:
2698            dry_run = True
2699        elif opt in ['-p', '--project']:
2700            project_name = value
2701        elif opt in ['-t', '--ticket_prefix']:
2702            ticket_prefix = value
2703        elif opt in ['-v', '--verbose']:
2704            verbose = True
2705
2706    if virtualenv and os.path.exists(virtualenv):
2707        activate_this = os.path.join(virtualenv, 'bin/activate_this.py')
2708        if os.path.exists(activate_this):
2709            execfile(activate_this, dict(__file__=activate_this))
2710
2711    try:
2712        from trac import __version__ as trac_version
2713        from trac import config as trac_config
2714
2715    except ImportError, detail:
2716        print "Can not find a a valid trac installation, solutions could be:"
2717        print "\tset PYTHONPATH"
2718        print "\tuse the --virtualenv <dir> option"
2719        sys.exit(1)
2720   
2721    settings = ReadConfig(configfile, project_name)
2722
2723    ## The default prefix for ticket values in email2trac.conf
2724    #
2725    settings.ticket_prefix = ticket_prefix
2726    settings.dry_run = dry_run
2727    settings.verbose = verbose
2728
2729    if not settings.debug and debug_interactive:
2730        settings.debug = debug_interactive
2731
2732    if not settings.project:
2733        print __doc__
2734        print 'No Trac project is defined in the email2trac config file.'
2735        sys.exit(1)
2736
2737    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2738   
2739    if component:
2740        settings['component'] = component
2741
2742    ## We are only interested in the major versions
2743    # 0.12.3 --> 0.12
2744    # 1.0.2  --> 1.0
2745    #
2746    l = trac_version.split('.')
2747    version = '.'.join(l[0:2])
2748
2749    logger.debug("Found trac version: %s" %(version))
2750   
2751    try:
2752        if version == '0.10':
2753            from trac import attachment
2754            from trac.env import Environment
2755            from trac.ticket import Ticket
2756            from trac.web.href import Href
2757            from trac import util
2758            from trac.ticket.web_ui import TicketModule
2759
2760            #
2761            # return  util.text.to_unicode(str)
2762            #
2763            # see http://projects.edgewall.com/trac/changeset/2799
2764            from trac.ticket.notification import TicketNotifyEmail
2765            from trac import config as trac_config
2766            from trac.core import TracError
2767
2768        elif version in ['0.11', '0.12', '0.13', '1.0', '1.1']:
2769            from trac import attachment
2770            from trac import config as trac_config
2771            from trac import util
2772            from trac.core import TracError
2773            from trac.env import Environment
2774            from trac.perm import PermissionSystem
2775            from trac.perm import PermissionCache
2776            from trac.test import Mock, MockPerm
2777            from trac.ticket.api import TicketSystem
2778            from trac.ticket.web_ui import TicketModule
2779            from trac.web.href import Href
2780
2781            if agilo:
2782
2783                try:
2784                    #from agilo.utils.config import AgiloConfig
2785                    #if AgiloConfig(self.env).is_agilo_enabled:
2786                    from agilo.ticket.model import Ticket
2787                except ImportError, detail:
2788                    logger.error('Could not find Trac  Agilo environemnt')
2789                    sys.exit(0)
2790
2791            else:
2792
2793                from trac.ticket import Ticket
2794
2795            #
2796            # return  util.text.to_unicode(str)
2797            #
2798            # see http://projects.edgewall.com/trac/changeset/2799
2799            from trac.ticket.notification import TicketNotifyEmail
2800            from trac.notification import NotifyEmail
2801
2802        else:
2803            logger.error('TRAC version %s is not supported' %version)
2804            sys.exit(0)
2805
2806        ## Must be set before environment is created
2807        #
2808        if settings.has_key('python_egg_cache'):
2809            python_egg_cache = str(settings['python_egg_cache'])
2810            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2811
2812        if settings.debug > 0:
2813            logger.debug('Loading environment %s', settings.project)
2814
2815        try:
2816            env = Environment(settings['project'], create=0)
2817        except IOError, detail:
2818            logger.error("trac error: %s" %detail)
2819            sys.exit(0)
2820        except TracError, detail:
2821            logger.error("trac error: %s" %detail)
2822            sys.exit(0)
2823
2824        tktparser = TicketEmailParser(env, settings, logger, float(version))
2825        tktparser.parse(sys.stdin)
2826
2827    ## Catch all errors and use the logging module
2828    #
2829    except Exception, error:
2830
2831        etype, evalue, etb = sys.exc_info()
2832        for e in traceback.format_exception(etype, evalue, etb):
2833            logger.critical(e)
2834
2835        if m:
2836            tktparser.save_email_for_debug(m, settings.project_name, True)
2837
2838        sys.exit(1)
2839
2840# EOB
Note: See TracBrowser for help on using the repository browser.