Attachment handling @IMAP

app/Entities.py

@@ -97,7 +97,6 @@ class QuarMail:
   msg_size = None
   href = None
   attach_count = None
-  attachments = None
 
   def __init__(self,qm_ref):
     if 'id' not in qm_ref:
@@ -140,9 +139,6 @@ class QuarMail:
       self.href = qm_ref['href']
     if 'attach_count' in qm_ref:
       self.attach_count = qm_ref['attach_count']
-    if 'attachments' in qm_ref:
-      self.attachments = qm_ref['attachments']
-
 
 class AttachmentException(Exception):
   message = None
@@ -153,7 +149,10 @@ class Attachment:
   id = None
   filename = None
   content_type = None
+  content_encoding = None
   comment = None
+  mailbox_id = None
+  imap_uid = None
   href = None
 
   def __init__(self,at_ref):
@@ -166,8 +165,16 @@ class Attachment:
     if 'content_type' not in at_ref:
       raise AttachmentException("'content_type' is mandatory!")
     self.content_type = at_ref['content_type']
+    if 'content_encoding' in at_ref:
+      self.content_encoding = at_ref['content_encoding']
     if 'comment' in at_ref:
       self.comment = at_ref['comment']
+    if 'mailbox_id' not in at_ref:
+      raise AttachmentException("'mailbox_id' is mandatory!")
+    self.mailbox_id = at_ref['mailbox_id']
+    if 'imap_uid' not in at_ref:
+      raise AttachmentException("'imap_uid' is mandatory!")
+    self.imap_uid = at_ref['imap_uid']
     if 'href' in at_ref:
       self.href = at_ref['href']
 

app/Gulag.py

@@ -43,8 +43,8 @@ class Gulag:
         quarmail_ids = []
         attachments = []
         uid = unseen['imap_uid']
-        msg = unseen['msg']
+        msg = email.message_from_bytes(unseen['msg'])
-        msg_size = len(str(msg))
+        msg_size = len(msg)
         r5321_from = email.header.decode_header(msg['Return-Path'])[0][0]
         if(r5321_from is not '<>'):
           r5321_from = r5321_from.replace("<","")
@@ -92,6 +92,7 @@ class Gulag:
             'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta',
             'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, 'msg_size': msg_size
           })
+          print("QuarMail (%s) imported" % (quarmail_id))
           quarmail_ids.append(quarmail_id)
         # Ende for rcpts
         # Alle MIME-Parts durchiterieren und Attachments
@@ -108,12 +109,13 @@ class Gulag:
               filename = filename[0][0]
             attach_id = self.db.add_attachment({
               'filename': filename,
-              'content_type': part.get_content_type()
+              'content_type': part.get_content_type(),
+              'content_encoding': part['Content-Transfer-Encoding']
             })
             attachments.append(attach_id)
           # Ende if part.get_filename()
         # Ende for msg.walk()
-        # QuarMails und Attachments verknüpfen
+        # QuarMail und Attachments verknüpfen
         if(len(attachments) > 0):
           for quarmail_id in quarmail_ids:
             for attachment_id in attachments:
@@ -145,12 +147,12 @@ class Gulag:
   def get_quarmail(self,args):
     qm_db = None
     try:
-      qm_db = self.db.get_quarmail({"id": args['id']})
+      qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
     except GulagDBException as e:
       raise GulagException("GulagDBException: " + e.message) from e
     if 'rfc822_message' not in args:
       return qm_db
-    # pull full RFC822 message from mailbox
+    # pull full RFC822 message from IMAP mailbox
     mailbox = None
     try:
       mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
@@ -159,19 +161,55 @@ class Gulag:
     imap_mb = None
     try:
       imap_mb = IMAPmailbox(mailbox)
-      qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid'])
+      qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid']).decode("utf-8")
       return qm_db
     except IMAPmailboxException as e:
       print(e.message)
       raise GulagException(e.message) from e
 
+  def get_quarmail_attachments(self,args):
+    try:
+      return self.db.get_quarmail_attachments(args['quarmail_id'])
+    except GulagDBException as e:
+      print(e.message)
+      raise GulagException(e.message) from e
+  
+  def get_quarmail_attachment(self,args):
+    qmat_db = None
+    try:
+      qmat_db = self.db.get_quarmail_attachment(
+        args['quarmail_id'],args['attachment_id']
+      )
+    except GulagDBException as e:
+      print(e.message)
+      raise GulagException(e.message) from e
+    if 'data' not in args:
+      return qmat_db
+    # pull attachment from IMAP mailbox
+    mailbox = None
+    try:
+      mailbox = self.db.get_mailbox(qmat_db['mailbox_id'])
+    except GulagDBException as e:
+      raise GulagException(e.message) from e 
+    imap_mb = None
+    try:
+      imap_mb = IMAPmailbox(mailbox)
+      qmat_db['data'] = imap_mb.get_attachment(
+        qmat_db['imap_uid'],qmat_db['filename']
+      )
+      return qmat_db
+    except IMAPmailboxException as e:
+      print(e.message)
+      raise GulagException(e.message) from e
+
   def get_attachment(self,args):
     at_db = None
     try:
       at_db = self.db.get_attachment({"id": args['id']})
-      return at_db
     except GulagDBException as e:
       raise GulagException(e.message) from e
+    if 'data' not in args:
+      return at_db
   
   def rspamd_http2imap(self,mailbox_id):
     mailbox = None

app/GulagDB.py

@@ -113,7 +113,6 @@ class GulagDB:
   def get_quarmails(self):
     try:
       cursor = self.conn.cursor()
-#      cursor.execute("select * from QuarMails;")
       query = "select *,(select count(*) from QuarMail2Attachment"
       query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count"
       query += " from QuarMails;"
@@ -159,10 +158,10 @@ class GulagDB:
         else:
           dict[name[0]] = value
       dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
-      try:
+#      try:
-        dict['attachments'] = self.get_attachments_by_quarmail(args['id'])
+#        dict['attachments'] = self.get_attachments_by_quarmail(args['id'])
-      except GulagDBException as e:
+#      except GulagDBException as e:
-        pass
+#        pass
       return QuarMail(dict).__dict__
     except mariadb.Error as e:
       raise GulagDBException(e) from e
@@ -191,17 +190,46 @@ class GulagDB:
     try:
       cursor = self.conn.cursor()
       cursor.execute("insert into Attachments " +
-        "(filename, content_type) values (%s,%s)",
+        "(filename, content_type, content_encoding) values (%s,%s,%s)",
-        (attach['filename'], attach['content_type'])
+        (attach['filename'], attach['content_type'], attach['content_encoding'])
       )
       return cursor.lastrowid
     except mariadb.Error as e:
       raise GulagDBException(e) from e
+ 
+  def get_attachments(self):
+    try:
+      query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
+      query += " from QuarMail2Attachment" 
+      query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
+      query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
+      query += " group by id;"
+      cursor = self.conn.cursor()
+      cursor.execute(query)
+      results = []
+      data = cursor.fetchall()
+      if not data:
+        raise GulagDBException("No attachments found!")
+      desc = cursor.description
+      for tuple in data:
+        dict = {}
+        for (name, value) in zip(desc, tuple):
+          dict[name[0]] = value
+        dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
+        results.append(Attachment(dict).__dict__)
+      return results
+    except mariadb.Error as e:
+      raise GulagDBException(e) from e
   
   def get_attachment(self, args):
     try:
       cursor = self.conn.cursor()
-      cursor.execute("select * from Attachments where id=" + str(args['id']) + ";")
+      query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
+      query += " from QuarMail2Attachment" 
+      query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
+      query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
+      query += " where id=" + str(args['id']) + ";"
+      cursor.execute(query)
       data = cursor.fetchall()
       if not data:
         raise GulagDBException("Attachment("+ str(args['id']) +") does not exist!")
@@ -215,9 +243,10 @@ class GulagDB:
     except mariadb.Error as e:
       raise GulagDBException(e) from e
   
-  def get_attachments_by_quarmail(self,quarmail_id):
+  def get_quarmail_attachments(self,quarmail_id):
     try:
-      query = "select Attachments.* from QuarMail2Attachment"
+      query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
+      query += " from QuarMail2Attachment"
       query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
       query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
       query += " where QuarMails.id = " + str(quarmail_id) + ";"
@@ -232,11 +261,39 @@ class GulagDB:
         dict = {}
         for (name, value) in zip(desc, tuple):
           dict[name[0]] = value
-        dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
+        dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id) 
+        dict['href'] += "/attachments/" + str(dict['id'])
         results.append(Attachment(dict).__dict__)
       return results
     except mariadb.Error as e:
       raise GulagDBException(e) from e
+
+  def get_quarmail_attachment(self,quarmail_id,attachment_id):
+    try:
+      query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
+      query += " from QuarMail2Attachment"
+      query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
+      query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
+      query += " where QuarMails.id = " + str(quarmail_id)
+      query += " and Attachments.id = " + str(attachment_id) + ";"
+      cursor = self.conn.cursor()
+      cursor.execute(query)
+      data = cursor.fetchall()
+      if not data:
+        raise GulagDBException("QuarMail("+ str(quarmail_id) +") "
+          + "has no attachment (" + str(attachment_id) + ")!"
+        )
+      desc = cursor.description
+      tuple = data[0]
+      dict = {}
+      for (name, value) in zip(desc, tuple):
+        dict[name[0]] = value
+      dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id)
+      dict['href'] += "/attachments/" + str(dict['id'])
+      return Attachment(dict).__dict__
+    except mariadb.Error as e:
+      raise GulagDBException(e) from e
+
   
   def quarmail2attachment(self,quarmail_id,attachment_id):
     try:

app/GulagMailbox.py

@@ -9,6 +9,7 @@ class IMAPmailboxException(Exception):
     self.message = str(message)
 
 class IMAPmailbox:
+  email_address = None
   imap_server = None
   imap_user = None
   imap_pass = None
@@ -16,6 +17,7 @@ class IMAPmailbox:
   mailbox = None
 
   def __init__(self, mb_ref):
+    self.email_address = mb_ref['email_address']
     self.imap_server = mb_ref['imap_server']
     self.imap_user = mb_ref['imap_user']
     self.imap_pass = mb_ref['imap_pass']
@@ -54,7 +56,7 @@ class IMAPmailbox:
         continue
       results.append({
         'imap_uid': uid,
-        'msg': email.message_from_bytes(data[0][1])
+        'msg': data[0][1]
       })
     return results
 
@@ -62,31 +64,29 @@ class IMAPmailbox:
     rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)')
     if rv != 'OK':
       raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
-    return data[0][1].decode("utf-8")
+    return data[0][1]
 
-  def get_attachments(self,imap_uid):
+  def get_attachment(self,imap_uid,filename):
-    results = []
+    msg = email.message_from_bytes(self.get_message(imap_uid))
-    rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)')
-    if rv != 'OK':
-      raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
-    msg = email.message_from_bytes(data[0][1])
     for part in msg.walk():
       if part.get_filename():
         # let´s define parts with filename as attachments
-        filename = email.header.decode_header(part.get_filename())
+        part_fn = email.header.decode_header(part.get_filename())
-        if filename[0][1]:
+        if part_fn[0][1]:
           # Encoded -> decode
-          filename = filename[0][0].decode(filename[0][1])
+          part_fn = part_fn[0][0].decode(part_fn[0][1])
         else:
           # not encoded
-          filename = filename[0][0]
+          part_fn = part_fn[0][0]
-        results.append({
+        print("C-T-E: " + str(part['Content-Transfer-Encoding']))
-          'filename': filename,
+        if(part_fn == filename):
-          'content-type': part.get_content_type(), 
+          return part.get_payload(decode=False)
-          'content': part.get_payload(decode=True)
-        })
       # End if part.get_filename()
-    return results
+    # End msg.walk() loop
+    raise IMAPmailboxException(
+      "Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@" 
+      + str(self.email_address) + " not found!"
+    )
 
   def append_message(self,message):
     rv, data = self.mailbox.append(

app/Resources.py

@@ -50,8 +50,8 @@ class ResQuarMails(GulagResource):
       abort(400, message=e.message)
 
 class ResQuarMail(GulagResource):
-  def get(self,id):
+  def get(self,quarmail_id):
-    args = {"id": id}
+    args = {"quarmail_id": quarmail_id}
     try:
       if(request.args.get('rfc822_message')):
         args['rfc822_message'] = True
@@ -59,13 +59,36 @@ class ResQuarMail(GulagResource):
     except GulagException as e:
       abort(400, message=e.message)
 
+class ResQuarMailAttachments(GulagResource):
+  def get(self,quarmail_id):
+    args = {"quarmail_id": quarmail_id}
+    if(request.args.get('data')):
+      args['data'] = True
+    try:
+      return self.gulag.get_quarmail_attachments(args)
+    except GulagException as e:
+      abort(400, message=e.message)
+
+class ResQuarMailAttachment(GulagResource):
+  def get(self,quarmail_id,attachment_id):
+    args = {
+      "quarmail_id": quarmail_id,
+      "attachment_id": attachment_id
+    }
+    if(request.args.get('data')):
+      args['data'] = True
+    try:
+      return self.gulag.get_quarmail_attachment(args)
+    except GulagException as e:
+      abort(400, message=e.message)
+
 class ResAttachments(GulagResource):
   def get(self):
     return {"resource": "Attachments"}
 
 class ResAttachment(GulagResource):
-  def get(self,id):
+  def get(self,attachment_id):
-    args = {"id": id}
+    args = {"id": attachment_id}
     try:
       return self.gulag.get_attachment(args)
     except GulagException as e:

app/gulag_helpers.py

@@ -20,7 +20,9 @@ if(importer_pid == 0):
     try:
       gulag.import_quarmails()
     except GulagException as e:
-      print("Importer-Exception: " + e.message)
+      print("Importer-Exception: " + e.message, file=sys.stderr)
+    except:
+      print("Importer-Exception: " + str(sys.exc_info()),file=sys.stderr)
     time.sleep(gulag.config['importer']['interval'])
 
 cleaner_pid = os.fork()
@@ -36,6 +38,8 @@ if(cleaner_pid == 0):
       gulag.cleanup_quarmails()
     except GulagException as e:
       print("Cleaner-Exception: " + e.message)
+    except:
+      print("Cleaner-Exception: " + str(sys.exc_info()),file=sys.stderr)
     time.sleep(gulag.config['cleaner']['interval'])
  
 # Parent
@@ -46,7 +50,7 @@ try:
   while True:
     time.sleep(10)
 except:
-  print("MAIN-EXCEPTION: " + str(sys.exc_info()))
+  print("Helpers MAIN-EXCEPTION: " + str(sys.exc_info()))
   # Destroy childs
   for child_pid in child_pids:
     print("Killing child pid: %s", child_pid)

app/gulag_server.py

@@ -5,8 +5,9 @@ from flask import Flask
 from flask_restful import Api
 from Gulag import Gulag,GulagException
 from Resources import (ResRoot,ResMailboxes,
-  ResQuarMails,ResQuarMail,ResAttachments,
+  ResQuarMails,ResQuarMail,ResQuarMailAttachments,
-  ResAttachment,ResRSPAMDImporter
+  ResQuarMailAttachment,ResAttachments,ResAttachment,
+  ResRSPAMDImporter
 )
 parser = argparse.ArgumentParser()
 parser.add_argument('--config', required=True, help="Path to config file")
@@ -32,7 +33,15 @@ try:
     resource_class_kwargs={'gulag_object': gulag}
   )
   api.add_resource(ResQuarMail,
-    '/api/v1/quarmails/<string:id>',
+    '/api/v1/quarmails/<int:quarmail_id>',
+    resource_class_kwargs={'gulag_object': gulag}
+  )
+  api.add_resource(ResQuarMailAttachments,
+    '/api/v1/quarmails/<int:quarmail_id>/attachments/',
+    resource_class_kwargs={'gulag_object': gulag}
+  )
+  api.add_resource(ResQuarMailAttachment,
+    '/api/v1/quarmails/<int:quarmail_id>/attachments/<int:attachment_id>',
     resource_class_kwargs={'gulag_object': gulag}
   )
   api.add_resource(ResAttachments,
@@ -40,7 +49,7 @@ try:
     resource_class_kwargs={'gulag_object': gulag}
   )
   api.add_resource(ResAttachment,
-    '/api/v1/attachments/<string:id>',
+    '/api/v1/attachments/<int:attachment_id>',
     resource_class_kwargs={'gulag_object': gulag}
   )
   api.add_resource(ResRSPAMDImporter,

db/gulag.sql

@@ -49,6 +49,7 @@ create table Attachments (
   id int unsigned auto_increment primary key,
   filename varchar(256) not null,
   content_type varchar(256) not null,
+  content_encoding varchar(64),
   comment varchar(256)
 )ENGINE = InnoDB;