diff --git a/app/Entities.py b/app/Entities.py index 4ff6eff..c4a4323 100644 --- a/app/Entities.py +++ b/app/Entities.py @@ -97,7 +97,6 @@ class QuarMail: msg_size = None href = None attach_count = None - attachments = None def __init__(self,qm_ref): if 'id' not in qm_ref: @@ -140,9 +139,6 @@ class QuarMail: self.href = qm_ref['href'] if 'attach_count' in qm_ref: self.attach_count = qm_ref['attach_count'] - if 'attachments' in qm_ref: - self.attachments = qm_ref['attachments'] - class AttachmentException(Exception): message = None @@ -153,7 +149,10 @@ class Attachment: id = None filename = None content_type = None + content_encoding = None comment = None + mailbox_id = None + imap_uid = None href = None def __init__(self,at_ref): @@ -166,8 +165,16 @@ class Attachment: if 'content_type' not in at_ref: raise AttachmentException("'content_type' is mandatory!") self.content_type = at_ref['content_type'] + if 'content_encoding' in at_ref: + self.content_encoding = at_ref['content_encoding'] if 'comment' in at_ref: self.comment = at_ref['comment'] + if 'mailbox_id' not in at_ref: + raise AttachmentException("'mailbox_id' is mandatory!") + self.mailbox_id = at_ref['mailbox_id'] + if 'imap_uid' not in at_ref: + raise AttachmentException("'imap_uid' is mandatory!") + self.imap_uid = at_ref['imap_uid'] if 'href' in at_ref: self.href = at_ref['href'] diff --git a/app/Gulag.py b/app/Gulag.py index 931e973..40bc966 100644 --- a/app/Gulag.py +++ b/app/Gulag.py @@ -43,8 +43,8 @@ class Gulag: quarmail_ids = [] attachments = [] uid = unseen['imap_uid'] - msg = unseen['msg'] - msg_size = len(str(msg)) + msg = email.message_from_bytes(unseen['msg']) + msg_size = len(msg) r5321_from = email.header.decode_header(msg['Return-Path'])[0][0] if(r5321_from is not '<>'): r5321_from = r5321_from.replace("<","") @@ -92,6 +92,7 @@ class Gulag: 'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta', 'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, 'msg_size': msg_size }) + print("QuarMail (%s) imported" % (quarmail_id)) quarmail_ids.append(quarmail_id) # Ende for rcpts # Alle MIME-Parts durchiterieren und Attachments @@ -108,12 +109,13 @@ class Gulag: filename = filename[0][0] attach_id = self.db.add_attachment({ 'filename': filename, - 'content_type': part.get_content_type() + 'content_type': part.get_content_type(), + 'content_encoding': part['Content-Transfer-Encoding'] }) attachments.append(attach_id) # Ende if part.get_filename() # Ende for msg.walk() - # QuarMails und Attachments verknüpfen + # QuarMail und Attachments verknüpfen if(len(attachments) > 0): for quarmail_id in quarmail_ids: for attachment_id in attachments: @@ -145,12 +147,12 @@ class Gulag: def get_quarmail(self,args): qm_db = None try: - qm_db = self.db.get_quarmail({"id": args['id']}) + qm_db = self.db.get_quarmail({"id": args['quarmail_id']}) except GulagDBException as e: raise GulagException("GulagDBException: " + e.message) from e if 'rfc822_message' not in args: return qm_db - # pull full RFC822 message from mailbox + # pull full RFC822 message from IMAP mailbox mailbox = None try: mailbox = self.db.get_mailbox(qm_db['mailbox_id']) @@ -159,19 +161,55 @@ class Gulag: imap_mb = None try: imap_mb = IMAPmailbox(mailbox) - qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid']) + qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid']).decode("utf-8") return qm_db except IMAPmailboxException as e: print(e.message) raise GulagException(e.message) from e + def get_quarmail_attachments(self,args): + try: + return self.db.get_quarmail_attachments(args['quarmail_id']) + except GulagDBException as e: + print(e.message) + raise GulagException(e.message) from e + + def get_quarmail_attachment(self,args): + qmat_db = None + try: + qmat_db = self.db.get_quarmail_attachment( + args['quarmail_id'],args['attachment_id'] + ) + except GulagDBException as e: + print(e.message) + raise GulagException(e.message) from e + if 'data' not in args: + return qmat_db + # pull attachment from IMAP mailbox + mailbox = None + try: + mailbox = self.db.get_mailbox(qmat_db['mailbox_id']) + except GulagDBException as e: + raise GulagException(e.message) from e + imap_mb = None + try: + imap_mb = IMAPmailbox(mailbox) + qmat_db['data'] = imap_mb.get_attachment( + qmat_db['imap_uid'],qmat_db['filename'] + ) + return qmat_db + except IMAPmailboxException as e: + print(e.message) + raise GulagException(e.message) from e + def get_attachment(self,args): at_db = None try: at_db = self.db.get_attachment({"id": args['id']}) - return at_db except GulagDBException as e: raise GulagException(e.message) from e + if 'data' not in args: + return at_db def rspamd_http2imap(self,mailbox_id): mailbox = None diff --git a/app/GulagDB.py b/app/GulagDB.py index af1ff3e..8df7d6a 100644 --- a/app/GulagDB.py +++ b/app/GulagDB.py @@ -113,7 +113,6 @@ class GulagDB: def get_quarmails(self): try: cursor = self.conn.cursor() -# cursor.execute("select * from QuarMails;") query = "select *,(select count(*) from QuarMail2Attachment" query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count" query += " from QuarMails;" @@ -159,10 +158,10 @@ class GulagDB: else: dict[name[0]] = value dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id']) - try: - dict['attachments'] = self.get_attachments_by_quarmail(args['id']) - except GulagDBException as e: - pass +# try: +# dict['attachments'] = self.get_attachments_by_quarmail(args['id']) +# except GulagDBException as e: +# pass return QuarMail(dict).__dict__ except mariadb.Error as e: raise GulagDBException(e) from e @@ -191,17 +190,46 @@ class GulagDB: try: cursor = self.conn.cursor() cursor.execute("insert into Attachments " + - "(filename, content_type) values (%s,%s)", - (attach['filename'], attach['content_type']) + "(filename, content_type, content_encoding) values (%s,%s,%s)", + (attach['filename'], attach['content_type'], attach['content_encoding']) ) return cursor.lastrowid except mariadb.Error as e: raise GulagDBException(e) from e + + def get_attachments(self): + try: + query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid" + query += " from QuarMail2Attachment" + query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id" + query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id" + query += " group by id;" + cursor = self.conn.cursor() + cursor.execute(query) + results = [] + data = cursor.fetchall() + if not data: + raise GulagDBException("No attachments found!") + desc = cursor.description + for tuple in data: + dict = {} + for (name, value) in zip(desc, tuple): + dict[name[0]] = value + dict['href'] = self.uri_prefixes['attachments'] + str(dict['id']) + results.append(Attachment(dict).__dict__) + return results + except mariadb.Error as e: + raise GulagDBException(e) from e def get_attachment(self, args): try: cursor = self.conn.cursor() - cursor.execute("select * from Attachments where id=" + str(args['id']) + ";") + query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid" + query += " from QuarMail2Attachment" + query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id" + query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id" + query += " where id=" + str(args['id']) + ";" + cursor.execute(query) data = cursor.fetchall() if not data: raise GulagDBException("Attachment("+ str(args['id']) +") does not exist!") @@ -215,9 +243,10 @@ class GulagDB: except mariadb.Error as e: raise GulagDBException(e) from e - def get_attachments_by_quarmail(self,quarmail_id): + def get_quarmail_attachments(self,quarmail_id): try: - query = "select Attachments.* from QuarMail2Attachment" + query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid" + query += " from QuarMail2Attachment" query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id" query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id" query += " where QuarMails.id = " + str(quarmail_id) + ";" @@ -232,11 +261,39 @@ class GulagDB: dict = {} for (name, value) in zip(desc, tuple): dict[name[0]] = value - dict['href'] = self.uri_prefixes['attachments'] + str(dict['id']) + dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id) + dict['href'] += "/attachments/" + str(dict['id']) results.append(Attachment(dict).__dict__) return results except mariadb.Error as e: raise GulagDBException(e) from e + + def get_quarmail_attachment(self,quarmail_id,attachment_id): + try: + query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid" + query += " from QuarMail2Attachment" + query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id" + query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id" + query += " where QuarMails.id = " + str(quarmail_id) + query += " and Attachments.id = " + str(attachment_id) + ";" + cursor = self.conn.cursor() + cursor.execute(query) + data = cursor.fetchall() + if not data: + raise GulagDBException("QuarMail("+ str(quarmail_id) +") " + + "has no attachment (" + str(attachment_id) + ")!" + ) + desc = cursor.description + tuple = data[0] + dict = {} + for (name, value) in zip(desc, tuple): + dict[name[0]] = value + dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id) + dict['href'] += "/attachments/" + str(dict['id']) + return Attachment(dict).__dict__ + except mariadb.Error as e: + raise GulagDBException(e) from e + def quarmail2attachment(self,quarmail_id,attachment_id): try: diff --git a/app/GulagMailbox.py b/app/GulagMailbox.py index 45dfbd1..5184b31 100644 --- a/app/GulagMailbox.py +++ b/app/GulagMailbox.py @@ -9,6 +9,7 @@ class IMAPmailboxException(Exception): self.message = str(message) class IMAPmailbox: + email_address = None imap_server = None imap_user = None imap_pass = None @@ -16,6 +17,7 @@ class IMAPmailbox: mailbox = None def __init__(self, mb_ref): + self.email_address = mb_ref['email_address'] self.imap_server = mb_ref['imap_server'] self.imap_user = mb_ref['imap_user'] self.imap_pass = mb_ref['imap_pass'] @@ -54,7 +56,7 @@ class IMAPmailbox: continue results.append({ 'imap_uid': uid, - 'msg': email.message_from_bytes(data[0][1]) + 'msg': data[0][1] }) return results @@ -62,31 +64,29 @@ class IMAPmailbox: rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)') if rv != 'OK': raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid)) - return data[0][1].decode("utf-8") + return data[0][1] - def get_attachments(self,imap_uid): - results = [] - rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)') - if rv != 'OK': - raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid)) - msg = email.message_from_bytes(data[0][1]) + def get_attachment(self,imap_uid,filename): + msg = email.message_from_bytes(self.get_message(imap_uid)) for part in msg.walk(): if part.get_filename(): # let´s define parts with filename as attachments - filename = email.header.decode_header(part.get_filename()) - if filename[0][1]: + part_fn = email.header.decode_header(part.get_filename()) + if part_fn[0][1]: # Encoded -> decode - filename = filename[0][0].decode(filename[0][1]) + part_fn = part_fn[0][0].decode(part_fn[0][1]) else: # not encoded - filename = filename[0][0] - results.append({ - 'filename': filename, - 'content-type': part.get_content_type(), - 'content': part.get_payload(decode=True) - }) + part_fn = part_fn[0][0] + print("C-T-E: " + str(part['Content-Transfer-Encoding'])) + if(part_fn == filename): + return part.get_payload(decode=False) # End if part.get_filename() - return results + # End msg.walk() loop + raise IMAPmailboxException( + "Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@" + + str(self.email_address) + " not found!" + ) def append_message(self,message): rv, data = self.mailbox.append( diff --git a/app/Resources.py b/app/Resources.py index 3277aaf..6af847c 100644 --- a/app/Resources.py +++ b/app/Resources.py @@ -50,8 +50,8 @@ class ResQuarMails(GulagResource): abort(400, message=e.message) class ResQuarMail(GulagResource): - def get(self,id): - args = {"id": id} + def get(self,quarmail_id): + args = {"quarmail_id": quarmail_id} try: if(request.args.get('rfc822_message')): args['rfc822_message'] = True @@ -59,13 +59,36 @@ class ResQuarMail(GulagResource): except GulagException as e: abort(400, message=e.message) +class ResQuarMailAttachments(GulagResource): + def get(self,quarmail_id): + args = {"quarmail_id": quarmail_id} + if(request.args.get('data')): + args['data'] = True + try: + return self.gulag.get_quarmail_attachments(args) + except GulagException as e: + abort(400, message=e.message) + +class ResQuarMailAttachment(GulagResource): + def get(self,quarmail_id,attachment_id): + args = { + "quarmail_id": quarmail_id, + "attachment_id": attachment_id + } + if(request.args.get('data')): + args['data'] = True + try: + return self.gulag.get_quarmail_attachment(args) + except GulagException as e: + abort(400, message=e.message) + class ResAttachments(GulagResource): def get(self): return {"resource": "Attachments"} class ResAttachment(GulagResource): - def get(self,id): - args = {"id": id} + def get(self,attachment_id): + args = {"id": attachment_id} try: return self.gulag.get_attachment(args) except GulagException as e: diff --git a/app/gulag_helpers.py b/app/gulag_helpers.py index 9b8ce2b..aeb2eff 100755 --- a/app/gulag_helpers.py +++ b/app/gulag_helpers.py @@ -20,7 +20,9 @@ if(importer_pid == 0): try: gulag.import_quarmails() except GulagException as e: - print("Importer-Exception: " + e.message) + print("Importer-Exception: " + e.message, file=sys.stderr) + except: + print("Importer-Exception: " + str(sys.exc_info()),file=sys.stderr) time.sleep(gulag.config['importer']['interval']) cleaner_pid = os.fork() @@ -36,6 +38,8 @@ if(cleaner_pid == 0): gulag.cleanup_quarmails() except GulagException as e: print("Cleaner-Exception: " + e.message) + except: + print("Cleaner-Exception: " + str(sys.exc_info()),file=sys.stderr) time.sleep(gulag.config['cleaner']['interval']) # Parent @@ -46,7 +50,7 @@ try: while True: time.sleep(10) except: - print("MAIN-EXCEPTION: " + str(sys.exc_info())) + print("Helpers MAIN-EXCEPTION: " + str(sys.exc_info())) # Destroy childs for child_pid in child_pids: print("Killing child pid: %s", child_pid) diff --git a/app/gulag_server.py b/app/gulag_server.py index 64e5258..0c407be 100755 --- a/app/gulag_server.py +++ b/app/gulag_server.py @@ -5,8 +5,9 @@ from flask import Flask from flask_restful import Api from Gulag import Gulag,GulagException from Resources import (ResRoot,ResMailboxes, - ResQuarMails,ResQuarMail,ResAttachments, - ResAttachment,ResRSPAMDImporter + ResQuarMails,ResQuarMail,ResQuarMailAttachments, + ResQuarMailAttachment,ResAttachments,ResAttachment, + ResRSPAMDImporter ) parser = argparse.ArgumentParser() parser.add_argument('--config', required=True, help="Path to config file") @@ -32,7 +33,15 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResQuarMail, - '/api/v1/quarmails/', + '/api/v1/quarmails/', + resource_class_kwargs={'gulag_object': gulag} + ) + api.add_resource(ResQuarMailAttachments, + '/api/v1/quarmails//attachments/', + resource_class_kwargs={'gulag_object': gulag} + ) + api.add_resource(ResQuarMailAttachment, + '/api/v1/quarmails//attachments/', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResAttachments, @@ -40,7 +49,7 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResAttachment, - '/api/v1/attachments/', + '/api/v1/attachments/', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResRSPAMDImporter, diff --git a/db/gulag.sql b/db/gulag.sql index f2458a2..e776293 100644 --- a/db/gulag.sql +++ b/db/gulag.sql @@ -49,6 +49,7 @@ create table Attachments ( id int unsigned auto_increment primary key, filename varchar(256) not null, content_type varchar(256) not null, + content_encoding varchar(64), comment varchar(256) )ENGINE = InnoDB;