From a1ceb75805c4280d0f32edb9105171ba13df92fa Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Sat, 8 Dec 2018 23:29:32 +0100 Subject: [PATCH] whoami@Utils, pagination, cleaner resource routing --- app/Gulag.py | 111 ++++++++++++++++++++----------------- app/GulagDB.py | 131 ++++++++++++++++++++++++++++++++++---------- app/GulagMailbox.py | 17 ++++-- app/GulagUtils.py | 23 ++++++++ app/Resources.py | 8 ++- app/gulag_server.py | 10 ++-- 6 files changed, 209 insertions(+), 91 deletions(-) create mode 100644 app/GulagUtils.py diff --git a/app/Gulag.py b/app/Gulag.py index 40bc966..8167db1 100644 --- a/app/Gulag.py +++ b/app/Gulag.py @@ -1,9 +1,8 @@ import json,sys import email,email.header,email.message -from flask import request -from smtplib import SMTP from GulagDB import GulagDB,GulagDBException from GulagMailbox import IMAPmailbox,IMAPmailboxException +from GulagUtils import whoami class GulagException(Exception): message = None @@ -14,6 +13,7 @@ class Gulag: version = None config = None db = None + fields = {} def __init__(self, path_to_config_file): self.version = "VERSION-TODO!" @@ -22,12 +22,29 @@ class Gulag: self.config = json.load(f) f.close() except: - raise GulagException("CONFIG-FILE-Exception: " + str(sys.exc_info())) - + raise GulagException(whoami(self) + str(sys.exc_info())) try: self.db = GulagDB(self.config['db'],self.config['uri_prefixes']) + self.fields['Mailboxes'] = self.db.get_fields('Mailboxes') + self.fields['QuarMails'] = self.db.get_fields('QuarMails') + self.fields['Attachments'] = self.db.get_fields('Attachments') except GulagDBException as e: - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e + + def check_fields(self,fields_target,args): + if fields_target not in self.fields: + raise GulagException( + whoami(self) + fields_target + " not found in Gulag.fields!" + ) + for arg in args: + if(arg == 'query_offset' or arg == 'query_limit' + or arg == 'sort_index' or arg == 'sort_order'): + continue + if arg not in self.fields[fields_target]: + raise GulagException( + whoami(self) + arg + " is not a valid field of " + + fields_target + "!" + ) # Iterate through all mailboxes, extract metadata # from all unseen mails and pump them into database @@ -122,7 +139,6 @@ class Gulag: self.db.quarmail2attachment(str(quarmail_id), str(attachment_id)) # Ende for(unseen) imap_mb.close() - # Ende for get_mailboxes def cleanup_quarmails(self): @@ -136,20 +152,23 @@ class Gulag: try: return self.db.get_mailboxes() except GulagDBException as e: - raise GulagException("GulagDBException: " + e.message) from e + raise GulagException(whoami(self) + e.message) from e - def get_quarmails(self): + def get_quarmails(self,args): try: - return self.db.get_quarmails() - except GulagDBException as e: - raise GulagException("GulagDBException: " + e.message) from e + self.check_fields('QuarMails',args) + return self.db.get_quarmails(args) + except(GulagException,GulagDBException) as e: + raise GulagException( + whoami(self) + e.message + ) from e def get_quarmail(self,args): qm_db = None try: qm_db = self.db.get_quarmail({"id": args['quarmail_id']}) except GulagDBException as e: - raise GulagException("GulagDBException: " + e.message) from e + raise GulagException(whoami(self) + e.message) from e if 'rfc822_message' not in args: return qm_db # pull full RFC822 message from IMAP mailbox @@ -157,7 +176,7 @@ class Gulag: try: mailbox = self.db.get_mailbox(qm_db['mailbox_id']) except GulagDBException as e: - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e imap_mb = None try: imap_mb = IMAPmailbox(mailbox) @@ -165,14 +184,14 @@ class Gulag: return qm_db except IMAPmailboxException as e: print(e.message) - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e def get_quarmail_attachments(self,args): try: return self.db.get_quarmail_attachments(args['quarmail_id']) except GulagDBException as e: print(e.message) - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e def get_quarmail_attachment(self,args): qmat_db = None @@ -182,7 +201,7 @@ class Gulag: ) except GulagDBException as e: print(e.message) - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e if 'data' not in args: return qmat_db # pull attachment from IMAP mailbox @@ -190,7 +209,7 @@ class Gulag: try: mailbox = self.db.get_mailbox(qmat_db['mailbox_id']) except GulagDBException as e: - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e imap_mb = None try: imap_mb = IMAPmailbox(mailbox) @@ -200,65 +219,57 @@ class Gulag: return qmat_db except IMAPmailboxException as e: print(e.message) - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e def get_attachment(self,args): at_db = None try: at_db = self.db.get_attachment({"id": args['id']}) except GulagDBException as e: - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e if 'data' not in args: return at_db - - def rspamd_http2imap(self,mailbox_id): + + def get_uris(self): + # https://stackoverflow.com/questions/1792366/extract-urls-out-of-email-in-python + return True + + def rspamd_http2imap(self,args): mailbox = None try: - mailbox = self.db.get_mailbox(mailbox_id) + mailbox = self.db.get_mailbox(args['mailbox_id']) except GulagDBException as e: - raise GulagException(e.message) from e + raise GulagException(whoami(self) + e.message) from e # check if the request comes really from rspamd´s metadata_exporter # default metadata_header prefix 'X-Rspamd' will be expected - if(request.headers.get('X-Rspamd-From') == None): - raise GulagException("Missing Rspamd-specific headers (e.g. X-Rspamd-From)!") - # Prepend gulag-specific headers to rejected mail + if('X-Rspamd-From' not in args['req_headers']): + raise GulagException(whoami(self) + + "Missing Rspamd-specific headers (e.g. X-Rspamd-From)!" + ) + # Prepend Gulag-specific headers to rejected mail # before pushing into quarantine mailbox msg = None try: rcpts_hdr = "" - for rcpt in json.loads(str(request.headers.get('X-Rspamd-Rcpt'))): + for rcpt in json.loads(str(args['req_headers']['X-Rspamd-Rcpt'])): if(len(rcpts_hdr) > 0): - rcpts_hdr = rcpts_hdr + "," + rcpt + rcpts_hdr += "," + rcpt else: rcpts_hdr = rcpt - msg = "Return-Path: <" + request.headers.get('X-Rspamd-From') + ">\r\n" - msg += "Received: from rspamd_http2imap relay by gulag-mailbox " + mailbox_id + "\r\n" + msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n" + msg += "Received: from rspamd_http2imap relay by gulag-mailbox " + msg += args['mailbox_id'] + "\r\n" msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n" - msg += "X-Spam-Status: " + request.headers.get('X-Rspamd-Symbols') + "\r\n" - msg += "X-Spam-QID: " + request.headers.get('X-Rspamd-Qid') + "\r\n" + msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n" + msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n" # append original mail - msg += request.get_data(as_text=True) + msg += args['rfc822_message'] except: - raise GulagException(str(sys.exc_info())) + raise GulagException(whoami(self) + str(sys.exc_info())) imap_mb = None try: imap_mb = IMAPmailbox(mailbox) imap_mb.append_message(msg) except IMAPmailboxException as e: - raise GulagException(e.message) from e - -# def send_mail(self,args): -# try: -# # FIXME: SMTP tranaport security and authentication! -# # with SMTP(host=mailbox['smtp_server'],port=mailbox['smtp_port']) as smtp: -# # try: -# # smtp.sendmail( -# # request.headers.get('X-Rspamd-From'), -# # mailbox_id, -# # msg -# # ) -# # except (SMTPRecipientsRefused,SMTPHeloError,SMTPSenderRefused,SMTPDataError) as e: -# # raise GulagException(str(e)) from e -# except TimeoutError as e: -# raise GulagException(str(e)) from e + raise GulagException(whoami(self) + e.message) from e diff --git a/app/GulagDB.py b/app/GulagDB.py index 8df7d6a..68723cb 100644 --- a/app/GulagDB.py +++ b/app/GulagDB.py @@ -1,5 +1,10 @@ import mysql.connector as mariadb -from Entities import Mailbox,MailboxException,QuarMail,QuarMailException,Attachment,AttachmentException +from Entities import( + Mailbox,MailboxException,QuarMail, + QuarMailException,Attachment, + AttachmentException +) +from GulagUtils import whoami class GulagDBException(Exception): message = None @@ -30,10 +35,69 @@ class GulagDB: ) self.uri_prefixes = uri_prefixes except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def close(self): self.conn.close() + + def get_fields(self,table_name): + try: + cursor = self.conn.cursor() + query = "describe " + str(table_name) + ";" + cursor.execute(query) + data = cursor.fetchall() + if not data: + raise GulagDBException(whoami(self) + + "describe " + table_name + " failed!" + ) + desc = cursor.description + cursor.close() + results = {} + for tuple in data: + for (name, value) in zip(desc, tuple): + if(name[0] == "Field"): + results[value] = True + return results + except mariadb.Error as e: + raise GulagDBException(whoami(self) + e) from e + + def get_limit_clause(self,args): + if('query_offset' in args and 'query_limit' in args): + try: + int(args['query_offset']) + except ValueError: + raise GulagDBException(whoami(self) + "query_offset must be numeric!") + try: + int(args['query_limit']) + except ValueError: + raise GulagDBException(whoami(self) + "query_limit must be numeric!") + return "limit "+args['query_offset']+","+args['query_limit'] + elif('query_offset' in args and 'query_limit' not in args): + raise GulagDBException(whois(self) + + "query_offset without query_limit is useless!" + ) + elif('query_limit' in args and 'query_offset' not in args): + try: + int(args['query_limit']) + except ValueError: + raise GulagDBException(whoami(self) + "query_limit must be numeric!") + return "limit " + args['query_limit'] + else: + return "" + + def get_where_clause(self,args): + where_clause = "" + cnt = 0 + for arg in args: + if(arg == 'query_offset' or arg == 'query_limit' + or arg == 'sort_index' or arg == 'sort_order'): + continue + if(cnt == 0): + where_clause += "where " + arg + "='" + args[arg] + "' " + else: + where_clause += "and " + arg + "='" + args[arg] + "' " + cnt += 1 + return where_clause def get_mailboxes(self): try: @@ -42,7 +106,7 @@ class GulagDB: results = [] data = cursor.fetchall() if not data: - raise GulagDBException("No mailboxes found in DB!") + raise GulagDBException(whoami(self) + "No mailboxes found in DB!") desc = cursor.description for tuple in data: dict = {} @@ -56,7 +120,7 @@ class GulagDB: continue return results except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_mailbox(self,mailbox_id): try: @@ -66,7 +130,9 @@ class GulagDB: ) data = cursor.fetchall() if not data: - raise GulagDBException("Mailbox '" + mailbox_id + "' does not exist!") + raise GulagDBException(whoami(self) + + "Mailbox '" + mailbox_id + "' does not exist!" + ) desc = cursor.description tuple = data[0] dict = {} @@ -76,10 +142,10 @@ class GulagDB: try: return Mailbox(dict).__dict__ except MailboxException as e: - raise GulagDBException(e.message) from e + raise GulagDBException(whoami(self) + e.message) from e except mariadb.Error as e: - raise GulagDBException(e) from e - + raise GulagDBException(whoami(self) + e) from e + def add_quarmail(self, quarmail): try: cursor = self.conn.cursor() @@ -99,7 +165,7 @@ class GulagDB: cursor.close() return id except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def del_quarmail(self, id): try: @@ -108,19 +174,20 @@ class GulagDB: cursor.close() return True except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e - def get_quarmails(self): - try: + def get_quarmails(self,args): + try: cursor = self.conn.cursor() query = "select *,(select count(*) from QuarMail2Attachment" query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count" - query += " from QuarMails;" + query += " from QuarMails " + self.get_where_clause(args) + query += " " + self.get_limit_clause(args) + " ;" cursor.execute(query) results = [] data = cursor.fetchall() if not data: - raise GulagDBException("No Quarmails found in DB!") + raise GulagDBException(whoami(self) + "No QuarMails found in DB!") desc = cursor.description cursor.close() for tuple in data: @@ -133,8 +200,10 @@ class GulagDB: dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id']) results.append(QuarMail(dict).__dict__) return results + except GulagDBException as e: + raise GulagDBException(whoami(self) + e.message) from e except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_quarmail(self,args): try: @@ -147,7 +216,9 @@ class GulagDB: cursor.execute(query) data = cursor.fetchall() if not data: - raise GulagDBException("Quarmail with id '"+ args['id'] + "' does not exist!") + raise GulagDBException(whoami(self) + + "Quarmail with id '"+ args['id'] + "' does not exist!" + ) desc = cursor.description cursor.close() tuple = data[0] @@ -164,7 +235,7 @@ class GulagDB: # pass return QuarMail(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_deprecated_mails(self,retention_period): try: @@ -184,7 +255,7 @@ class GulagDB: results.append(dict) return results except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def add_attachment(self, attach): try: @@ -195,7 +266,7 @@ class GulagDB: ) return cursor.lastrowid except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_attachments(self): try: @@ -209,7 +280,7 @@ class GulagDB: results = [] data = cursor.fetchall() if not data: - raise GulagDBException("No attachments found!") + raise GulagDBException(whoami(self) + "No attachments found!") desc = cursor.description for tuple in data: dict = {} @@ -219,7 +290,7 @@ class GulagDB: results.append(Attachment(dict).__dict__) return results except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_attachment(self, args): try: @@ -232,7 +303,9 @@ class GulagDB: cursor.execute(query) data = cursor.fetchall() if not data: - raise GulagDBException("Attachment("+ str(args['id']) +") does not exist!") + raise GulagDBException(whoami(self) + + "Attachment("+ str(args['id']) +") does not exist!" + ) desc = cursor.description tuple = data[0] dict = {} @@ -241,7 +314,7 @@ class GulagDB: dict['href'] = self.uri_prefixes['attachments'] + str(dict['id']) return Attachment(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_quarmail_attachments(self,quarmail_id): try: @@ -255,7 +328,9 @@ class GulagDB: results = [] data = cursor.fetchall() if not data: - raise GulagDBException("QuarMail("+ str(quarmail_id) +") has no attachments!") + raise GulagDBException(whoami(self) + + "QuarMail("+ str(quarmail_id) +") has no attachments!" + ) desc = cursor.description for tuple in data: dict = {} @@ -266,7 +341,7 @@ class GulagDB: results.append(Attachment(dict).__dict__) return results except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def get_quarmail_attachment(self,quarmail_id,attachment_id): try: @@ -280,7 +355,7 @@ class GulagDB: cursor.execute(query) data = cursor.fetchall() if not data: - raise GulagDBException("QuarMail("+ str(quarmail_id) +") " + raise GulagDBException(whoami(self) + "QuarMail("+ str(quarmail_id) +") " + "has no attachment (" + str(attachment_id) + ")!" ) desc = cursor.description @@ -292,7 +367,7 @@ class GulagDB: dict['href'] += "/attachments/" + str(dict['id']) return Attachment(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e def quarmail2attachment(self,quarmail_id,attachment_id): @@ -303,5 +378,5 @@ class GulagDB: (quarmail_id, attachment_id) ) except mariadb.Error as e: - raise GulagDBException(e) from e + raise GulagDBException(whoami(self) + e) from e diff --git a/app/GulagMailbox.py b/app/GulagMailbox.py index 5184b31..89edf25 100644 --- a/app/GulagMailbox.py +++ b/app/GulagMailbox.py @@ -2,6 +2,7 @@ import imaplib import email import email.header import time +from GulagUtils import whoami class IMAPmailboxException(Exception): message = None @@ -26,17 +27,17 @@ class IMAPmailbox: self.mailbox = imaplib.IMAP4(self.imap_server) rv, data = self.mailbox.login(self.imap_user, self.imap_pass) except imaplib.IMAP4.error as e: - raise IMAPmailboxException( + raise IMAPmailboxException(whoami(self) + "LOGIN FAILED FOR " + self.imap_user + '@' + self.imap_server ) from e except ConnectionRefusedError as e: - raise IMAPmailboxException( + raise IMAPmailboxException(whoami(self) + self.imap_user + ": IMAP server " + self.imap_server + " refused connection" ) from e rv, data = self.mailbox.select(self.imap_mailbox) if rv != 'OK': - raise IMAPmailboxException( + raise IMAPmailboxException(whoami(self) + "ERROR: Unable to select mailbox: " + self.imap_mailbox ) @@ -63,7 +64,9 @@ class IMAPmailbox: def get_message(self,imap_uid): rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)') if rv != 'OK': - raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid)) + raise IMAPmailboxException(whoami(self) + + "ERROR getting message: %s", str(imap_uid) + ) return data[0][1] def get_attachment(self,imap_uid,filename): @@ -83,7 +86,7 @@ class IMAPmailbox: return part.get_payload(decode=False) # End if part.get_filename() # End msg.walk() loop - raise IMAPmailboxException( + raise IMAPmailboxException(whoami(self) + "Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@" + str(self.email_address) + " not found!" ) @@ -96,7 +99,9 @@ class IMAPmailbox: str(message).encode('utf-8') ) if rv != 'OK': - raise IMAPmailboxException("ERROR appending message: " + rv) + raise IMAPmailboxException(whoami(self)+ + "ERROR appending message: " + rv + ) def expunge_message(self,imap_uid): return True diff --git a/app/GulagUtils.py b/app/GulagUtils.py new file mode 100644 index 0000000..63b81de --- /dev/null +++ b/app/GulagUtils.py @@ -0,0 +1,23 @@ +import sys +from smtplib import SMTP + + +def whoami(obj): + return type(obj).__name__ + "::" + sys._getframe(1).f_code.co_name + "(): " + +def send_mail(args): + try: + # FIXME: SMTP tranaport security and authentication! + # with SMTP(host=mailbox['smtp_server'],port=mailbox['smtp_port']) as smtp: + # try: + # smtp.sendmail( + # request.headers.get('X-Rspamd-From'), + # mailbox_id, + # msg + # ) + # except (SMTPRecipientsRefused,SMTPHeloError,SMTPSenderRefused,SMTPDataError) as e: + # raise GulagException(str(e)) from e + print("TODO") + except TimeoutError as e: + raise Exception('xyz') from e + diff --git a/app/Resources.py b/app/Resources.py index 6af847c..4e5efbd 100644 --- a/app/Resources.py +++ b/app/Resources.py @@ -45,7 +45,7 @@ class ResMailbox(GulagResource): class ResQuarMails(GulagResource): def get(self): try: - return self.gulag.get_quarmails() + return self.gulag.get_quarmails(request.args.to_dict()) except GulagException as e: abort(400, message=e.message) @@ -97,7 +97,11 @@ class ResAttachment(GulagResource): class ResRSPAMDImporter(GulagResource): def post(self,mailbox_id): try: - self.gulag.rspamd_http2imap(mailbox_id) + self.gulag.rspamd_http2imap({ + "mailbox_id": mailbox_id, + "req_headers": request.headers, + "rfc822_message": request.get_data(as_text=True) + }) # TODO: Response mit Location-Header? return {"resource: ": "HTTP2IMAP for RSPAMD"} except GulagException as e: diff --git a/app/gulag_server.py b/app/gulag_server.py index 0c407be..014e7fa 100755 --- a/app/gulag_server.py +++ b/app/gulag_server.py @@ -25,11 +25,11 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResMailboxes, - '/api/v1/mailboxes/', + '/api/v1/mailboxes', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResQuarMails, - '/api/v1/quarmails/', + '/api/v1/quarmails', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResQuarMail, @@ -37,7 +37,7 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResQuarMailAttachments, - '/api/v1/quarmails//attachments/', + '/api/v1/quarmails//attachments', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResQuarMailAttachment, @@ -45,7 +45,7 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResAttachments, - '/api/v1/attachments/', + '/api/v1/attachments', resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResAttachment, @@ -53,7 +53,7 @@ try: resource_class_kwargs={'gulag_object': gulag} ) api.add_resource(ResRSPAMDImporter, - '/api/v1/mailboxes//rspamdimporter/', + '/api/v1/mailboxes//rspamdimporter', resource_class_kwargs={'gulag_object': gulag} ) if __name__ == '__main__':