From ebb762ba84f88efea888c396861fcdd9a65f4466 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 1 Jan 2019 23:49:14 +0100 Subject: [PATCH] Mailradar support --- app/Entities.py | 43 +++++---- app/Gulag.py | 217 +++++++++++++++++++++++++++++--------------- app/GulagDB.py | 62 +++++++------ app/Resources.py | 33 ++++--- app/gulag_server.py | 11 ++- db/gulag.sql | 51 +++++++---- 6 files changed, 268 insertions(+), 149 deletions(-) diff --git a/app/Entities.py b/app/Entities.py index 8de6e87..66d3fd4 100644 --- a/app/Entities.py +++ b/app/Entities.py @@ -16,11 +16,6 @@ class Mailbox: imap_mailbox = None imap_mailbox_fp = None imap_separator = None - smtp_server = None - smtp_port = None - smtp_security = None - smtp_user = None - smtp_pass = None comment = None href = None @@ -38,7 +33,7 @@ class Mailbox: if re.match("^(plain|starttls|tls)$", mb_ref['imap_security']) is not None: self.imap_security = mb_ref['imap_security'] else: - raise MailboxException('imap_security: {} is invalid! '+ + raise MailboxException('imap_security: {} is invalid! '+ 'Valid values: plain,starttls,tls'.format(mb_ref['imap_security']) ) else: @@ -60,16 +55,6 @@ class Mailbox: if 'imap_separator' not in mb_ref: raise MailboxException("'imap_separator' is mandatory!") self.imap_seperator = mb_ref['imap_separator'] - if 'smtp_server' in mb_ref: - self.smtp_server = mb_ref['smtp_server'] - if 'smtp_port' in mb_ref: - self.smtp_port = mb_ref['smtp_port'] - if 'smtp_security' in mb_ref: - self.smtp_security = mb_ref['smtp_security'] - if 'smtp_user' in mb_ref: - self.smtp_user = mb_ref['smtp_user'] - if 'smtp_pass' in mb_ref: - self.smtp_pass = mb_ref['smtp_pass'] if 'comment' in mb_ref: self.comment = mb_ref['comment'] if 'href' in mb_ref: @@ -98,6 +83,8 @@ class QuarMail: href = None attach_count = None uri_count = None + source_id = None + ssdeep = None def __init__(self,qm_ref): if 'id' not in qm_ref: @@ -142,6 +129,12 @@ class QuarMail: self.attach_count = qm_ref['attach_count'] if 'uri_count' in qm_ref: self.uri_count = qm_ref['uri_count'] + if 'source_id' not in qm_ref: + raise QuarMailException("'source_id' is mandatory!") + self.source_id = qm_ref['source_id'] + if 'ssdeep' not in qm_ref: + raise QuarMailException("'ssdeep' is mandatory!") + self.ssdeep = qm_ref['ssdeep'] class AttachmentException(Exception): message = None @@ -157,6 +150,10 @@ class Attachment: comment = None mailbox_id = None imap_uid = None + size = None + sha256 = None + ssdeep = None + sandbox_results = None href = None def __init__(self,at_ref): @@ -182,6 +179,16 @@ class Attachment: if 'imap_uid' not in at_ref: raise AttachmentException("'imap_uid' is mandatory!") self.imap_uid = at_ref['imap_uid'] + if 'size' not in at_ref: + raise AttachmentException("'size' is mandatory!") + if 'sha256' not in at_ref: + raise AttachmentException("'sha256' is mandatory!") + self.sha256 = at_ref['sha256'] + if 'ssdeep' not in at_ref: + raise AttachmentException("'ssdeep' is mandatory!") + self.ssdeep = at_ref['ssdeep'] + if 'sandbox_results' in at_ref: + self.sandbox_results = at_ref['sandbox_results'] if 'href' in at_ref: self.href = at_ref['href'] @@ -194,6 +201,7 @@ class URI: id = None uri = None fqdn = None + sandbox_results = None href = None def __init__(self,uri_ref): @@ -206,6 +214,7 @@ class URI: if 'fqdn' not in uri_ref: raise URIException("'fqdn' is mandatory!") self.fqdn = uri_ref['fqdn'] + if 'sandbox_results' in uri_ref: + self.sandbox_results = uri_ref['sandbox_results'] if 'href' in uri_ref: self.href = uri_ref['href'] - diff --git a/app/Gulag.py b/app/Gulag.py index 9c9541a..c77a1f4 100644 --- a/app/Gulag.py +++ b/app/Gulag.py @@ -97,7 +97,9 @@ class Gulag: uris = {} uid = unseen['imap_uid'] msg = email.message_from_bytes(unseen['msg']) - msg_size = len(msg.as_string()) + source_id = 'amavis' + if 'X-Gulag-Source' in msg: + source_id = email.header.decode_header(msg['X-Gulag-Source'])[0][0] r5321_from = email.header.decode_header(msg['Return-Path'])[0][0] if(r5321_from is not '<>'): r5321_from = r5321_from.replace("<","") @@ -144,14 +146,22 @@ class Gulag: # Die E-Mail im IMAP-Backend existiert jedoch nur ein Mal und wird # über die mailbox_id sowie die imap_uid mehrfach referenziert. for r5321_rcpt in r5321_rcpts.split(","): - quarmail_id = self.db.add_quarmail({ - 'mx_queue_id': mx_queue_id, 'env_from': r5321_from, - 'env_rcpt': r5321_rcpt, 'hdr_cf': x_spam_status, - 'hdr_from': r5322_from, 'hdr_subject': subject,'hdr_msgid': msg_id, - 'hdr_date': date, 'cf_meta': 'cf_meta', - 'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, - 'msg_size': msg_size - }) + try: + quarmail_id = self.db.add_quarmail({ + 'mx_queue_id': mx_queue_id, 'env_from': r5321_from, + 'env_rcpt': r5321_rcpt, 'hdr_cf': x_spam_status, + 'hdr_from': r5322_from, 'hdr_subject': subject, + 'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta', + 'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, + 'source_id': source_id, 'msg_size': len(msg.as_string()), + 'ssdeep': ssdeep.hash(msg.as_string()) + }) + except GulagDBBadInputException as e: + logging.warn(whoami(self) + e.message) + raise GulagBadInputException(whoami(self) + e.message) from e + except GulagDBException as e: + logging.warn(whoami(self) + e.message) + raise GulagException(whoami(self) + e.message) from e logging.info(whoami(self) + "QuarMail (%s) imported" % (quarmail_id)) quarmail_ids.append(quarmail_id) # Ende for rcpts @@ -167,24 +177,15 @@ class Gulag: else: # filename isn´t encoded filename = filename[0][0] - logging.info(whoami(self) + - "SSDEEP: " + ssdeep.hash(part.get_payload(decode=True)) - ) - logging.info(whoami(self) + - "SHA256 " + hashlib.sha256( - part.get_payload(decode=True) - ).hexdigest() - ) - attach_magic = None - try: - attach_magic = magic.from_buffer(part.get_payload(decode=True)) - except: - logging.info(whoami(self) + ": " + str(sys.exc_info())) + attach_decoded = part.get_payload(decode=True) attach_id = self.db.add_attachment({ 'filename': filename, 'content_type': part.get_content_type(), 'content_encoding': part['Content-Transfer-Encoding'], - 'magic': attach_magic + 'magic': magic.from_buffer(attach_decoded), + 'sha256': hashlib.sha256(attach_decoded).hexdigest(), + 'ssdeep': ssdeep.hash(attach_decoded), + 'size': len(attach_decoded) }) attachments.append(attach_id) # End if part.get_filename() @@ -464,7 +465,7 @@ class Gulag: except GulagDBException as e: raise GulagException(whoami(self) + e.message) from e - def rspamd_http2imap(self,args): + def rspamd2mailbox(self,args): mailbox = None try: mailbox = self.db.get_mailbox(args['mailbox_id']) @@ -479,60 +480,128 @@ class Gulag: + "Missing Rspamd-specific request header X-Rspamd-From!" ) logging.error(err) - raise GulagException(err) + raise GulagBadInputException(err) # Prepend Gulag-specific headers to rejected mail # before pushing into quarantine mailbox msg = None + if('X-Rspamd-From' not in args['req_headers']): + err = str(whoami(self) + + "Missing Rspamd-specific request header X-Rspamd-From!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('X-Rspamd-Rcpt' not in args['req_headers']): + err = str(whoami(self) + + "Missing Rspamd-specific request header X-Rspamd-Rcpt!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('X-Rspamd-Symbols' not in args['req_headers']): + err = str(whoami(self) + + "Missing Rspamd-specific request header X-Rspamd-Symbols!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('X-Rspamd-Qid' not in args['req_headers']): + err = str(whoami(self) + + "Missing Rspamd-specific request header X-Rspamd-Qid!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('rfc822_message' not in args): + err = str(whoami(self) + "Missing rfc822_message!") + logging.error(err) + raise GulagBadInputException(err) + # all mandatory request headers and body are present + rcpts_hdr = "" + rcpts = None try: - if('X-Rspamd-From' not in args['req_headers']): - err = str(whoami(self) - + "Missing Rspamd-specific request header X-Rspamd-From!" - ) - logging.error(err) - raise GulagException(err) - if('X-Rspamd-Rcpt' not in args['req_headers']): - err = str(whoami(self) - + "Missing Rspamd-specific request header X-Rspamd-Rcpt!" - ) - logging.error(err) - raise GulagException(err) - if('X-Rspamd-Symbols' not in args['req_headers']): - err = str(whoami(self) - + "Missing Rspamd-specific request header X-Rspamd-Symbols!" - ) - logging.error(err) - raise GulagException(err) - if('X-Rspamd-Qid' not in args['req_headers']): - err = str(whoami(self) - + "Missing Rspamd-specific request header X-Rspamd-Qid!" - ) - logging.error(err) - raise GulagException(err) - if('rfc822_message' not in args): - err = str(whoami(self) - + "Missing rfc822_message!" - ) - logging.error(err) - raise GulagException(err) - # all mandatory request headers and body are present - rcpts_hdr = "" - for rcpt in json.loads(str(args['req_headers']['X-Rspamd-Rcpt'])): - if(len(rcpts_hdr) > 0): - rcpts_hdr += "," + rcpt - else: - rcpts_hdr = rcpt - msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n" - msg += "Received: from rspamd_http2imap relay by gulag-mailbox " - msg += args['mailbox_id'] + "\r\n" - msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n" - msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n" - msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n" - # append original mail - msg += args['rfc822_message'] - except GulagException as e: - raise GulagException(e.message) from e - except: - raise GulagException(whoami(self) + str(sys.exc_info())) + rcpts = json.loads(str(args['req_headers']['X-Rspamd-Rcpt'])) + except json.JSONDecodeError as e: + raise GulagBadInputException(e.msg) from e + for rcpt in rcpts: + if(len(rcpts_hdr) > 0): + rcpts_hdr += "," + rcpt + else: + rcpts_hdr = rcpt + msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n" + msg += "Received: from Rspamd http2imap relay by gulag-mailbox IMAP: " + msg += args['mailbox_id'] + "\r\n" + msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n" + msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n" + msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n" + msg += "X-Gulag-Source: rspamd\r\n" + # append original mail + msg += args['rfc822_message'] + imap_mb = None + try: + imap_mb = IMAPmailbox(mailbox) + imap_uid = imap_mb.add_message(msg, unseen=True) + logging.info(whoami(self) + "IMAP_UID: " + str(imap_uid)) + imap_mb.close() + except IMAPmailboxException as e: + raise GulagException(whoami(self) + e.message) from e + + def mailradar2mailbox(self,args): + mailbox = None + try: + mailbox = self.db.get_mailbox(args['mailbox_id']) + except GulagDBNotFoundException as e: + raise GulagNotFoundException(whoami(self) + e.message) from e + except GulagDBException as e: + raise GulagException(whoami(self) + e.message) from e + # Prepend Gulag-specific headers to rejected mail + # before pushing into quarantine mailbox + msg = None + if('X-Mailradar-From' not in args['req_headers']): + err = str(whoami(self) + + "Missing Mailradar-specific request header X-Mailradar-From!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('X-Mailradar-Rcpt' not in args['req_headers']): + err = str(whoami(self) + + "Missing Mailradar-specific request header X-Mailradar-Rcpt!" + ) + logging.error(err) + raise GulagBadInputException(err) + #if('X-Rspamd-Symbols' not in args['req_headers']): + # err = str(whoami(self) + + # "Missing Rspamd-specific request header X-Rspamd-Symbols!" + # ) + # logging.error(err) + # raise GulagBadInputException(err) + if('X-Mailradar-Qid' not in args['req_headers']): + err = str(whoami(self) + + "Missing Mailradar-specific request header X-Mailradar-Qid!" + ) + logging.error(err) + raise GulagBadInputException(err) + if('rfc822_message' not in args): + err = str(whoami(self) + "Missing rfc822_message!") + logging.error(err) + raise GulagBadInputException(err) + # all mandatory request headers and body are present + rcpts_hdr = "" + rcpts = None + try: + rcpts = json.loads(str(args['req_headers']['X-Mailradar-Rcpt'])) + except json.JSONDecodeError as e: + raise GulagBadInputException(e.msg) from e + for rcpt in rcpts: + if(len(rcpts_hdr) > 0): + rcpts_hdr += "," + rcpt + else: + rcpts_hdr = rcpt + msg = "Return-Path: <" + args['req_headers']['X-Mailradar-From'] + ">\r\n" + msg += "Received: from Mailradar http2imap relay by gulag-mailbox IMAP: " + msg += args['mailbox_id'] + "\r\n" + msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n" +# msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n" + msg += "X-Spam-QID: " + args['req_headers']['X-Mailradar-Qid'] + "\r\n" + msg += "X-Gulag-Source: mailradar\r\n" + # append original mail + msg += args['rfc822_message'] imap_mb = None try: imap_mb = IMAPmailbox(mailbox) diff --git a/app/GulagDB.py b/app/GulagDB.py index 9c59897..7aecdef 100644 --- a/app/GulagDB.py +++ b/app/GulagDB.py @@ -45,7 +45,7 @@ class GulagDB: autocommit=True ) except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e self.uri_prefixes = uri_prefixes # virtual columns cannot not be stated in where-clause self.vcols['attach_count'] = {} @@ -73,7 +73,7 @@ class GulagDB: results[value] = True return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_limit_clause(self,args): if('query_offset' in args and 'query_limit' in args): @@ -198,7 +198,7 @@ class GulagDB: continue return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_mailbox(self,mailbox_id): try: @@ -222,7 +222,7 @@ class GulagDB: except MailboxException as e: raise GulagDBException(whoami(self) + e.message) from e except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def add_quarmail(self, quarmail): try: @@ -231,19 +231,20 @@ class GulagDB: "(mx_queue_id,env_from,env_rcpt,"+ "hdr_cf,hdr_from,hdr_subject,"+ "hdr_msgid,hdr_date,cf_meta,"+ - "mailbox_id,imap_uid,msg_size) " + - "values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "mailbox_id,imap_uid,msg_size,ssdeep,source_id) " + + "values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", (quarmail['mx_queue_id'],quarmail['env_from'],quarmail['env_rcpt'], quarmail['hdr_cf'],quarmail['hdr_from'],quarmail['hdr_subject'], quarmail['hdr_msgid'],quarmail['hdr_date'],quarmail['cf_meta'], - quarmail['mailbox_id'],quarmail['imap_uid'],quarmail['msg_size'] + quarmail['mailbox_id'],quarmail['imap_uid'],quarmail['msg_size'], + quarmail['ssdeep'],quarmail['source_id'] ) ) id = cursor.lastrowid cursor.close() return id except mariadb.Error as e: - raise GulagDBException(whoami(self) + (e)) from e + raise GulagDBException(whoami(self) + (e.msg)) from e def delete_quarmail(self, id): try: @@ -252,7 +253,7 @@ class GulagDB: cursor.close() return True except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_quarmails(self,args): try: @@ -290,7 +291,7 @@ class GulagDB: except GulagDBException as e: raise GulagDBException(whoami(self) + e.message) from e except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_quarmail(self,args): try: @@ -318,7 +319,7 @@ class GulagDB: dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id']) return QuarMail(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_deprecated_mails(self,retention_period): try: @@ -338,19 +339,22 @@ class GulagDB: results.append(dict) return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def add_attachment(self, attach): try: cursor = self.conn.cursor() cursor.execute("insert into Attachments " + - "(filename,content_type,content_encoding,magic) values (%s,%s,%s,%s)", + "(filename,content_type,content_encoding,magic,sha256,ssdeep,size) " + + "values (%s,%s,%s,%s,%s,%s,%s)", (attach['filename'],attach['content_type'], - attach['content_encoding'],attach['magic']) + attach['content_encoding'],attach['magic'], + attach['sha256'],attach['ssdeep'],attach['size'] + ) ) return cursor.lastrowid except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_attachments(self): try: @@ -374,7 +378,7 @@ class GulagDB: results.append(Attachment(dict).__dict__) return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_attachment(self, args): try: @@ -398,7 +402,7 @@ class GulagDB: dict['href'] = self.uri_prefixes['attachments'] + str(dict['id']) return Attachment(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_quarmail_attachments(self,quarmail_id): try: @@ -423,7 +427,9 @@ class GulagDB: results.append(Attachment(dict).__dict__) return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e + except AttachmentException as e: + raise GulagDBException(whoami(self) + e.message) from e def get_quarmail_attachment(self,quarmail_id,attachment_id): try: @@ -450,7 +456,9 @@ class GulagDB: dict['href'] += "/attachments/" + str(dict['id']) return Attachment(dict).__dict__ except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e + except AttachmentException as e: + raise GulagDBException(whoami(self) + e.message) from e def delete_quarmail_attachments(self, quarmail_id): cursor = None @@ -462,7 +470,7 @@ class GulagDB: try: cursor.execute("delete from Attachments where id=" + str(qm_at['id'])) except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e cursor.close() return True @@ -474,7 +482,7 @@ class GulagDB: (quarmail_id, attachment_id) ) except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def add_uri(self,args): try: @@ -485,7 +493,7 @@ class GulagDB: ) return cursor.lastrowid except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def quarmail2uri(self,quarmail_id,uri_id): try: @@ -495,7 +503,7 @@ class GulagDB: (quarmail_id, uri_id) ) except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def get_quarmail_uris(self,quarmail_id): try: @@ -520,7 +528,9 @@ class GulagDB: results.append(URI(dict).__dict__) return results except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e + except URIException as e: + raise GulagDBException(whoami(self) + e.message) from e def get_quarmail_uri(self,quarmail_id,uri_id): try: @@ -550,7 +560,7 @@ class GulagDB: except URIException as e: raise GulagDBException(whoami(self) + e.message) from e except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e def delete_quarmail_uris(self, quarmail_id): cursor = None @@ -562,6 +572,6 @@ class GulagDB: try: cursor.execute("delete from URIs where id=" + str(qm_uri['id'])) except mariadb.Error as e: - raise GulagDBException(whoami(self) + str(e)) from e + raise GulagDBException(whoami(self) + str(e.msg)) from e cursor.close() return True diff --git a/app/Resources.py b/app/Resources.py index 6212fb5..93251a9 100644 --- a/app/Resources.py +++ b/app/Resources.py @@ -157,23 +157,34 @@ class ResAttachment(GulagResource): except GulagException as e: abort(500, message=e.message) -class ResRSPAMDImporter(GulagResource): +class ResRspamd2Mailbox(GulagResource): def post(self,mailbox_id): try: - self.gulag.rspamd_http2imap({ + self.gulag.rspamd2mailbox({ "mailbox_id": mailbox_id, "req_headers": request.headers, "rfc822_message": request.get_data(as_text=True) }) return {} - # response = Response( - # response=json.dumps(resp), - # status=201, - # mimetype='application/json', - # headers=Headers([ - # ('Location', 'https://invalid.local/api/v1/blablabla/123') - # ]) - # ) - # return response + except GulagNotFoundException as e: + abort(404, message=e.message) + except GulagBadInputException as e: + abort(400, message=e.message) + except GulagException as e: + abort(500, message=e.message) + +class ResMailradar2Mailbox(GulagResource): + def post(self,mailbox_id): + try: + self.gulag.mailradar2mailbox({ + "mailbox_id": mailbox_id, + "req_headers": request.headers, + "rfc822_message": request.get_data(as_text=True) + }) + return {} + except GulagNotFoundException as e: + abort(404, message=e.message) + except GulagBadInputException as e: + abort(400, message=e.message) except GulagException as e: abort(500, message=e.message) diff --git a/app/gulag_server.py b/app/gulag_server.py index f803767..d229a6e 100755 --- a/app/gulag_server.py +++ b/app/gulag_server.py @@ -7,7 +7,8 @@ from Gulag import Gulag,GulagException from Resources import (ResRoot,ResMailboxes, ResQuarMails,ResQuarMail,ResQuarMailAttachments, ResQuarMailAttachment,ResAttachments,ResAttachment, - ResRSPAMDImporter,ResQuarMailURIs,ResQuarMailURI + ResRspamd2Mailbox,ResQuarMailURIs,ResQuarMailURI, + ResMailradar2Mailbox ) parser = argparse.ArgumentParser() parser.add_argument('--config', required=True, help="Path to config file") @@ -62,8 +63,12 @@ try: '/api/v1/attachments/', resource_class_kwargs={'gulag_object': gulag} ) - api.add_resource(ResRSPAMDImporter, - '/api/v1/mailboxes//rspamdimporter', + api.add_resource(ResRspamd2Mailbox, + '/api/v1/mailboxes//rspamd2mailbox', + resource_class_kwargs={'gulag_object': gulag} + ) + api.add_resource(ResMailradar2Mailbox, + '/api/v1/mailboxes//mailradar2mailbox', resource_class_kwargs={'gulag_object': gulag} ) if __name__ == '__main__': diff --git a/db/gulag.sql b/db/gulag.sql index 9c9fe45..8e7a4f7 100644 --- a/db/gulag.sql +++ b/db/gulag.sql @@ -2,24 +2,24 @@ create database Gulag; use Gulag; -create table SMTPrelays( +create table Mailrelays( id varchar(64) not null primary key, smtp_server varchar(256) default '127.0.0.1' collate 'ascii_general_ci', smtp_port smallint unsigned not null default 25, - smtp_security varchar(32) not null default 'plain', - smtp_user varchar(256) default null, - smtp_pass varchar(1024) default null, + smtp_security varchar(32) not null default 'plain' collate 'ascii_general_ci', + smtp_user varchar(256) default null collate 'ascii_general_ci', + smtp_pass varchar(1024) default null collate 'ascii_general_ci', comment varchar(256) default null )ENGINE = InnoDB; create table Mailboxes( email_address varchar(767) not null primary key collate 'ascii_general_ci', name varchar(256) not null, - imap_server varchar(256) not null default '127.0.0.1', + imap_server varchar(256) not null default '127.0.0.1' collate 'ascii_general_ci', imap_port smallint unsigned not null default 143, - imap_security varchar(32) not null default 'plain', - imap_user varchar(256) not null, - imap_pass varchar(1024) not null, + imap_security varchar(32) not null default 'plain' collate 'ascii_general_ci', + imap_user varchar(256) not null collate 'ascii_general_ci', + imap_pass varchar(1024) not null collate 'ascii_general_ci', imap_mailbox varchar(256) not null default 'INBOX', imap_mailbox_fp varchar(256) not null default 'false-positives', imap_separator varchar(4) not null default '/', @@ -28,31 +28,45 @@ create table Mailboxes( insert into Mailboxes (email_address,name,imap_user,imap_pass) values('quarantine@example.org','E-Mail inbound quarantine','quarantine','quarantine_secure_password'); +create table Sources ( + id varchar(32) not null collate 'ascii_general_ci' primary key +)ENGINE=InnoDB; +insert into Sources (id) values ('amavis'); +insert into Sources (id) values ('rspamd'); +insert into Sources (id) values ('mailradar'); + create table QuarMails ( id int unsigned auto_increment primary key, ctime TIMESTAMP, - mx_queue_id varchar(64) not null, - env_from varchar(256) not null, + mx_queue_id varchar(64) not null collate 'ascii_general_ci', + env_from varchar(256) not null , env_rcpt varchar(256) not null, hdr_cf TEXT, hdr_from varchar(256) default null, hdr_subject varchar(1024) default null, hdr_msgid varchar(512) default null, - hdr_date varchar(128) default null, + hdr_date varchar(128) default null collate 'ascii_general_ci', cf_meta TEXT default null, + imap_uid int unsigned not null, mailbox_id varchar(256) not null collate 'ascii_general_ci', foreign key (mailbox_id) references Mailboxes (email_address) on update cascade on delete cascade, - imap_uid int unsigned not null, - msg_size int unsigned not null + source_id varchar(32) not null collate 'ascii_general_ci', + foreign key (source_id) references Sources (id) on update cascade on delete cascade, + msg_size int unsigned not null, + ssdeep varchar(592) not null collate 'ascii_general_ci' )ENGINE = InnoDB; create table Attachments ( id int unsigned auto_increment primary key, filename varchar(256) not null, - content_type varchar(256) not null, - content_encoding varchar(64), + content_type varchar(256) not null collate 'ascii_general_ci', + content_encoding varchar(64) collate 'ascii_general_ci', magic varchar(128), - comment varchar(256) + comment varchar(256), + size int unsigned not null, + sandbox_results varchar(1024) default null collate 'ascii_general_ci', + sha256 varchar(64) not null collate 'ascii_general_ci', + ssdeep varchar(592) not null collate 'ascii_general_ci' )ENGINE = InnoDB; create table QuarMail2Attachment ( @@ -64,8 +78,9 @@ create table QuarMail2Attachment ( create table URIs ( id int unsigned auto_increment primary key, - uri varchar(2048), - fqdn varchar(512) + uri varchar(2048) not null collate 'ascii_general_ci', + fqdn varchar(512) not null collate 'ascii_general_ci', + sandbox_results varchar(1024) default null collate 'ascii_general_ci' )ENGINE = InnoDB; create table QuarMail2URI (