github-public-mirror/gulag
1
0
Fork 0
mirror of https://github.com/chillout2k/gulag.git synced 2025-12-13 16:00:18 +00:00
Code Issues Projects Releases Wiki Activity

Mailradar support

Browse Source
This commit is contained in:
Dominik Chilla 2019-01-01 23:49:14 +01:00
parent f4302025dd
commit ebb762ba84
6 changed files with 268 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
app/Entities.py
View File

@ -16,11 +16,6 @@ class Mailbox:
imap_mailbox = None imap_mailbox = None
imap_mailbox_fp = None imap_mailbox_fp = None
imap_separator = None imap_separator = None
smtp_server = None
smtp_port = None
smtp_security = None
smtp_user = None
smtp_pass = None
comment = None comment = None
href = None href = None
@ -38,7 +33,7 @@ class Mailbox:
if re.match("^(plain|starttls|tls)$", mb_ref['imap_security']) is not None: if re.match("^(plain|starttls|tls)$", mb_ref['imap_security']) is not None:
self.imap_security = mb_ref['imap_security'] self.imap_security = mb_ref['imap_security']
else: else:
raise MailboxException('imap_security: {} is invalid! '+ raise MailboxException('imap_security: {} is invalid! '+
'Valid values: plain,starttls,tls'.format(mb_ref['imap_security']) 'Valid values: plain,starttls,tls'.format(mb_ref['imap_security'])
) )
else: else:
@ -60,16 +55,6 @@ class Mailbox:
if 'imap_separator' not in mb_ref: if 'imap_separator' not in mb_ref:
raise MailboxException("'imap_separator' is mandatory!") raise MailboxException("'imap_separator' is mandatory!")
self.imap_seperator = mb_ref['imap_separator'] self.imap_seperator = mb_ref['imap_separator']
if 'smtp_server' in mb_ref:
self.smtp_server = mb_ref['smtp_server']
if 'smtp_port' in mb_ref:
self.smtp_port = mb_ref['smtp_port']
if 'smtp_security' in mb_ref:
self.smtp_security = mb_ref['smtp_security']
if 'smtp_user' in mb_ref:
self.smtp_user = mb_ref['smtp_user']
if 'smtp_pass' in mb_ref:
self.smtp_pass = mb_ref['smtp_pass']
if 'comment' in mb_ref: if 'comment' in mb_ref:
self.comment = mb_ref['comment'] self.comment = mb_ref['comment']
if 'href' in mb_ref: if 'href' in mb_ref:
@ -98,6 +83,8 @@ class QuarMail:
href = None href = None
attach_count = None attach_count = None
uri_count = None uri_count = None
source_id = None
ssdeep = None
def __init__(self,qm_ref): def __init__(self,qm_ref):
if 'id' not in qm_ref: if 'id' not in qm_ref:
@ -142,6 +129,12 @@ class QuarMail:
self.attach_count = qm_ref['attach_count'] self.attach_count = qm_ref['attach_count']
if 'uri_count' in qm_ref: if 'uri_count' in qm_ref:
self.uri_count = qm_ref['uri_count'] self.uri_count = qm_ref['uri_count']
if 'source_id' not in qm_ref:
raise QuarMailException("'source_id' is mandatory!")
self.source_id = qm_ref['source_id']
if 'ssdeep' not in qm_ref:
raise QuarMailException("'ssdeep' is mandatory!")
self.ssdeep = qm_ref['ssdeep']
class AttachmentException(Exception): class AttachmentException(Exception):
message = None message = None
@ -157,6 +150,10 @@ class Attachment:
comment = None comment = None
mailbox_id = None mailbox_id = None
imap_uid = None imap_uid = None
size = None
sha256 = None
ssdeep = None
sandbox_results = None
href = None href = None
def __init__(self,at_ref): def __init__(self,at_ref):
@ -182,6 +179,16 @@ class Attachment:
if 'imap_uid' not in at_ref: if 'imap_uid' not in at_ref:
raise AttachmentException("'imap_uid' is mandatory!") raise AttachmentException("'imap_uid' is mandatory!")
self.imap_uid = at_ref['imap_uid'] self.imap_uid = at_ref['imap_uid']
if 'size' not in at_ref:
raise AttachmentException("'size' is mandatory!")
if 'sha256' not in at_ref:
raise AttachmentException("'sha256' is mandatory!")
self.sha256 = at_ref['sha256']
if 'ssdeep' not in at_ref:
raise AttachmentException("'ssdeep' is mandatory!")
self.ssdeep = at_ref['ssdeep']
if 'sandbox_results' in at_ref:
self.sandbox_results = at_ref['sandbox_results']
if 'href' in at_ref: if 'href' in at_ref:
self.href = at_ref['href'] self.href = at_ref['href']
@ -194,6 +201,7 @@ class URI:
id = None id = None
uri = None uri = None
fqdn = None fqdn = None
sandbox_results = None
href = None href = None
def __init__(self,uri_ref): def __init__(self,uri_ref):
@ -206,6 +214,7 @@ class URI:
if 'fqdn' not in uri_ref: if 'fqdn' not in uri_ref:
raise URIException("'fqdn' is mandatory!") raise URIException("'fqdn' is mandatory!")
self.fqdn = uri_ref['fqdn'] self.fqdn = uri_ref['fqdn']
if 'sandbox_results' in uri_ref:
self.sandbox_results = uri_ref['sandbox_results']
if 'href' in uri_ref: if 'href' in uri_ref:
self.href = uri_ref['href'] self.href = uri_ref['href']

217
app/Gulag.py
View File

@ -97,7 +97,9 @@ class Gulag:
uris = {} uris = {}
uid = unseen['imap_uid'] uid = unseen['imap_uid']
msg = email.message_from_bytes(unseen['msg']) msg = email.message_from_bytes(unseen['msg'])
msg_size = len(msg.as_string()) source_id = 'amavis'
if 'X-Gulag-Source' in msg:
source_id = email.header.decode_header(msg['X-Gulag-Source'])[0][0]
r5321_from = email.header.decode_header(msg['Return-Path'])[0][0] r5321_from = email.header.decode_header(msg['Return-Path'])[0][0]
if(r5321_from is not '<>'): if(r5321_from is not '<>'):
r5321_from = r5321_from.replace("<","") r5321_from = r5321_from.replace("<","")
@ -144,14 +146,22 @@ class Gulag:
# Die E-Mail im IMAP-Backend existiert jedoch nur ein Mal und wird # Die E-Mail im IMAP-Backend existiert jedoch nur ein Mal und wird
# über die mailbox_id sowie die imap_uid mehrfach referenziert. # über die mailbox_id sowie die imap_uid mehrfach referenziert.
for r5321_rcpt in r5321_rcpts.split(","): for r5321_rcpt in r5321_rcpts.split(","):
quarmail_id = self.db.add_quarmail({ try:
'mx_queue_id': mx_queue_id, 'env_from': r5321_from, quarmail_id = self.db.add_quarmail({
'env_rcpt': r5321_rcpt, 'hdr_cf': x_spam_status, 'mx_queue_id': mx_queue_id, 'env_from': r5321_from,
'hdr_from': r5322_from, 'hdr_subject': subject,'hdr_msgid': msg_id, 'env_rcpt': r5321_rcpt, 'hdr_cf': x_spam_status,
'hdr_date': date, 'cf_meta': 'cf_meta', 'hdr_from': r5322_from, 'hdr_subject': subject,
'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, 'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta',
'msg_size': msg_size 'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid,
}) 'source_id': source_id, 'msg_size': len(msg.as_string()),
'ssdeep': ssdeep.hash(msg.as_string())
})
except GulagDBBadInputException as e:
logging.warn(whoami(self) + e.message)
raise GulagBadInputException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warn(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
logging.info(whoami(self) + "QuarMail (%s) imported" % (quarmail_id)) logging.info(whoami(self) + "QuarMail (%s) imported" % (quarmail_id))
quarmail_ids.append(quarmail_id) quarmail_ids.append(quarmail_id)
# Ende for rcpts # Ende for rcpts
@ -167,24 +177,15 @@ class Gulag:
else: else:
# filename isn´t encoded # filename isn´t encoded
filename = filename[0][0] filename = filename[0][0]
logging.info(whoami(self) + attach_decoded = part.get_payload(decode=True)
"SSDEEP: " + ssdeep.hash(part.get_payload(decode=True))
)
logging.info(whoami(self) +
"SHA256 " + hashlib.sha256(
part.get_payload(decode=True)
).hexdigest()
)
attach_magic = None
try:
attach_magic = magic.from_buffer(part.get_payload(decode=True))
except:
logging.info(whoami(self) + ": " + str(sys.exc_info()))
attach_id = self.db.add_attachment({ attach_id = self.db.add_attachment({
'filename': filename, 'filename': filename,
'content_type': part.get_content_type(), 'content_type': part.get_content_type(),
'content_encoding': part['Content-Transfer-Encoding'], 'content_encoding': part['Content-Transfer-Encoding'],
'magic': attach_magic 'magic': magic.from_buffer(attach_decoded),
'sha256': hashlib.sha256(attach_decoded).hexdigest(),
'ssdeep': ssdeep.hash(attach_decoded),
'size': len(attach_decoded)
}) })
attachments.append(attach_id) attachments.append(attach_id)
# End if part.get_filename() # End if part.get_filename()
@ -464,7 +465,7 @@ class Gulag:
except GulagDBException as e: except GulagDBException as e:
raise GulagException(whoami(self) + e.message) from e raise GulagException(whoami(self) + e.message) from e
def rspamd_http2imap(self,args): def rspamd2mailbox(self,args):
mailbox = None mailbox = None
try: try:
mailbox = self.db.get_mailbox(args['mailbox_id']) mailbox = self.db.get_mailbox(args['mailbox_id'])
@ -479,60 +480,128 @@ class Gulag:
+ "Missing Rspamd-specific request header X-Rspamd-From!" + "Missing Rspamd-specific request header X-Rspamd-From!"
) )
logging.error(err) logging.error(err)
raise GulagException(err) raise GulagBadInputException(err)
# Prepend Gulag-specific headers to rejected mail # Prepend Gulag-specific headers to rejected mail
# before pushing into quarantine mailbox # before pushing into quarantine mailbox
msg = None msg = None
if('X-Rspamd-From' not in args['req_headers']):
err = str(whoami(self) +
"Missing Rspamd-specific request header X-Rspamd-From!"
)
logging.error(err)
raise GulagBadInputException(err)
if('X-Rspamd-Rcpt' not in args['req_headers']):
err = str(whoami(self) +
"Missing Rspamd-specific request header X-Rspamd-Rcpt!"
)
logging.error(err)
raise GulagBadInputException(err)
if('X-Rspamd-Symbols' not in args['req_headers']):
err = str(whoami(self) +
"Missing Rspamd-specific request header X-Rspamd-Symbols!"
)
logging.error(err)
raise GulagBadInputException(err)
if('X-Rspamd-Qid' not in args['req_headers']):
err = str(whoami(self) +
"Missing Rspamd-specific request header X-Rspamd-Qid!"
)
logging.error(err)
raise GulagBadInputException(err)
if('rfc822_message' not in args):
err = str(whoami(self) + "Missing rfc822_message!")
logging.error(err)
raise GulagBadInputException(err)
# all mandatory request headers and body are present
rcpts_hdr = ""
rcpts = None
try: try:
if('X-Rspamd-From' not in args['req_headers']): rcpts = json.loads(str(args['req_headers']['X-Rspamd-Rcpt']))
err = str(whoami(self) except json.JSONDecodeError as e:
+ "Missing Rspamd-specific request header X-Rspamd-From!" raise GulagBadInputException(e.msg) from e
) for rcpt in rcpts:
logging.error(err) if(len(rcpts_hdr) > 0):
raise GulagException(err) rcpts_hdr += "," + rcpt
if('X-Rspamd-Rcpt' not in args['req_headers']): else:
err = str(whoami(self) rcpts_hdr = rcpt
+ "Missing Rspamd-specific request header X-Rspamd-Rcpt!" msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n"
) msg += "Received: from Rspamd http2imap relay by gulag-mailbox IMAP: "
logging.error(err) msg += args['mailbox_id'] + "\r\n"
raise GulagException(err) msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n"
if('X-Rspamd-Symbols' not in args['req_headers']): msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n"
err = str(whoami(self) msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n"
+ "Missing Rspamd-specific request header X-Rspamd-Symbols!" msg += "X-Gulag-Source: rspamd\r\n"
) # append original mail
logging.error(err) msg += args['rfc822_message']
raise GulagException(err) imap_mb = None
if('X-Rspamd-Qid' not in args['req_headers']): try:
err = str(whoami(self) imap_mb = IMAPmailbox(mailbox)
+ "Missing Rspamd-specific request header X-Rspamd-Qid!" imap_uid = imap_mb.add_message(msg, unseen=True)
) logging.info(whoami(self) + "IMAP_UID: " + str(imap_uid))
logging.error(err) imap_mb.close()
raise GulagException(err) except IMAPmailboxException as e:
if('rfc822_message' not in args): raise GulagException(whoami(self) + e.message) from e
err = str(whoami(self)
+ "Missing rfc822_message!" def mailradar2mailbox(self,args):
) mailbox = None
logging.error(err) try:
raise GulagException(err) mailbox = self.db.get_mailbox(args['mailbox_id'])
# all mandatory request headers and body are present except GulagDBNotFoundException as e:
rcpts_hdr = "" raise GulagNotFoundException(whoami(self) + e.message) from e
for rcpt in json.loads(str(args['req_headers']['X-Rspamd-Rcpt'])): except GulagDBException as e:
if(len(rcpts_hdr) > 0): raise GulagException(whoami(self) + e.message) from e
rcpts_hdr += "," + rcpt # Prepend Gulag-specific headers to rejected mail
else: # before pushing into quarantine mailbox
rcpts_hdr = rcpt msg = None
msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n" if('X-Mailradar-From' not in args['req_headers']):
msg += "Received: from rspamd_http2imap relay by gulag-mailbox " err = str(whoami(self) +
msg += args['mailbox_id'] + "\r\n" "Missing Mailradar-specific request header X-Mailradar-From!"
msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n" )
msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n" logging.error(err)
msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n" raise GulagBadInputException(err)
# append original mail if('X-Mailradar-Rcpt' not in args['req_headers']):
msg += args['rfc822_message'] err = str(whoami(self) +
except GulagException as e: "Missing Mailradar-specific request header X-Mailradar-Rcpt!"
raise GulagException(e.message) from e )
except: logging.error(err)
raise GulagException(whoami(self) + str(sys.exc_info())) raise GulagBadInputException(err)
#if('X-Rspamd-Symbols' not in args['req_headers']):
# err = str(whoami(self) +
# "Missing Rspamd-specific request header X-Rspamd-Symbols!"
# )
# logging.error(err)
# raise GulagBadInputException(err)
if('X-Mailradar-Qid' not in args['req_headers']):
err = str(whoami(self) +
"Missing Mailradar-specific request header X-Mailradar-Qid!"
)
logging.error(err)
raise GulagBadInputException(err)
if('rfc822_message' not in args):
err = str(whoami(self) + "Missing rfc822_message!")
logging.error(err)
raise GulagBadInputException(err)
# all mandatory request headers and body are present
rcpts_hdr = ""
rcpts = None
try:
rcpts = json.loads(str(args['req_headers']['X-Mailradar-Rcpt']))
except json.JSONDecodeError as e:
raise GulagBadInputException(e.msg) from e
for rcpt in rcpts:
if(len(rcpts_hdr) > 0):
rcpts_hdr += "," + rcpt
else:
rcpts_hdr = rcpt
msg = "Return-Path: <" + args['req_headers']['X-Mailradar-From'] + ">\r\n"
msg += "Received: from Mailradar http2imap relay by gulag-mailbox IMAP: "
msg += args['mailbox_id'] + "\r\n"
msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n"
# msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n"
msg += "X-Spam-QID: " + args['req_headers']['X-Mailradar-Qid'] + "\r\n"
msg += "X-Gulag-Source: mailradar\r\n"
# append original mail
msg += args['rfc822_message']
imap_mb = None imap_mb = None
try: try:
imap_mb = IMAPmailbox(mailbox) imap_mb = IMAPmailbox(mailbox)

62
app/GulagDB.py
View File

@ -45,7 +45,7 @@ class GulagDB:
autocommit=True autocommit=True
) )
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
self.uri_prefixes = uri_prefixes self.uri_prefixes = uri_prefixes
# virtual columns cannot not be stated in where-clause # virtual columns cannot not be stated in where-clause
self.vcols['attach_count'] = {} self.vcols['attach_count'] = {}
@ -73,7 +73,7 @@ class GulagDB:
results[value] = True results[value] = True
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_limit_clause(self,args): def get_limit_clause(self,args):
if('query_offset' in args and 'query_limit' in args): if('query_offset' in args and 'query_limit' in args):
@ -198,7 +198,7 @@ class GulagDB:
continue continue
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_mailbox(self,mailbox_id): def get_mailbox(self,mailbox_id):
try: try:
@ -222,7 +222,7 @@ class GulagDB:
except MailboxException as e: except MailboxException as e:
raise GulagDBException(whoami(self) + e.message) from e raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def add_quarmail(self, quarmail): def add_quarmail(self, quarmail):
try: try:
@ -231,19 +231,20 @@ class GulagDB:
"(mx_queue_id,env_from,env_rcpt,"+ "(mx_queue_id,env_from,env_rcpt,"+
"hdr_cf,hdr_from,hdr_subject,"+ "hdr_cf,hdr_from,hdr_subject,"+
"hdr_msgid,hdr_date,cf_meta,"+ "hdr_msgid,hdr_date,cf_meta,"+
"mailbox_id,imap_uid,msg_size) " + "mailbox_id,imap_uid,msg_size,ssdeep,source_id) " +
"values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", "values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(quarmail['mx_queue_id'],quarmail['env_from'],quarmail['env_rcpt'], (quarmail['mx_queue_id'],quarmail['env_from'],quarmail['env_rcpt'],
quarmail['hdr_cf'],quarmail['hdr_from'],quarmail['hdr_subject'], quarmail['hdr_cf'],quarmail['hdr_from'],quarmail['hdr_subject'],
quarmail['hdr_msgid'],quarmail['hdr_date'],quarmail['cf_meta'], quarmail['hdr_msgid'],quarmail['hdr_date'],quarmail['cf_meta'],
quarmail['mailbox_id'],quarmail['imap_uid'],quarmail['msg_size'] quarmail['mailbox_id'],quarmail['imap_uid'],quarmail['msg_size'],
quarmail['ssdeep'],quarmail['source_id']
) )
) )
id = cursor.lastrowid id = cursor.lastrowid
cursor.close() cursor.close()
return id return id
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + (e)) from e raise GulagDBException(whoami(self) + (e.msg)) from e
def delete_quarmail(self, id): def delete_quarmail(self, id):
try: try:
@ -252,7 +253,7 @@ class GulagDB:
cursor.close() cursor.close()
return True return True
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_quarmails(self,args): def get_quarmails(self,args):
try: try:
@ -290,7 +291,7 @@ class GulagDB:
except GulagDBException as e: except GulagDBException as e:
raise GulagDBException(whoami(self) + e.message) from e raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_quarmail(self,args): def get_quarmail(self,args):
try: try:
@ -318,7 +319,7 @@ class GulagDB:
dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id']) dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
return QuarMail(dict).__dict__ return QuarMail(dict).__dict__
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_deprecated_mails(self,retention_period): def get_deprecated_mails(self,retention_period):
try: try:
@ -338,19 +339,22 @@ class GulagDB:
results.append(dict) results.append(dict)
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def add_attachment(self, attach): def add_attachment(self, attach):
try: try:
cursor = self.conn.cursor() cursor = self.conn.cursor()
cursor.execute("insert into Attachments " + cursor.execute("insert into Attachments " +
"(filename,content_type,content_encoding,magic) values (%s,%s,%s,%s)", "(filename,content_type,content_encoding,magic,sha256,ssdeep,size) " +
"values (%s,%s,%s,%s,%s,%s,%s)",
(attach['filename'],attach['content_type'], (attach['filename'],attach['content_type'],
attach['content_encoding'],attach['magic']) attach['content_encoding'],attach['magic'],
attach['sha256'],attach['ssdeep'],attach['size']
)
) )
return cursor.lastrowid return cursor.lastrowid
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_attachments(self): def get_attachments(self):
try: try:
@ -374,7 +378,7 @@ class GulagDB:
results.append(Attachment(dict).__dict__) results.append(Attachment(dict).__dict__)
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_attachment(self, args): def get_attachment(self, args):
try: try:
@ -398,7 +402,7 @@ class GulagDB:
dict['href'] = self.uri_prefixes['attachments'] + str(dict['id']) dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
return Attachment(dict).__dict__ return Attachment(dict).__dict__
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_quarmail_attachments(self,quarmail_id): def get_quarmail_attachments(self,quarmail_id):
try: try:
@ -423,7 +427,9 @@ class GulagDB:
results.append(Attachment(dict).__dict__) results.append(Attachment(dict).__dict__)
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
except AttachmentException as e:
raise GulagDBException(whoami(self) + e.message) from e
def get_quarmail_attachment(self,quarmail_id,attachment_id): def get_quarmail_attachment(self,quarmail_id,attachment_id):
try: try:
@ -450,7 +456,9 @@ class GulagDB:
dict['href'] += "/attachments/" + str(dict['id']) dict['href'] += "/attachments/" + str(dict['id'])
return Attachment(dict).__dict__ return Attachment(dict).__dict__
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
except AttachmentException as e:
raise GulagDBException(whoami(self) + e.message) from e
def delete_quarmail_attachments(self, quarmail_id): def delete_quarmail_attachments(self, quarmail_id):
cursor = None cursor = None
@ -462,7 +470,7 @@ class GulagDB:
try: try:
cursor.execute("delete from Attachments where id=" + str(qm_at['id'])) cursor.execute("delete from Attachments where id=" + str(qm_at['id']))
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
cursor.close() cursor.close()
return True return True
@ -474,7 +482,7 @@ class GulagDB:
(quarmail_id, attachment_id) (quarmail_id, attachment_id)
) )
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def add_uri(self,args): def add_uri(self,args):
try: try:
@ -485,7 +493,7 @@ class GulagDB:
) )
return cursor.lastrowid return cursor.lastrowid
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def quarmail2uri(self,quarmail_id,uri_id): def quarmail2uri(self,quarmail_id,uri_id):
try: try:
@ -495,7 +503,7 @@ class GulagDB:
(quarmail_id, uri_id) (quarmail_id, uri_id)
) )
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def get_quarmail_uris(self,quarmail_id): def get_quarmail_uris(self,quarmail_id):
try: try:
@ -520,7 +528,9 @@ class GulagDB:
results.append(URI(dict).__dict__) results.append(URI(dict).__dict__)
return results return results
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
except URIException as e:
raise GulagDBException(whoami(self) + e.message) from e
def get_quarmail_uri(self,quarmail_id,uri_id): def get_quarmail_uri(self,quarmail_id,uri_id):
try: try:
@ -550,7 +560,7 @@ class GulagDB:
except URIException as e: except URIException as e:
raise GulagDBException(whoami(self) + e.message) from e raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
def delete_quarmail_uris(self, quarmail_id): def delete_quarmail_uris(self, quarmail_id):
cursor = None cursor = None
@ -562,6 +572,6 @@ class GulagDB:
try: try:
cursor.execute("delete from URIs where id=" + str(qm_uri['id'])) cursor.execute("delete from URIs where id=" + str(qm_uri['id']))
except mariadb.Error as e: except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e raise GulagDBException(whoami(self) + str(e.msg)) from e
cursor.close() cursor.close()
return True return True

33
app/Resources.py
View File

@ -157,23 +157,34 @@ class ResAttachment(GulagResource):
except GulagException as e: except GulagException as e:
abort(500, message=e.message) abort(500, message=e.message)
class ResRSPAMDImporter(GulagResource): class ResRspamd2Mailbox(GulagResource):
def post(self,mailbox_id): def post(self,mailbox_id):
try: try:
self.gulag.rspamd_http2imap({ self.gulag.rspamd2mailbox({
"mailbox_id": mailbox_id, "mailbox_id": mailbox_id,
"req_headers": request.headers, "req_headers": request.headers,
"rfc822_message": request.get_data(as_text=True) "rfc822_message": request.get_data(as_text=True)
}) })
return {} return {}
# response = Response( except GulagNotFoundException as e:
# response=json.dumps(resp), abort(404, message=e.message)
# status=201, except GulagBadInputException as e:
# mimetype='application/json', abort(400, message=e.message)
# headers=Headers([ except GulagException as e:
# ('Location', 'https://invalid.local/api/v1/blablabla/123') abort(500, message=e.message)
# ])
# ) class ResMailradar2Mailbox(GulagResource):
# return response def post(self,mailbox_id):
try:
self.gulag.mailradar2mailbox({
"mailbox_id": mailbox_id,
"req_headers": request.headers,
"rfc822_message": request.get_data(as_text=True)
})
return {}
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagBadInputException as e:
abort(400, message=e.message)
except GulagException as e: except GulagException as e:
abort(500, message=e.message) abort(500, message=e.message)

11
app/gulag_server.py
View File

@ -7,7 +7,8 @@ from Gulag import Gulag,GulagException
from Resources import (ResRoot,ResMailboxes, from Resources import (ResRoot,ResMailboxes,
ResQuarMails,ResQuarMail,ResQuarMailAttachments, ResQuarMails,ResQuarMail,ResQuarMailAttachments,
ResQuarMailAttachment,ResAttachments,ResAttachment, ResQuarMailAttachment,ResAttachments,ResAttachment,
ResRSPAMDImporter,ResQuarMailURIs,ResQuarMailURI ResRspamd2Mailbox,ResQuarMailURIs,ResQuarMailURI,
ResMailradar2Mailbox
) )
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help="Path to config file") parser.add_argument('--config', required=True, help="Path to config file")
@ -62,8 +63,12 @@ try:
'/api/v1/attachments/<int:attachment_id>', '/api/v1/attachments/<int:attachment_id>',
resource_class_kwargs={'gulag_object': gulag} resource_class_kwargs={'gulag_object': gulag}
) )
api.add_resource(ResRSPAMDImporter, api.add_resource(ResRspamd2Mailbox,
'/api/v1/mailboxes/<string:mailbox_id>/rspamdimporter', '/api/v1/mailboxes/<string:mailbox_id>/rspamd2mailbox',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResMailradar2Mailbox,
'/api/v1/mailboxes/<string:mailbox_id>/mailradar2mailbox',
resource_class_kwargs={'gulag_object': gulag} resource_class_kwargs={'gulag_object': gulag}
) )
if __name__ == '__main__': if __name__ == '__main__':

51
db/gulag.sql
View File

@ -2,24 +2,24 @@ create database Gulag;
use Gulag; use Gulag;
create table SMTPrelays( create table Mailrelays(
id varchar(64) not null primary key, id varchar(64) not null primary key,
smtp_server varchar(256) default '127.0.0.1' collate 'ascii_general_ci', smtp_server varchar(256) default '127.0.0.1' collate 'ascii_general_ci',
smtp_port smallint unsigned not null default 25, smtp_port smallint unsigned not null default 25,
smtp_security varchar(32) not null default 'plain', smtp_security varchar(32) not null default 'plain' collate 'ascii_general_ci',
smtp_user varchar(256) default null, smtp_user varchar(256) default null collate 'ascii_general_ci',
smtp_pass varchar(1024) default null, smtp_pass varchar(1024) default null collate 'ascii_general_ci',
comment varchar(256) default null comment varchar(256) default null
)ENGINE = InnoDB; )ENGINE = InnoDB;
create table Mailboxes( create table Mailboxes(
email_address varchar(767) not null primary key collate 'ascii_general_ci', email_address varchar(767) not null primary key collate 'ascii_general_ci',
name varchar(256) not null, name varchar(256) not null,
imap_server varchar(256) not null default '127.0.0.1', imap_server varchar(256) not null default '127.0.0.1' collate 'ascii_general_ci',
imap_port smallint unsigned not null default 143, imap_port smallint unsigned not null default 143,
imap_security varchar(32) not null default 'plain', imap_security varchar(32) not null default 'plain' collate 'ascii_general_ci',
imap_user varchar(256) not null, imap_user varchar(256) not null collate 'ascii_general_ci',
imap_pass varchar(1024) not null, imap_pass varchar(1024) not null collate 'ascii_general_ci',
imap_mailbox varchar(256) not null default 'INBOX', imap_mailbox varchar(256) not null default 'INBOX',
imap_mailbox_fp varchar(256) not null default 'false-positives', imap_mailbox_fp varchar(256) not null default 'false-positives',
imap_separator varchar(4) not null default '/', imap_separator varchar(4) not null default '/',
@ -28,31 +28,45 @@ create table Mailboxes(
insert into Mailboxes (email_address,name,imap_user,imap_pass) insert into Mailboxes (email_address,name,imap_user,imap_pass)
values('quarantine@example.org','E-Mail inbound quarantine','quarantine','quarantine_secure_password'); values('quarantine@example.org','E-Mail inbound quarantine','quarantine','quarantine_secure_password');
create table Sources (
id varchar(32) not null collate 'ascii_general_ci' primary key
)ENGINE=InnoDB;
insert into Sources (id) values ('amavis');
insert into Sources (id) values ('rspamd');
insert into Sources (id) values ('mailradar');
create table QuarMails ( create table QuarMails (
id int unsigned auto_increment primary key, id int unsigned auto_increment primary key,
ctime TIMESTAMP, ctime TIMESTAMP,
mx_queue_id varchar(64) not null, mx_queue_id varchar(64) not null collate 'ascii_general_ci',
env_from varchar(256) not null, env_from varchar(256) not null ,
env_rcpt varchar(256) not null, env_rcpt varchar(256) not null,
hdr_cf TEXT, hdr_cf TEXT,
hdr_from varchar(256) default null, hdr_from varchar(256) default null,
hdr_subject varchar(1024) default null, hdr_subject varchar(1024) default null,
hdr_msgid varchar(512) default null, hdr_msgid varchar(512) default null,
hdr_date varchar(128) default null, hdr_date varchar(128) default null collate 'ascii_general_ci',
cf_meta TEXT default null, cf_meta TEXT default null,
imap_uid int unsigned not null,
mailbox_id varchar(256) not null collate 'ascii_general_ci', mailbox_id varchar(256) not null collate 'ascii_general_ci',
foreign key (mailbox_id) references Mailboxes (email_address) on update cascade on delete cascade, foreign key (mailbox_id) references Mailboxes (email_address) on update cascade on delete cascade,
imap_uid int unsigned not null, source_id varchar(32) not null collate 'ascii_general_ci',
msg_size int unsigned not null foreign key (source_id) references Sources (id) on update cascade on delete cascade,
msg_size int unsigned not null,
ssdeep varchar(592) not null collate 'ascii_general_ci'
)ENGINE = InnoDB; )ENGINE = InnoDB;
create table Attachments ( create table Attachments (
id int unsigned auto_increment primary key, id int unsigned auto_increment primary key,
filename varchar(256) not null, filename varchar(256) not null,
content_type varchar(256) not null, content_type varchar(256) not null collate 'ascii_general_ci',
content_encoding varchar(64), content_encoding varchar(64) collate 'ascii_general_ci',
magic varchar(128), magic varchar(128),
comment varchar(256) comment varchar(256),
size int unsigned not null,
sandbox_results varchar(1024) default null collate 'ascii_general_ci',
sha256 varchar(64) not null collate 'ascii_general_ci',
ssdeep varchar(592) not null collate 'ascii_general_ci'
)ENGINE = InnoDB; )ENGINE = InnoDB;
create table QuarMail2Attachment ( create table QuarMail2Attachment (
@ -64,8 +78,9 @@ create table QuarMail2Attachment (
create table URIs ( create table URIs (
id int unsigned auto_increment primary key, id int unsigned auto_increment primary key,
uri varchar(2048), uri varchar(2048) not null collate 'ascii_general_ci',
fqdn varchar(512) fqdn varchar(512) not null collate 'ascii_general_ci',
sandbox_results varchar(1024) default null collate 'ascii_general_ci'
)ENGINE = InnoDB; )ENGINE = InnoDB;
create table QuarMail2URI ( create table QuarMail2URI (