whoami@Utils, pagination, cleaner resource routing

This commit is contained in:
Dominik Chilla 2018-12-08 23:29:32 +01:00
parent d51ae6b2f1
commit a1ceb75805
6 changed files with 209 additions and 91 deletions

View File

@ -1,9 +1,8 @@
import json,sys
import email,email.header,email.message
from flask import request
from smtplib import SMTP
from GulagDB import GulagDB,GulagDBException
from GulagMailbox import IMAPmailbox,IMAPmailboxException
from GulagUtils import whoami
class GulagException(Exception):
message = None
@ -14,6 +13,7 @@ class Gulag:
version = None
config = None
db = None
fields = {}
def __init__(self, path_to_config_file):
self.version = "VERSION-TODO!"
@ -22,12 +22,29 @@ class Gulag:
self.config = json.load(f)
f.close()
except:
raise GulagException("CONFIG-FILE-Exception: " + str(sys.exc_info()))
raise GulagException(whoami(self) + str(sys.exc_info()))
try:
self.db = GulagDB(self.config['db'],self.config['uri_prefixes'])
self.fields['Mailboxes'] = self.db.get_fields('Mailboxes')
self.fields['QuarMails'] = self.db.get_fields('QuarMails')
self.fields['Attachments'] = self.db.get_fields('Attachments')
except GulagDBException as e:
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
def check_fields(self,fields_target,args):
if fields_target not in self.fields:
raise GulagException(
whoami(self) + fields_target + " not found in Gulag.fields!"
)
for arg in args:
if(arg == 'query_offset' or arg == 'query_limit'
or arg == 'sort_index' or arg == 'sort_order'):
continue
if arg not in self.fields[fields_target]:
raise GulagException(
whoami(self) + arg + " is not a valid field of "
+ fields_target + "!"
)
# Iterate through all mailboxes, extract metadata
# from all unseen mails and pump them into database
@ -122,7 +139,6 @@ class Gulag:
self.db.quarmail2attachment(str(quarmail_id), str(attachment_id))
# Ende for(unseen)
imap_mb.close()
# Ende for get_mailboxes
def cleanup_quarmails(self):
@ -136,20 +152,23 @@ class Gulag:
try:
return self.db.get_mailboxes()
except GulagDBException as e:
raise GulagException("GulagDBException: " + e.message) from e
raise GulagException(whoami(self) + e.message) from e
def get_quarmails(self):
def get_quarmails(self,args):
try:
return self.db.get_quarmails()
except GulagDBException as e:
raise GulagException("GulagDBException: " + e.message) from e
self.check_fields('QuarMails',args)
return self.db.get_quarmails(args)
except(GulagException,GulagDBException) as e:
raise GulagException(
whoami(self) + e.message
) from e
def get_quarmail(self,args):
qm_db = None
try:
qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
except GulagDBException as e:
raise GulagException("GulagDBException: " + e.message) from e
raise GulagException(whoami(self) + e.message) from e
if 'rfc822_message' not in args:
return qm_db
# pull full RFC822 message from IMAP mailbox
@ -157,7 +176,7 @@ class Gulag:
try:
mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
except GulagDBException as e:
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
imap_mb = None
try:
imap_mb = IMAPmailbox(mailbox)
@ -165,14 +184,14 @@ class Gulag:
return qm_db
except IMAPmailboxException as e:
print(e.message)
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
def get_quarmail_attachments(self,args):
try:
return self.db.get_quarmail_attachments(args['quarmail_id'])
except GulagDBException as e:
print(e.message)
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
def get_quarmail_attachment(self,args):
qmat_db = None
@ -182,7 +201,7 @@ class Gulag:
)
except GulagDBException as e:
print(e.message)
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
if 'data' not in args:
return qmat_db
# pull attachment from IMAP mailbox
@ -190,7 +209,7 @@ class Gulag:
try:
mailbox = self.db.get_mailbox(qmat_db['mailbox_id'])
except GulagDBException as e:
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
imap_mb = None
try:
imap_mb = IMAPmailbox(mailbox)
@ -200,65 +219,57 @@ class Gulag:
return qmat_db
except IMAPmailboxException as e:
print(e.message)
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
def get_attachment(self,args):
at_db = None
try:
at_db = self.db.get_attachment({"id": args['id']})
except GulagDBException as e:
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
if 'data' not in args:
return at_db
def rspamd_http2imap(self,mailbox_id):
def get_uris(self):
# https://stackoverflow.com/questions/1792366/extract-urls-out-of-email-in-python
return True
def rspamd_http2imap(self,args):
mailbox = None
try:
mailbox = self.db.get_mailbox(mailbox_id)
mailbox = self.db.get_mailbox(args['mailbox_id'])
except GulagDBException as e:
raise GulagException(e.message) from e
raise GulagException(whoami(self) + e.message) from e
# check if the request comes really from rspamd´s metadata_exporter
# default metadata_header prefix 'X-Rspamd' will be expected
if(request.headers.get('X-Rspamd-From') == None):
raise GulagException("Missing Rspamd-specific headers (e.g. X-Rspamd-From)!")
# Prepend gulag-specific headers to rejected mail
if('X-Rspamd-From' not in args['req_headers']):
raise GulagException(whoami(self)
+ "Missing Rspamd-specific headers (e.g. X-Rspamd-From)!"
)
# Prepend Gulag-specific headers to rejected mail
# before pushing into quarantine mailbox
msg = None
try:
rcpts_hdr = ""
for rcpt in json.loads(str(request.headers.get('X-Rspamd-Rcpt'))):
for rcpt in json.loads(str(args['req_headers']['X-Rspamd-Rcpt'])):
if(len(rcpts_hdr) > 0):
rcpts_hdr = rcpts_hdr + "," + rcpt
rcpts_hdr += "," + rcpt
else:
rcpts_hdr = rcpt
msg = "Return-Path: <" + request.headers.get('X-Rspamd-From') + ">\r\n"
msg += "Received: from rspamd_http2imap relay by gulag-mailbox " + mailbox_id + "\r\n"
msg = "Return-Path: <" + args['req_headers']['X-Rspamd-From'] + ">\r\n"
msg += "Received: from rspamd_http2imap relay by gulag-mailbox "
msg += args['mailbox_id'] + "\r\n"
msg += "X-Envelope-To-Blocked: " + rcpts_hdr + "\r\n"
msg += "X-Spam-Status: " + request.headers.get('X-Rspamd-Symbols') + "\r\n"
msg += "X-Spam-QID: " + request.headers.get('X-Rspamd-Qid') + "\r\n"
msg += "X-Spam-Status: " + args['req_headers']['X-Rspamd-Symbols'] + "\r\n"
msg += "X-Spam-QID: " + args['req_headers']['X-Rspamd-Qid'] + "\r\n"
# append original mail
msg += request.get_data(as_text=True)
msg += args['rfc822_message']
except:
raise GulagException(str(sys.exc_info()))
raise GulagException(whoami(self) + str(sys.exc_info()))
imap_mb = None
try:
imap_mb = IMAPmailbox(mailbox)
imap_mb.append_message(msg)
except IMAPmailboxException as e:
raise GulagException(e.message) from e
# def send_mail(self,args):
# try:
# # FIXME: SMTP tranaport security and authentication!
# # with SMTP(host=mailbox['smtp_server'],port=mailbox['smtp_port']) as smtp:
# # try:
# # smtp.sendmail(
# # request.headers.get('X-Rspamd-From'),
# # mailbox_id,
# # msg
# # )
# # except (SMTPRecipientsRefused,SMTPHeloError,SMTPSenderRefused,SMTPDataError) as e:
# # raise GulagException(str(e)) from e
# except TimeoutError as e:
# raise GulagException(str(e)) from e
raise GulagException(whoami(self) + e.message) from e

View File

@ -1,5 +1,10 @@
import mysql.connector as mariadb
from Entities import Mailbox,MailboxException,QuarMail,QuarMailException,Attachment,AttachmentException
from Entities import(
Mailbox,MailboxException,QuarMail,
QuarMailException,Attachment,
AttachmentException
)
from GulagUtils import whoami
class GulagDBException(Exception):
message = None
@ -30,10 +35,69 @@ class GulagDB:
)
self.uri_prefixes = uri_prefixes
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def close(self):
self.conn.close()
def get_fields(self,table_name):
try:
cursor = self.conn.cursor()
query = "describe " + str(table_name) + ";"
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self)
+ "describe " + table_name + " failed!"
)
desc = cursor.description
cursor.close()
results = {}
for tuple in data:
for (name, value) in zip(desc, tuple):
if(name[0] == "Field"):
results[value] = True
return results
except mariadb.Error as e:
raise GulagDBException(whoami(self) + e) from e
def get_limit_clause(self,args):
if('query_offset' in args and 'query_limit' in args):
try:
int(args['query_offset'])
except ValueError:
raise GulagDBException(whoami(self) + "query_offset must be numeric!")
try:
int(args['query_limit'])
except ValueError:
raise GulagDBException(whoami(self) + "query_limit must be numeric!")
return "limit "+args['query_offset']+","+args['query_limit']
elif('query_offset' in args and 'query_limit' not in args):
raise GulagDBException(whois(self) +
"query_offset without query_limit is useless!"
)
elif('query_limit' in args and 'query_offset' not in args):
try:
int(args['query_limit'])
except ValueError:
raise GulagDBException(whoami(self) + "query_limit must be numeric!")
return "limit " + args['query_limit']
else:
return ""
def get_where_clause(self,args):
where_clause = ""
cnt = 0
for arg in args:
if(arg == 'query_offset' or arg == 'query_limit'
or arg == 'sort_index' or arg == 'sort_order'):
continue
if(cnt == 0):
where_clause += "where " + arg + "='" + args[arg] + "' "
else:
where_clause += "and " + arg + "='" + args[arg] + "' "
cnt += 1
return where_clause
def get_mailboxes(self):
try:
@ -42,7 +106,7 @@ class GulagDB:
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException("No mailboxes found in DB!")
raise GulagDBException(whoami(self) + "No mailboxes found in DB!")
desc = cursor.description
for tuple in data:
dict = {}
@ -56,7 +120,7 @@ class GulagDB:
continue
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_mailbox(self,mailbox_id):
try:
@ -66,7 +130,9 @@ class GulagDB:
)
data = cursor.fetchall()
if not data:
raise GulagDBException("Mailbox '" + mailbox_id + "' does not exist!")
raise GulagDBException(whoami(self)
+ "Mailbox '" + mailbox_id + "' does not exist!"
)
desc = cursor.description
tuple = data[0]
dict = {}
@ -76,10 +142,10 @@ class GulagDB:
try:
return Mailbox(dict).__dict__
except MailboxException as e:
raise GulagDBException(e.message) from e
raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def add_quarmail(self, quarmail):
try:
cursor = self.conn.cursor()
@ -99,7 +165,7 @@ class GulagDB:
cursor.close()
return id
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def del_quarmail(self, id):
try:
@ -108,19 +174,20 @@ class GulagDB:
cursor.close()
return True
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_quarmails(self):
try:
def get_quarmails(self,args):
try:
cursor = self.conn.cursor()
query = "select *,(select count(*) from QuarMail2Attachment"
query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count"
query += " from QuarMails;"
query += " from QuarMails " + self.get_where_clause(args)
query += " " + self.get_limit_clause(args) + " ;"
cursor.execute(query)
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException("No Quarmails found in DB!")
raise GulagDBException(whoami(self) + "No QuarMails found in DB!")
desc = cursor.description
cursor.close()
for tuple in data:
@ -133,8 +200,10 @@ class GulagDB:
dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
results.append(QuarMail(dict).__dict__)
return results
except GulagDBException as e:
raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_quarmail(self,args):
try:
@ -147,7 +216,9 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException("Quarmail with id '"+ args['id'] + "' does not exist!")
raise GulagDBException(whoami(self)
+ "Quarmail with id '"+ args['id'] + "' does not exist!"
)
desc = cursor.description
cursor.close()
tuple = data[0]
@ -164,7 +235,7 @@ class GulagDB:
# pass
return QuarMail(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_deprecated_mails(self,retention_period):
try:
@ -184,7 +255,7 @@ class GulagDB:
results.append(dict)
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def add_attachment(self, attach):
try:
@ -195,7 +266,7 @@ class GulagDB:
)
return cursor.lastrowid
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_attachments(self):
try:
@ -209,7 +280,7 @@ class GulagDB:
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException("No attachments found!")
raise GulagDBException(whoami(self) + "No attachments found!")
desc = cursor.description
for tuple in data:
dict = {}
@ -219,7 +290,7 @@ class GulagDB:
results.append(Attachment(dict).__dict__)
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_attachment(self, args):
try:
@ -232,7 +303,9 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException("Attachment("+ str(args['id']) +") does not exist!")
raise GulagDBException(whoami(self)
+ "Attachment("+ str(args['id']) +") does not exist!"
)
desc = cursor.description
tuple = data[0]
dict = {}
@ -241,7 +314,7 @@ class GulagDB:
dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
return Attachment(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_quarmail_attachments(self,quarmail_id):
try:
@ -255,7 +328,9 @@ class GulagDB:
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException("QuarMail("+ str(quarmail_id) +") has no attachments!")
raise GulagDBException(whoami(self)
+ "QuarMail("+ str(quarmail_id) +") has no attachments!"
)
desc = cursor.description
for tuple in data:
dict = {}
@ -266,7 +341,7 @@ class GulagDB:
results.append(Attachment(dict).__dict__)
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def get_quarmail_attachment(self,quarmail_id,attachment_id):
try:
@ -280,7 +355,7 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException("QuarMail("+ str(quarmail_id) +") "
raise GulagDBException(whoami(self) + "QuarMail("+ str(quarmail_id) +") "
+ "has no attachment (" + str(attachment_id) + ")!"
)
desc = cursor.description
@ -292,7 +367,7 @@ class GulagDB:
dict['href'] += "/attachments/" + str(dict['id'])
return Attachment(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e
def quarmail2attachment(self,quarmail_id,attachment_id):
@ -303,5 +378,5 @@ class GulagDB:
(quarmail_id, attachment_id)
)
except mariadb.Error as e:
raise GulagDBException(e) from e
raise GulagDBException(whoami(self) + e) from e

View File

@ -2,6 +2,7 @@ import imaplib
import email
import email.header
import time
from GulagUtils import whoami
class IMAPmailboxException(Exception):
message = None
@ -26,17 +27,17 @@ class IMAPmailbox:
self.mailbox = imaplib.IMAP4(self.imap_server)
rv, data = self.mailbox.login(self.imap_user, self.imap_pass)
except imaplib.IMAP4.error as e:
raise IMAPmailboxException(
raise IMAPmailboxException(whoami(self) +
"LOGIN FAILED FOR " + self.imap_user + '@' + self.imap_server
) from e
except ConnectionRefusedError as e:
raise IMAPmailboxException(
raise IMAPmailboxException(whoami(self) +
self.imap_user + ": IMAP server " + self.imap_server + " refused connection"
) from e
rv, data = self.mailbox.select(self.imap_mailbox)
if rv != 'OK':
raise IMAPmailboxException(
raise IMAPmailboxException(whoami(self) +
"ERROR: Unable to select mailbox: " + self.imap_mailbox
)
@ -63,7 +64,9 @@ class IMAPmailbox:
def get_message(self,imap_uid):
rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)')
if rv != 'OK':
raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
raise IMAPmailboxException(whoami(self) +
"ERROR getting message: %s", str(imap_uid)
)
return data[0][1]
def get_attachment(self,imap_uid,filename):
@ -83,7 +86,7 @@ class IMAPmailbox:
return part.get_payload(decode=False)
# End if part.get_filename()
# End msg.walk() loop
raise IMAPmailboxException(
raise IMAPmailboxException(whoami(self) +
"Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@"
+ str(self.email_address) + " not found!"
)
@ -96,7 +99,9 @@ class IMAPmailbox:
str(message).encode('utf-8')
)
if rv != 'OK':
raise IMAPmailboxException("ERROR appending message: " + rv)
raise IMAPmailboxException(whoami(self)+
"ERROR appending message: " + rv
)
def expunge_message(self,imap_uid):
return True

23
app/GulagUtils.py Normal file
View File

@ -0,0 +1,23 @@
import sys
from smtplib import SMTP
def whoami(obj):
return type(obj).__name__ + "::" + sys._getframe(1).f_code.co_name + "(): "
def send_mail(args):
try:
# FIXME: SMTP tranaport security and authentication!
# with SMTP(host=mailbox['smtp_server'],port=mailbox['smtp_port']) as smtp:
# try:
# smtp.sendmail(
# request.headers.get('X-Rspamd-From'),
# mailbox_id,
# msg
# )
# except (SMTPRecipientsRefused,SMTPHeloError,SMTPSenderRefused,SMTPDataError) as e:
# raise GulagException(str(e)) from e
print("TODO")
except TimeoutError as e:
raise Exception('xyz') from e

View File

@ -45,7 +45,7 @@ class ResMailbox(GulagResource):
class ResQuarMails(GulagResource):
def get(self):
try:
return self.gulag.get_quarmails()
return self.gulag.get_quarmails(request.args.to_dict())
except GulagException as e:
abort(400, message=e.message)
@ -97,7 +97,11 @@ class ResAttachment(GulagResource):
class ResRSPAMDImporter(GulagResource):
def post(self,mailbox_id):
try:
self.gulag.rspamd_http2imap(mailbox_id)
self.gulag.rspamd_http2imap({
"mailbox_id": mailbox_id,
"req_headers": request.headers,
"rfc822_message": request.get_data(as_text=True)
})
# TODO: Response mit Location-Header?
return {"resource: ": "HTTP2IMAP for RSPAMD"}
except GulagException as e:

View File

@ -25,11 +25,11 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResMailboxes,
'/api/v1/mailboxes/',
'/api/v1/mailboxes',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMails,
'/api/v1/quarmails/',
'/api/v1/quarmails',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMail,
@ -37,7 +37,7 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMailAttachments,
'/api/v1/quarmails/<int:quarmail_id>/attachments/',
'/api/v1/quarmails/<int:quarmail_id>/attachments',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMailAttachment,
@ -45,7 +45,7 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResAttachments,
'/api/v1/attachments/',
'/api/v1/attachments',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResAttachment,
@ -53,7 +53,7 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResRSPAMDImporter,
'/api/v1/mailboxes/<string:mailbox_id>/rspamdimporter/',
'/api/v1/mailboxes/<string:mailbox_id>/rspamdimporter',
resource_class_kwargs={'gulag_object': gulag}
)
if __name__ == '__main__':