github-public-mirror/gulag
1
0
Fork 0
mirror of https://github.com/chillout2k/gulag.git synced 2025-12-13 16:00:18 +00:00
Code Issues Projects Releases Wiki Activity

Attachment handling @IMAP

Browse Source
This commit is contained in:
Dominik Chilla 2018-12-02 20:08:10 +01:00
parent bc4796b885
commit d51ae6b2f1
8 changed files with 190 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/Entities.py
View File

@ -97,7 +97,6 @@ class QuarMail:
msg_size = None
href = None
attach_count = None
attachments = None
def __init__(self,qm_ref):
if 'id' not in qm_ref:
@ -140,9 +139,6 @@ class QuarMail:
self.href = qm_ref['href']
if 'attach_count' in qm_ref:
self.attach_count = qm_ref['attach_count']
if 'attachments' in qm_ref:
self.attachments = qm_ref['attachments']
class AttachmentException(Exception):
message = None
@ -153,7 +149,10 @@ class Attachment:
id = None
filename = None
content_type = None
content_encoding = None
comment = None
mailbox_id = None
imap_uid = None
href = None
def __init__(self,at_ref):
@ -166,8 +165,16 @@ class Attachment:
if 'content_type' not in at_ref:
raise AttachmentException("'content_type' is mandatory!")
self.content_type = at_ref['content_type']
if 'content_encoding' in at_ref:
self.content_encoding = at_ref['content_encoding']
if 'comment' in at_ref:
self.comment = at_ref['comment']
if 'mailbox_id' not in at_ref:
raise AttachmentException("'mailbox_id' is mandatory!")
self.mailbox_id = at_ref['mailbox_id']
if 'imap_uid' not in at_ref:
raise AttachmentException("'imap_uid' is mandatory!")
self.imap_uid = at_ref['imap_uid']
if 'href' in at_ref:
self.href = at_ref['href']

54
app/Gulag.py
View File

@ -43,8 +43,8 @@ class Gulag:
quarmail_ids = []
attachments = []
uid = unseen['imap_uid']
msg = unseen['msg']
msg_size = len(str(msg))
msg = email.message_from_bytes(unseen['msg'])
msg_size = len(msg)
r5321_from = email.header.decode_header(msg['Return-Path'])[0][0]
if(r5321_from is not '<>'):
r5321_from = r5321_from.replace("<","")
@ -92,6 +92,7 @@ class Gulag:
'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta',
'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, 'msg_size': msg_size
})
print("QuarMail (%s) imported" % (quarmail_id))
quarmail_ids.append(quarmail_id)
# Ende for rcpts
# Alle MIME-Parts durchiterieren und Attachments
@ -108,12 +109,13 @@ class Gulag:
filename = filename[0][0]
attach_id = self.db.add_attachment({
'filename': filename,
'content_type': part.get_content_type()
'content_type': part.get_content_type(),
'content_encoding': part['Content-Transfer-Encoding']
})
attachments.append(attach_id)
# Ende if part.get_filename()
# Ende for msg.walk()
# QuarMails und Attachments verknüpfen
# QuarMail und Attachments verknüpfen
if(len(attachments) > 0):
for quarmail_id in quarmail_ids:
for attachment_id in attachments:
@ -145,12 +147,12 @@ class Gulag:
def get_quarmail(self,args):
qm_db = None
try:
qm_db = self.db.get_quarmail({"id": args['id']})
qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
except GulagDBException as e:
raise GulagException("GulagDBException: " + e.message) from e
if 'rfc822_message' not in args:
return qm_db
# pull full RFC822 message from mailbox
# pull full RFC822 message from IMAP mailbox
mailbox = None
try:
mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
@ -159,19 +161,55 @@ class Gulag:
imap_mb = None
try:
imap_mb = IMAPmailbox(mailbox)
qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid'])
qm_db['rfc822_message'] = imap_mb.get_message(qm_db['imap_uid']).decode("utf-8")
return qm_db
except IMAPmailboxException as e:
print(e.message)
raise GulagException(e.message) from e
def get_quarmail_attachments(self,args):
try:
return self.db.get_quarmail_attachments(args['quarmail_id'])
except GulagDBException as e:
print(e.message)
raise GulagException(e.message) from e
def get_quarmail_attachment(self,args):
qmat_db = None
try:
qmat_db = self.db.get_quarmail_attachment(
args['quarmail_id'],args['attachment_id']
)
except GulagDBException as e:
print(e.message)
raise GulagException(e.message) from e
if 'data' not in args:
return qmat_db
# pull attachment from IMAP mailbox
mailbox = None
try:
mailbox = self.db.get_mailbox(qmat_db['mailbox_id'])
except GulagDBException as e:
raise GulagException(e.message) from e
imap_mb = None
try:
imap_mb = IMAPmailbox(mailbox)
qmat_db['data'] = imap_mb.get_attachment(
qmat_db['imap_uid'],qmat_db['filename']
)
return qmat_db
except IMAPmailboxException as e:
print(e.message)
raise GulagException(e.message) from e
def get_attachment(self,args):
at_db = None
try:
at_db = self.db.get_attachment({"id": args['id']})
return at_db
except GulagDBException as e:
raise GulagException(e.message) from e
if 'data' not in args:
return at_db
def rspamd_http2imap(self,mailbox_id):
mailbox = None

79
app/GulagDB.py
View File

@ -113,7 +113,6 @@ class GulagDB:
def get_quarmails(self):
try:
cursor = self.conn.cursor()
# cursor.execute("select * from QuarMails;")
query = "select *,(select count(*) from QuarMail2Attachment"
query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count"
query += " from QuarMails;"
@ -159,10 +158,10 @@ class GulagDB:
else:
dict[name[0]] = value
dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
try:
dict['attachments'] = self.get_attachments_by_quarmail(args['id'])
except GulagDBException as e:
pass
# try:
# dict['attachments'] = self.get_attachments_by_quarmail(args['id'])
# except GulagDBException as e:
# pass
return QuarMail(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(e) from e
@ -191,17 +190,46 @@ class GulagDB:
try:
cursor = self.conn.cursor()
cursor.execute("insert into Attachments " +
"(filename, content_type) values (%s,%s)",
(attach['filename'], attach['content_type'])
"(filename, content_type, content_encoding) values (%s,%s,%s)",
(attach['filename'], attach['content_type'], attach['content_encoding'])
)
return cursor.lastrowid
except mariadb.Error as e:
raise GulagDBException(e) from e
def get_attachments(self):
try:
query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
query += " from QuarMail2Attachment"
query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
query += " group by id;"
cursor = self.conn.cursor()
cursor.execute(query)
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException("No attachments found!")
desc = cursor.description
for tuple in data:
dict = {}
for (name, value) in zip(desc, tuple):
dict[name[0]] = value
dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
results.append(Attachment(dict).__dict__)
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
def get_attachment(self, args):
try:
cursor = self.conn.cursor()
cursor.execute("select * from Attachments where id=" + str(args['id']) + ";")
query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
query += " from QuarMail2Attachment"
query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
query += " where id=" + str(args['id']) + ";"
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException("Attachment("+ str(args['id']) +") does not exist!")
@ -215,9 +243,10 @@ class GulagDB:
except mariadb.Error as e:
raise GulagDBException(e) from e
def get_attachments_by_quarmail(self,quarmail_id):
def get_quarmail_attachments(self,quarmail_id):
try:
query = "select Attachments.* from QuarMail2Attachment"
query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
query += " from QuarMail2Attachment"
query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
query += " where QuarMails.id = " + str(quarmail_id) + ";"
@ -232,11 +261,39 @@ class GulagDB:
dict = {}
for (name, value) in zip(desc, tuple):
dict[name[0]] = value
dict['href'] = self.uri_prefixes['attachments'] + str(dict['id'])
dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id)
dict['href'] += "/attachments/" + str(dict['id'])
results.append(Attachment(dict).__dict__)
return results
except mariadb.Error as e:
raise GulagDBException(e) from e
def get_quarmail_attachment(self,quarmail_id,attachment_id):
try:
query = "select Attachments.*,QuarMails.mailbox_id,QuarMails.imap_uid"
query += " from QuarMail2Attachment"
query += " left join QuarMails ON QuarMails.id = QuarMail2Attachment.quarmail_id"
query += " left join Attachments ON Attachments.id = QuarMail2Attachment.attachment_id"
query += " where QuarMails.id = " + str(quarmail_id)
query += " and Attachments.id = " + str(attachment_id) + ";"
cursor = self.conn.cursor()
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException("QuarMail("+ str(quarmail_id) +") "
+ "has no attachment (" + str(attachment_id) + ")!"
)
desc = cursor.description
tuple = data[0]
dict = {}
for (name, value) in zip(desc, tuple):
dict[name[0]] = value
dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id)
dict['href'] += "/attachments/" + str(dict['id'])
return Attachment(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(e) from e
def quarmail2attachment(self,quarmail_id,attachment_id):
try:

36
app/GulagMailbox.py
View File

@ -9,6 +9,7 @@ class IMAPmailboxException(Exception):
self.message = str(message)
class IMAPmailbox:
email_address = None
imap_server = None
imap_user = None
imap_pass = None
@ -16,6 +17,7 @@ class IMAPmailbox:
mailbox = None
def __init__(self, mb_ref):
self.email_address = mb_ref['email_address']
self.imap_server = mb_ref['imap_server']
self.imap_user = mb_ref['imap_user']
self.imap_pass = mb_ref['imap_pass']
@ -54,7 +56,7 @@ class IMAPmailbox:
continue
results.append({
'imap_uid': uid,
'msg': email.message_from_bytes(data[0][1])
'msg': data[0][1]
})
return results
@ -62,31 +64,29 @@ class IMAPmailbox:
rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)')
if rv != 'OK':
raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
return data[0][1].decode("utf-8")
return data[0][1]
def get_attachments(self,imap_uid):
results = []
rv, data = self.mailbox.uid('FETCH', str(imap_uid), '(RFC822)')
if rv != 'OK':
raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
msg = email.message_from_bytes(data[0][1])
def get_attachment(self,imap_uid,filename):
msg = email.message_from_bytes(self.get_message(imap_uid))
for part in msg.walk():
if part.get_filename():
# let´s define parts with filename as attachments
filename = email.header.decode_header(part.get_filename())
if filename[0][1]:
part_fn = email.header.decode_header(part.get_filename())
if part_fn[0][1]:
# Encoded -> decode
filename = filename[0][0].decode(filename[0][1])
part_fn = part_fn[0][0].decode(part_fn[0][1])
else:
# not encoded
filename = filename[0][0]
results.append({
'filename': filename,
'content-type': part.get_content_type(),
'content': part.get_payload(decode=True)
})
part_fn = part_fn[0][0]
print("C-T-E: " + str(part['Content-Transfer-Encoding']))
if(part_fn == filename):
return part.get_payload(decode=False)
# End if part.get_filename()
return results
# End msg.walk() loop
raise IMAPmailboxException(
"Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@"
+ str(self.email_address) + " not found!"
)
def append_message(self,message):
rv, data = self.mailbox.append(

31
app/Resources.py
View File

@ -50,8 +50,8 @@ class ResQuarMails(GulagResource):
abort(400, message=e.message)
class ResQuarMail(GulagResource):
def get(self,id):
args = {"id": id}
def get(self,quarmail_id):
args = {"quarmail_id": quarmail_id}
try:
if(request.args.get('rfc822_message')):
args['rfc822_message'] = True
@ -59,13 +59,36 @@ class ResQuarMail(GulagResource):
except GulagException as e:
abort(400, message=e.message)
class ResQuarMailAttachments(GulagResource):
def get(self,quarmail_id):
args = {"quarmail_id": quarmail_id}
if(request.args.get('data')):
args['data'] = True
try:
return self.gulag.get_quarmail_attachments(args)
except GulagException as e:
abort(400, message=e.message)
class ResQuarMailAttachment(GulagResource):
def get(self,quarmail_id,attachment_id):
args = {
"quarmail_id": quarmail_id,
"attachment_id": attachment_id
}
if(request.args.get('data')):
args['data'] = True
try:
return self.gulag.get_quarmail_attachment(args)
except GulagException as e:
abort(400, message=e.message)
class ResAttachments(GulagResource):
def get(self):
return {"resource": "Attachments"}
class ResAttachment(GulagResource):
def get(self,id):
args = {"id": id}
def get(self,attachment_id):
args = {"id": attachment_id}
try:
return self.gulag.get_attachment(args)
except GulagException as e:

8
app/gulag_helpers.py
View File

@ -20,7 +20,9 @@ if(importer_pid == 0):
try:
gulag.import_quarmails()
except GulagException as e:
print("Importer-Exception: " + e.message)
print("Importer-Exception: " + e.message, file=sys.stderr)
except:
print("Importer-Exception: " + str(sys.exc_info()),file=sys.stderr)
time.sleep(gulag.config['importer']['interval'])
cleaner_pid = os.fork()
@ -36,6 +38,8 @@ if(cleaner_pid == 0):
gulag.cleanup_quarmails()
except GulagException as e:
print("Cleaner-Exception: " + e.message)
except:
print("Cleaner-Exception: " + str(sys.exc_info()),file=sys.stderr)
time.sleep(gulag.config['cleaner']['interval'])
# Parent
@ -46,7 +50,7 @@ try:
while True:
time.sleep(10)
except:
print("MAIN-EXCEPTION: " + str(sys.exc_info()))
print("Helpers MAIN-EXCEPTION: " + str(sys.exc_info()))
# Destroy childs
for child_pid in child_pids:
print("Killing child pid: %s", child_pid)

17
app/gulag_server.py
View File

@ -5,8 +5,9 @@ from flask import Flask
from flask_restful import Api
from Gulag import Gulag,GulagException
from Resources import (ResRoot,ResMailboxes,
ResQuarMails,ResQuarMail,ResAttachments,
ResAttachment,ResRSPAMDImporter
ResQuarMails,ResQuarMail,ResQuarMailAttachments,
ResQuarMailAttachment,ResAttachments,ResAttachment,
ResRSPAMDImporter
)
parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help="Path to config file")
@ -32,7 +33,15 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMail,
'/api/v1/quarmails/<string:id>',
'/api/v1/quarmails/<int:quarmail_id>',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMailAttachments,
'/api/v1/quarmails/<int:quarmail_id>/attachments/',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMailAttachment,
'/api/v1/quarmails/<int:quarmail_id>/attachments/<int:attachment_id>',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResAttachments,
@ -40,7 +49,7 @@ try:
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResAttachment,
'/api/v1/attachments/<string:id>',
'/api/v1/attachments/<int:attachment_id>',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResRSPAMDImporter,

1
db/gulag.sql
View File

@ -49,6 +49,7 @@ create table Attachments (
id int unsigned auto_increment primary key,
filename varchar(256) not null,
content_type varchar(256) not null,
content_encoding varchar(64),
comment varchar(256)
)ENGINE = InnoDB;