mirror of
https://github.com/chillout2k/gulag.git
synced 2025-12-13 16:00:18 +00:00
first dockerized approach
This commit is contained in:
parent
544dabe94c
commit
943e7bee04
51
README.md
51
README.md
@ -1,2 +1,53 @@
|
|||||||
# gulag
|
# gulag
|
||||||
Gulag quarantine
|
Gulag quarantine
|
||||||
|
|
||||||
|
config.json:
|
||||||
|
`{
|
||||||
|
"daemon":{
|
||||||
|
"listen_host": "127.0.0.1",
|
||||||
|
"listen_port": 5001
|
||||||
|
},
|
||||||
|
"trusted_proxies": {
|
||||||
|
"rprx01":[
|
||||||
|
"172.16.100.5", "fd00:100::5"
|
||||||
|
],
|
||||||
|
"rprx02":[
|
||||||
|
"172.16.100.6", "fd00:100::6"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"api_keys": {
|
||||||
|
"HIGHLY_SECURE_API_KEY": {
|
||||||
|
"user": "GULAG APP"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uri_prefixes": {
|
||||||
|
"root": "https://<fqdn>/api/v1/",
|
||||||
|
"quarmails": "https://<fqdn>/api/v1/quarmails/",
|
||||||
|
"attachments": "https://<fqdn>/api/v1/attachments/"
|
||||||
|
},
|
||||||
|
"db":{
|
||||||
|
"server": "127.0.0.1",
|
||||||
|
"user": "root",
|
||||||
|
"password": "",
|
||||||
|
"name": "Gulag"
|
||||||
|
},
|
||||||
|
"cleaner":{
|
||||||
|
"retention_period": "12 hour",
|
||||||
|
"interval": 10
|
||||||
|
},
|
||||||
|
"importer":{
|
||||||
|
"interval": 10
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
|
||||||
|
uwsgi.ini:
|
||||||
|
`[uwsgi]
|
||||||
|
processes = 4
|
||||||
|
cheaper = 1
|
||||||
|
cheaper-initial = 1
|
||||||
|
cheaper-step = 1
|
||||||
|
python-path = /app
|
||||||
|
wsgi-file = /app/uwsgi.py
|
||||||
|
pyargv = --config /config/config.json
|
||||||
|
socket = /socket/gulag_uwsgi.sock`
|
||||||
|
|
||||||
|
|||||||
BIN
app/.config.json.swp
Normal file
BIN
app/.config.json.swp
Normal file
Binary file not shown.
117
app/Gulag.py
Normal file
117
app/Gulag.py
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
import json,sys
|
||||||
|
import email,email.header
|
||||||
|
from GulagDB import GulagDB,GulagDBException
|
||||||
|
from GulagMailbox import IMAPmailbox,IMAPmailboxException
|
||||||
|
|
||||||
|
class GulagException(Exception):
|
||||||
|
message = None
|
||||||
|
def __init__(self,message):
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
class Gulag:
|
||||||
|
config = None
|
||||||
|
db = None
|
||||||
|
|
||||||
|
def __init__(self, path_to_config_file):
|
||||||
|
try:
|
||||||
|
with open(path_to_config_file, 'r') as f:
|
||||||
|
self.config = json.load(f)
|
||||||
|
f.close()
|
||||||
|
except:
|
||||||
|
raise GulagException("CONFIG-FILE-Exception: " + str(sys.exc_info()))
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.db = GulagDB(
|
||||||
|
self.config['db']['server'],
|
||||||
|
self.config['db']['user'],
|
||||||
|
self.config['db']['password'],
|
||||||
|
self.config['db']['name']
|
||||||
|
)
|
||||||
|
except GulagDBException as e:
|
||||||
|
raise GulagException(e.message) from e
|
||||||
|
|
||||||
|
def import_quarmails(self):
|
||||||
|
# Alle Mailboxes durchiterieren und die Meta-Infos aller neuen (unseen)
|
||||||
|
# Nachrichten in die Datenbank importieren
|
||||||
|
for mailbox in self.db.get_mailboxes():
|
||||||
|
imap_mb = None
|
||||||
|
try:
|
||||||
|
imap_mb = IMAPmailbox(
|
||||||
|
mailbox['imap_server'],
|
||||||
|
mailbox['imap_user'],
|
||||||
|
mailbox['imap_pass'],
|
||||||
|
mailbox['imap_mailbox']
|
||||||
|
)
|
||||||
|
except IMAPmailboxException as e:
|
||||||
|
print(e.message)
|
||||||
|
continue
|
||||||
|
quarmail_ids = []
|
||||||
|
attachments = []
|
||||||
|
for unseen in imap_mb.get_unseen_messages():
|
||||||
|
uid = unseen['imap_uid']
|
||||||
|
msg = unseen['msg']
|
||||||
|
msg_size = len(str(msg))
|
||||||
|
r5321_from = email.header.decode_header(msg['Return-Path'])[0][0]
|
||||||
|
r5321_rcpts = email.header.decode_header(msg['X-Envelope-To-Blocked'])[0][0]
|
||||||
|
r5322_from = email.header.decode_header(msg['From'])[0][0]
|
||||||
|
subject = email.header.decode_header(msg['Subject'])[0][0]
|
||||||
|
msg_id = None
|
||||||
|
try:
|
||||||
|
msg_id = email.header.decode_header(msg['Message-ID'])[0][0]
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
date = None
|
||||||
|
try:
|
||||||
|
date = email.header.decode_header(msg['Date'])[0][0]
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
x_spam_status = email.header.decode_header(msg['X-Spam-Status'])[0][0]
|
||||||
|
r5321_rcpts = str(r5321_rcpts).lower()
|
||||||
|
r5321_rcpts = r5321_rcpts.replace(" ", "")
|
||||||
|
# Pro Envelope-RCPT einen Eintrag in die DB schreiben.
|
||||||
|
# Die E-Mail im IMAP-Backend existiert jedoch nur ein Mal und wird
|
||||||
|
# über die mailbox_id sowie die imap_uid mehrfach referenziert.
|
||||||
|
for r5321_rcpt in r5321_rcpts.split(","):
|
||||||
|
quarmail_id = self.db.add_quarmail({
|
||||||
|
'mx_queue_id': 'queue_id', 'env_from': r5321_from, 'env_rcpt': r5321_rcpt,
|
||||||
|
'hdr_cf': x_spam_status, 'hdr_from': r5322_from, 'hdr_subject': subject,
|
||||||
|
'hdr_msgid': msg_id, 'hdr_date': date, 'cf_meta': 'cf_meta',
|
||||||
|
'mailbox_id': 'quarantine@zwackl.de', 'imap_uid': uid, 'msg_size': msg_size
|
||||||
|
})
|
||||||
|
quarmail_ids.append(quarmail_id)
|
||||||
|
# Ende for rcpts
|
||||||
|
# Alle MIME-Parts durchiterieren und Attachments
|
||||||
|
# (MIME-Parts mit name/filename Attribut) extrahieren
|
||||||
|
for part in msg.walk():
|
||||||
|
if part.get_filename():
|
||||||
|
# ist ein Attachment
|
||||||
|
filename = email.header.decode_header(part.get_filename())
|
||||||
|
if filename[0][1]:
|
||||||
|
# Encoded
|
||||||
|
filename = filename[0][0].decode(filename[0][1])
|
||||||
|
else:
|
||||||
|
# Nicht encoded
|
||||||
|
filename = filename[0][0]
|
||||||
|
attach_id = self.db.add_attachment({
|
||||||
|
'filename': filename,
|
||||||
|
'content_type': part.get_content_type()
|
||||||
|
})
|
||||||
|
attachments.append(attach_id)
|
||||||
|
# Ende if part.get_filename()
|
||||||
|
# Ende for msg.walk()
|
||||||
|
# Ende for(unseen)
|
||||||
|
imap_mb.close()
|
||||||
|
# QuarMails und Attachments verknüpfen
|
||||||
|
if(len(attachments) > 0):
|
||||||
|
for quarmail_id in quarmail_ids:
|
||||||
|
for attachment_id in attachments:
|
||||||
|
self.db.quarmail2attachment(str(quarmail_id), str(attachment_id))
|
||||||
|
# Ende for get_mailboxes
|
||||||
|
|
||||||
|
def cleanup_quarmails(self):
|
||||||
|
print("Mails to expunge: " +
|
||||||
|
str(len(
|
||||||
|
self.db.get_deprecated_mails(self.config['cleaner']['retention_period'])
|
||||||
|
))
|
||||||
|
)
|
||||||
|
|
||||||
134
app/GulagDB.py
Normal file
134
app/GulagDB.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
import mysql.connector as mariadb
|
||||||
|
|
||||||
|
class GulagDBException(Exception):
|
||||||
|
message = None
|
||||||
|
def __init__(self,message):
|
||||||
|
self.message = str(message)
|
||||||
|
|
||||||
|
class GulagDB:
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
def __init__(self, server, user, password, name):
|
||||||
|
try:
|
||||||
|
self.conn = mariadb.connect(
|
||||||
|
host=server,
|
||||||
|
user=user,
|
||||||
|
password=password,
|
||||||
|
database=name
|
||||||
|
)
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def get_mailboxes(self):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute("select * from Mailboxes;")
|
||||||
|
results = []
|
||||||
|
data = cursor.fetchall()
|
||||||
|
if data == None:
|
||||||
|
return results
|
||||||
|
desc = cursor.description
|
||||||
|
for tuple in data:
|
||||||
|
dict = {}
|
||||||
|
for (name, value) in zip(desc, tuple):
|
||||||
|
dict[name[0]] = value
|
||||||
|
results.append(dict)
|
||||||
|
return results
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def add_quarmail(self, quarmail):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute("insert into QuarMails " +
|
||||||
|
"(mx_queue_id,env_from,env_rcpt,"+
|
||||||
|
"hdr_cf,hdr_from,hdr_subject,"+
|
||||||
|
"hdr_msgid,hdr_date,cf_meta,"+
|
||||||
|
"mailbox_id,imap_uid,msg_size) " +
|
||||||
|
"values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||||
|
(quarmail['mx_queue_id'],quarmail['env_from'],quarmail['env_rcpt'],
|
||||||
|
quarmail['hdr_cf'],quarmail['hdr_from'],quarmail['hdr_subject'],
|
||||||
|
quarmail['hdr_msgid'],quarmail['hdr_date'],quarmail['cf_meta'],
|
||||||
|
quarmail['mailbox_id'],quarmail['imap_uid'],quarmail['msg_size']
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.conn.commit()
|
||||||
|
return cursor.lastrowid
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def del_quarmail(self, id):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute("delete from QuarMails where id=%s;", (id))
|
||||||
|
self.conn.commit()
|
||||||
|
return cursor.lastrowid
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def get_quarmails(self, mailbox_id):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
"select * from QuarMails where mailbox_id='%s';",
|
||||||
|
(mailbox_id)
|
||||||
|
)
|
||||||
|
results = []
|
||||||
|
data = cursor.fetchall()
|
||||||
|
if data == None:
|
||||||
|
return results
|
||||||
|
desc = cursor.description
|
||||||
|
for tuple in data:
|
||||||
|
dict = {}
|
||||||
|
for (name, value) in zip(desc, tuple):
|
||||||
|
dict[name[0]] = value
|
||||||
|
results.append(dict)
|
||||||
|
return results
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def get_deprecated_mails(self,retention_period):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
query = "select ctime,mailbox_id,imap_uid from QuarMails where ctime < date_sub(NOW(), INTERVAL "+ retention_period +");"
|
||||||
|
cursor.execute(query)
|
||||||
|
results = []
|
||||||
|
data = cursor.fetchall()
|
||||||
|
if data == None:
|
||||||
|
return results
|
||||||
|
desc = cursor.description
|
||||||
|
for tuple in data:
|
||||||
|
dict = {}
|
||||||
|
for (name, value) in zip(desc, tuple):
|
||||||
|
dict[name[0]] = value
|
||||||
|
results.append(dict)
|
||||||
|
return results
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def add_attachment(self, attach):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute("insert into Attachments " +
|
||||||
|
"(filename, content_type) values (%s,%s)",
|
||||||
|
(attach['filename'], attach['content_type'])
|
||||||
|
)
|
||||||
|
self.conn.commit()
|
||||||
|
return cursor.lastrowid
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
|
def quarmail2attachment(self,quarmail_id,attachment_id):
|
||||||
|
try:
|
||||||
|
cursor = self.conn.cursor()
|
||||||
|
cursor.execute("insert into QuarMail2Attachment " +
|
||||||
|
"(quarmail_id, attachment_id) values (%s,%s)",
|
||||||
|
(quarmail_id, attachment_id)
|
||||||
|
)
|
||||||
|
self.conn.commit()
|
||||||
|
except mariadb.Error as e:
|
||||||
|
raise GulagDBException(e) from e
|
||||||
|
|
||||||
89
app/GulagMailbox.py
Normal file
89
app/GulagMailbox.py
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
import imaplib
|
||||||
|
import email
|
||||||
|
import email.header
|
||||||
|
|
||||||
|
class IMAPmailboxException(Exception):
|
||||||
|
message = None
|
||||||
|
def __init__(self,message):
|
||||||
|
self.message = str(message)
|
||||||
|
|
||||||
|
class IMAPmailbox:
|
||||||
|
imap_server = None
|
||||||
|
imap_user = None
|
||||||
|
imap_pass = None
|
||||||
|
imap_mailbox = None
|
||||||
|
mailbox = None
|
||||||
|
|
||||||
|
def __init__(self, imap_server, imap_user, imap_pass, imap_mailbox):
|
||||||
|
self.imap_server = imap_server
|
||||||
|
self.imap_user = imap_user
|
||||||
|
self.imap_pass = imap_pass
|
||||||
|
self.imap_mailbox = imap_mailbox
|
||||||
|
try:
|
||||||
|
self.mailbox = imaplib.IMAP4(self.imap_server)
|
||||||
|
rv, data = self.mailbox.login(self.imap_user, self.imap_pass)
|
||||||
|
except imaplib.IMAP4.error as e:
|
||||||
|
raise IMAPmailboxException(
|
||||||
|
"LOGIN FAILED FOR " + self.imap_user + '@' + self.imap_server
|
||||||
|
) from e
|
||||||
|
except ConnectionRefusedError as e:
|
||||||
|
raise IMAPmailboxException(
|
||||||
|
self.imap_user + ": IMAP server " + self.imap_server + " refused connection"
|
||||||
|
) from e
|
||||||
|
|
||||||
|
rv, data = self.mailbox.select(self.imap_mailbox)
|
||||||
|
if rv != 'OK':
|
||||||
|
raise IMAPmailboxException(
|
||||||
|
"ERROR: Unable to select mailbox: " + self.imap_mailbox
|
||||||
|
)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.mailbox.close()
|
||||||
|
self.mailbox.logout()
|
||||||
|
|
||||||
|
def get_unseen_messages(self):
|
||||||
|
results = []
|
||||||
|
rv, data = self.mailbox.uid('SEARCH', 'UNSEEN')
|
||||||
|
if rv != 'OK':
|
||||||
|
return
|
||||||
|
for uid in data[0].split():
|
||||||
|
rv, data = self.mailbox.uid('FETCH', uid, '(RFC822)')
|
||||||
|
if rv != 'OK':
|
||||||
|
print("ERROR getting message", str(uid))
|
||||||
|
continue
|
||||||
|
results.append({
|
||||||
|
'imap_uid': uid,
|
||||||
|
'msg': email.message_from_bytes(data[0][1])
|
||||||
|
})
|
||||||
|
return results
|
||||||
|
|
||||||
|
def get_message(self,imap_uid):
|
||||||
|
rv, data = self.mailbox.uid('FETCH', imap_uid, '(RFC822)')
|
||||||
|
if rv != 'OK':
|
||||||
|
raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
|
||||||
|
return email.message_from_bytes(data[0][1])
|
||||||
|
|
||||||
|
def get_attachments(self,imap_uid):
|
||||||
|
results = []
|
||||||
|
rv, data = self.mailbox.uid('FETCH', imap_uid, '(RFC822)')
|
||||||
|
if rv != 'OK':
|
||||||
|
raise IMAPmailboxException("ERROR getting message: %s", str(imap_uid))
|
||||||
|
msg = email.message_from_bytes(data[0][1])
|
||||||
|
for part in msg.walk():
|
||||||
|
if part.get_filename():
|
||||||
|
# ist ein Attachment
|
||||||
|
filename = email.header.decode_header(part.get_filename())
|
||||||
|
if filename[0][1]:
|
||||||
|
# Encoded
|
||||||
|
filename = filename[0][0].decode(filename[0][1])
|
||||||
|
else:
|
||||||
|
# Nicht encoded
|
||||||
|
filename = filename[0][0]
|
||||||
|
results.append({
|
||||||
|
'filename': filename,
|
||||||
|
'content-type': part.get_content_type(), # Ist das wirklich wahr?
|
||||||
|
'content': part.get_payload(decode=True)
|
||||||
|
})
|
||||||
|
# Ende if part.get_filename()
|
||||||
|
return results
|
||||||
|
|
||||||
37
app/Resources.py
Normal file
37
app/Resources.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
from flask import request
|
||||||
|
from flask_restful import Resource, Api, abort
|
||||||
|
import json
|
||||||
|
|
||||||
|
class GulagResource(Resource):
|
||||||
|
gulag = None
|
||||||
|
def __init__(self,gulag_object):
|
||||||
|
self.gulag = gulag_object
|
||||||
|
#XXX self.check_trusted_proxy()
|
||||||
|
#XXX self.check_auth()
|
||||||
|
|
||||||
|
def check_trusted_proxy(self):
|
||||||
|
remote_ip = request.remote_addr
|
||||||
|
if 'trusted_proxies' not in self.gulag.config:
|
||||||
|
# Trusted-proxies not configured
|
||||||
|
return True
|
||||||
|
for proxy in self.gulag.config['trusted_proxies']:
|
||||||
|
for trusted_proxy_ip in self.gulag.config['trusted_proxies'][proxy]:
|
||||||
|
if(remote_ip == trusted_proxy_ip):
|
||||||
|
return True
|
||||||
|
abort(403, message="Untrusted client IP-address!")
|
||||||
|
|
||||||
|
def check_auth(self):
|
||||||
|
if not 'API-KEY' in request.headers:
|
||||||
|
abort(400, message="API-KEY header missing!")
|
||||||
|
api_key = request.headers['API-KEY']
|
||||||
|
if api_key not in self.gulag.config['api_keys']:
|
||||||
|
abort(401, message="NOT AUTHORIZED!")
|
||||||
|
|
||||||
|
class ResRoot(GulagResource):
|
||||||
|
def get(self):
|
||||||
|
return {"resource": "root :)"}
|
||||||
|
|
||||||
|
class ResQuarMails(GulagResource):
|
||||||
|
def get(self):
|
||||||
|
return {"abc": "1234"}
|
||||||
|
# return self.gulag.get_quarmails()
|
||||||
77
app/gulag_server.py
Executable file
77
app/gulag_server.py
Executable file
@ -0,0 +1,77 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse,sys,os,time,signal
|
||||||
|
from flask import Flask
|
||||||
|
from flask_restful import Api
|
||||||
|
from Gulag import Gulag,GulagException
|
||||||
|
from Resources import ResRoot,ResQuarMails
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--config', required=True, help="Path to config file")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
#child_pids = []
|
||||||
|
#importer_pid = os.fork()
|
||||||
|
#if(importer_pid == 0):
|
||||||
|
# # Child process: importer
|
||||||
|
# try:
|
||||||
|
# gulag = Gulag(args.config)
|
||||||
|
# except GulagException as e:
|
||||||
|
# print(e.message)
|
||||||
|
# sys.exit(1)
|
||||||
|
# while True:
|
||||||
|
# try:
|
||||||
|
# gulag.import_quarmails()
|
||||||
|
# except GulagException as e:
|
||||||
|
# print("Importer-Exception: " + e.message)
|
||||||
|
# time.sleep(gulag.config['importer']['interval'])
|
||||||
|
#
|
||||||
|
#cleaner_pid = os.fork()
|
||||||
|
#if(cleaner_pid == 0):
|
||||||
|
# # Child process: cleaner
|
||||||
|
# try:
|
||||||
|
# gulag = Gulag(args.config)
|
||||||
|
# except GulagException as e:
|
||||||
|
# print(e.message)
|
||||||
|
# sys.exit(1)
|
||||||
|
# while True:
|
||||||
|
# try:
|
||||||
|
# gulag.cleanup_quarmails()
|
||||||
|
# except GulagException as e:
|
||||||
|
# print("Cleaner-Exception: " + e.message)
|
||||||
|
# time.sleep(gulag.config['cleaner']['interval'])
|
||||||
|
|
||||||
|
# Parent
|
||||||
|
#child_pids.append(importer_pid)
|
||||||
|
#child_pids.append(cleaner_pid)
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
gulag = Gulag(args.config)
|
||||||
|
except GulagException as e:
|
||||||
|
raise Exception(e.message) from e
|
||||||
|
app = Flask(__name__)
|
||||||
|
api = Api(app, catch_all_404s=True)
|
||||||
|
api.add_resource(ResRoot,
|
||||||
|
'/api/v1/',
|
||||||
|
resource_class_kwargs={'gulag_object': gulag}
|
||||||
|
)
|
||||||
|
api.add_resource(ResQuarMails,
|
||||||
|
'/api/v1/quarmails/',
|
||||||
|
resource_class_kwargs={'gulag_object': gulag}
|
||||||
|
)
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.run(debug=False,
|
||||||
|
# will be overriden by uwsgi.ini
|
||||||
|
host=gulag.config['daemon']['listen_host'],
|
||||||
|
port=gulag.config['daemon']['listen_port']
|
||||||
|
)
|
||||||
|
gulag.db.close()
|
||||||
|
sys.exit(0)
|
||||||
|
except:
|
||||||
|
print("MAIN-EXCEPTION: " + str(sys.exc_info()))
|
||||||
|
# # Destroy childs
|
||||||
|
# for child_pid in child_pids:
|
||||||
|
# print("Killing child pid: %s", child_pid)
|
||||||
|
# os.kill(child_pid, signal.SIGTERM)
|
||||||
|
|
||||||
|
|
||||||
5
app/uwsgi.py
Executable file
5
app/uwsgi.py
Executable file
@ -0,0 +1,5 @@
|
|||||||
|
from gulag_server import app as application
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
application.run()
|
||||||
|
|
||||||
37
config/config.json
Normal file
37
config/config.json
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"daemon":{
|
||||||
|
"listen_host": "127.0.0.1",
|
||||||
|
"listen_port": 5001
|
||||||
|
},
|
||||||
|
"trusted_proxies": {
|
||||||
|
"rprx01":[
|
||||||
|
"172.16.100.5", "fd00:100::5"
|
||||||
|
],
|
||||||
|
"rprx02":[
|
||||||
|
"172.16.100.6", "fd00:100::6"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"api_keys": {
|
||||||
|
"HIGHLY_SECURE_API_KEY": {
|
||||||
|
"user": "GULAG APP"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uri_prefixes": {
|
||||||
|
"root": "https://<fqdn>/api/v1/",
|
||||||
|
"quarmails": "https://<fqdn>/api/v1/quarmails/",
|
||||||
|
"attachments": "https://<fqdn>/api/v1/attachments/"
|
||||||
|
},
|
||||||
|
"db":{
|
||||||
|
"server": "127.0.0.1",
|
||||||
|
"user": "root",
|
||||||
|
"password": "",
|
||||||
|
"name": "Gulag"
|
||||||
|
},
|
||||||
|
"cleaner":{
|
||||||
|
"retention_period": "12 hour",
|
||||||
|
"interval": 10
|
||||||
|
},
|
||||||
|
"importer":{
|
||||||
|
"interval": 10
|
||||||
|
}
|
||||||
|
}
|
||||||
9
config/uwsgi.ini
Normal file
9
config/uwsgi.ini
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[uwsgi]
|
||||||
|
processes = 4
|
||||||
|
cheaper = 1
|
||||||
|
cheaper-initial = 1
|
||||||
|
cheaper-step = 1
|
||||||
|
python-path = /app
|
||||||
|
wsgi-file = /app/uwsgi.py
|
||||||
|
pyargv = --config /config/config.json
|
||||||
|
socket = /socket/gulag_uwsgi.sock
|
||||||
53
db/gulag.sql
Normal file
53
db/gulag.sql
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
create database Gulag;
|
||||||
|
|
||||||
|
create table Mailboxes(
|
||||||
|
email_address varchar(767) not null primary key collate 'ascii_general_ci',
|
||||||
|
name varchar(256) not null,
|
||||||
|
imap_server varchar(256) not null default '127.0.0.1',
|
||||||
|
imap_security varchar(32) not null default 'plain',
|
||||||
|
imap_user varchar(256) not null,
|
||||||
|
imap_pass varchar(256) not null,
|
||||||
|
imap_mailbox varchar(256) not null default 'INBOX',
|
||||||
|
imap_mailbox_fp varchar(256) not null default 'false-positives',
|
||||||
|
imap_separator varchar(4) not null default '/'
|
||||||
|
comment varchar(256) default null
|
||||||
|
)ENGINE = InnoDB;
|
||||||
|
insert into Mailboxes (email_address,name,imap_user,imap_pass,)
|
||||||
|
values('quarantine-in@example.org','E-Mail inbound quarantine','quarantine-in','quarantine-in_secure_password');
|
||||||
|
insert into Mailboxes (email_address,name,imap_user,imap_pass)
|
||||||
|
values('quarantine-out@example.org','E-Mail outbound quarantine','quarantine-out','quarantine-out_secure_password');
|
||||||
|
insert into Mailboxes (email_address,name,imap_user,imap_pass)
|
||||||
|
values('quarantine-sandbox@example.org','E-Mail sandbox quarantine','quarantine-sb','quarantine-sb_secure_password');
|
||||||
|
|
||||||
|
create table QuarMails (
|
||||||
|
id int unsigned auto_increment primary key,
|
||||||
|
ctime TIMESTAMP,
|
||||||
|
mx_queue_id varchar(64) not null,
|
||||||
|
env_from varchar(256) not null,
|
||||||
|
env_rcpt varchar(256) not null,
|
||||||
|
hdr_cf TEXT,
|
||||||
|
hdr_from varchar(256) default null,
|
||||||
|
hdr_subject varchar(1024) default null,
|
||||||
|
hdr_msgid varchar(512) default null,
|
||||||
|
hdr_date varchar(128) default null,
|
||||||
|
cf_meta TEXT default null,
|
||||||
|
mailbox_id varchar(256) not null collate 'ascii_general_ci',
|
||||||
|
foreign key (mailbox_id) references Mailboxes (email_address) on update cascade on delete cascade,
|
||||||
|
imap_uid int unsigned not null,
|
||||||
|
msg_size int unsigned not null
|
||||||
|
)ENGINE = InnoDB;
|
||||||
|
|
||||||
|
create table Attachments (
|
||||||
|
id int unsigned auto_increment primary key,
|
||||||
|
filename varchar(256) not null,
|
||||||
|
content_type varchar(256) not null,
|
||||||
|
comment varchar(256)
|
||||||
|
)ENGINE = InnoDB;
|
||||||
|
|
||||||
|
create table QuarMail2Attachment (
|
||||||
|
quarmail_id int unsigned,
|
||||||
|
attachment_id int unsigned,
|
||||||
|
foreign key (quarmail_id) references QuarMails (id) on delete cascade on update cascade,
|
||||||
|
foreign key (attachment_id) references Attachments (id) on delete cascade on update cascade
|
||||||
|
)ENGINE = InnoDB;
|
||||||
|
|
||||||
33
docker-build.sh
Executable file
33
docker-build.sh
Executable file
@ -0,0 +1,33 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
BRANCH="$(/usr/bin/git branch|/bin/grep \*|/usr/bin/awk {'print $2'})"
|
||||||
|
VERSION="$(/bin/cat VERSION)"
|
||||||
|
BASEOS="$(/bin/cat BASEOS)"
|
||||||
|
#REGISTRY="some-registry.invalid"
|
||||||
|
GO=""
|
||||||
|
|
||||||
|
while getopts g opt
|
||||||
|
do
|
||||||
|
case $opt in
|
||||||
|
g) GO="go";;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "${GO}" ] ; then
|
||||||
|
echo "Building GULAG@docker on '${BASEOS}' for version '${VERSION}' in branch '${BRANCH}'!"
|
||||||
|
echo "GO serious with '-g'!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
IMAGES="gulag-server gulag-db"
|
||||||
|
|
||||||
|
for IMAGE in ${IMAGES}; do
|
||||||
|
/usr/bin/docker build -t "${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}" -f "docker/${IMAGE}/${BASEOS}/Dockerfile" .
|
||||||
|
# /usr/bin/docker tag "${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}" "${REGISTRY}/${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}"
|
||||||
|
done
|
||||||
|
|
||||||
|
#/bin/echo "Push images to registry:"
|
||||||
|
#for IMAGE in ${IMAGES}; do
|
||||||
|
# /bin/echo "/usr/bin/docker push ${REGISTRY}/${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}"
|
||||||
|
#done
|
||||||
|
|
||||||
5
docker/gulag-db/debian/Dockerfile
Normal file
5
docker/gulag-db/debian/Dockerfile
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
FROM debian
|
||||||
|
RUN apt update && \
|
||||||
|
apt -yq --no-install-recommends install procps net-tools mariadb-server mariadb-client
|
||||||
|
COPY db/gulag.sql /.
|
||||||
|
CMD ["/usr/bin/mysqld_safe"]
|
||||||
15
docker/gulag-server/debian/Dockerfile
Normal file
15
docker/gulag-server/debian/Dockerfile
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
FROM debian
|
||||||
|
LABEL maintainer="Dominik Chilla"
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive \
|
||||||
|
TZ=Europe/Berlin
|
||||||
|
|
||||||
|
RUN set -ex ; \
|
||||||
|
apt-get -qq update \
|
||||||
|
&& apt-get -qq --no-install-recommends install \
|
||||||
|
uwsgi-plugin-python3 python3-setuptools python3-flask \
|
||||||
|
python3-flask-restful python3-mysql.connector \
|
||||||
|
uwsgi uwsgi-plugin-python3
|
||||||
|
|
||||||
|
RUN /bin/mkdir /config /socket /app
|
||||||
|
COPY app/*.py /app/
|
||||||
@ -1,130 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import imaplib
|
|
||||||
import email
|
|
||||||
import email.header
|
|
||||||
import json
|
|
||||||
import fcntl
|
|
||||||
import uuid
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
DB_FILE = "quar_db.json"
|
|
||||||
|
|
||||||
def process_mailbox(MAILBOX):
|
|
||||||
db = []
|
|
||||||
try:
|
|
||||||
f = open(DB_FILE, 'r')
|
|
||||||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
||||||
db = json.load(f)
|
|
||||||
fcntl.flock(f, fcntl.LOCK_UN)
|
|
||||||
f.close()
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
except:
|
|
||||||
print("DB-file-open-exception: " + str(sys.exc_info()))
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
rv, data = MAILBOX.uid('SEARCH', 'UNSEEN')
|
|
||||||
if rv != 'OK':
|
|
||||||
return
|
|
||||||
# Alle "neuen" Nachrichten durchiterieren und
|
|
||||||
# Meta-Infos in die Datenbank importieren
|
|
||||||
for uid in data[0].split():
|
|
||||||
uid = uid.decode()
|
|
||||||
rv, data = MAILBOX.uid('FETCH', uid, '(RFC822)')
|
|
||||||
if rv != 'OK':
|
|
||||||
print("ERROR getting message", uid)
|
|
||||||
sys.exit(1)
|
|
||||||
msg = email.message_from_bytes(data[0][1])
|
|
||||||
msg_size = len(str(msg))
|
|
||||||
attachments = []
|
|
||||||
# Alle MIME-Parts durchiterieren und Attachments
|
|
||||||
# (MIME-Parts mit name/filename Attribut) extrahieren
|
|
||||||
for part in msg.walk():
|
|
||||||
if part.get_filename():
|
|
||||||
# ist ein Attachment
|
|
||||||
filename = email.header.decode_header(part.get_filename())
|
|
||||||
if filename[0][1]:
|
|
||||||
# Encoded
|
|
||||||
filename = filename[0][0].decode(filename[0][1])
|
|
||||||
else:
|
|
||||||
# Nicht encoded
|
|
||||||
filename = filename[0][0]
|
|
||||||
attachments.append({
|
|
||||||
'filename': filename,
|
|
||||||
'content-type': part.get_content_type() # Ist das wirklich wahr?
|
|
||||||
})
|
|
||||||
# Ende if part.get_filename()
|
|
||||||
# Ende for ... msg.walk()
|
|
||||||
r5321_from = email.header.decode_header(msg['Return-Path'])[0][0]
|
|
||||||
r5321_rcpts = email.header.decode_header(msg['X-Envelope-To-Blocked'])[0][0]
|
|
||||||
r5322_from = email.header.decode_header(msg['From'])[0][0]
|
|
||||||
subject = email.header.decode_header(msg['Subject'])[0][0]
|
|
||||||
quar_id = str(uuid.uuid4())
|
|
||||||
msg_id = None
|
|
||||||
try:
|
|
||||||
msg_id = email.header.decode_header(msg['Message-ID'])[0][0]
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
x_spam_status = email.header.decode_header(msg['X-Spam-Status'])[0][0]
|
|
||||||
r5321_rcpts = str(r5321_rcpts).lower()
|
|
||||||
r5321_rcpts = r5321_rcpts.replace(" ", "")
|
|
||||||
db.append({
|
|
||||||
'quarantine_id': quar_id, # AUTO_INCREMENT
|
|
||||||
'envelope_sender': r5321_from,
|
|
||||||
'envelope_recipients': r5321_rcpts,
|
|
||||||
'from_header': r5322_from,
|
|
||||||
'subject': subject,
|
|
||||||
'message_id': msg_id,
|
|
||||||
'imap_uid': uid,
|
|
||||||
'spam_status': x_spam_status,
|
|
||||||
'msg_size': msg_size,
|
|
||||||
'attachments': attachments
|
|
||||||
})
|
|
||||||
# Ende for(unseen)
|
|
||||||
try:
|
|
||||||
f = open(DB_FILE, 'w')
|
|
||||||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
||||||
json.dump(db, f)
|
|
||||||
fcntl.flock(f, fcntl.LOCK_UN)
|
|
||||||
f.close()
|
|
||||||
except:
|
|
||||||
print("DB-file-open-exception: " + str(sys.exc_info()))
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--config', required=True, help="Path to config file")
|
|
||||||
args = parser.parse_args()
|
|
||||||
config = {}
|
|
||||||
try:
|
|
||||||
with open(args.config, 'r') as f:
|
|
||||||
config = json.load(f)
|
|
||||||
f.close()
|
|
||||||
except:
|
|
||||||
print("CONFIG-FILE-Exception: " + str(sys.exc_info()))
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# INBOX = imaplib.IMAP4('mbox02.zwackl.local')
|
|
||||||
INBOX = imaplib.IMAP4(config['imap']['server'])
|
|
||||||
|
|
||||||
try:
|
|
||||||
# rv, data = INBOX.login(EMAIL_ACCOUNT, EMAIL_PASS)
|
|
||||||
rv, data = INBOX.login(
|
|
||||||
config['imap']['user'], config['imap']['password']
|
|
||||||
)
|
|
||||||
except imaplib.IMAP4.error:
|
|
||||||
print ("LOGIN FAILED!!! ")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# rv, data = INBOX.select(EMAIL_FOLDER)
|
|
||||||
rv, data = INBOX.select(config['imap']['folder'])
|
|
||||||
if rv == 'OK':
|
|
||||||
process_mailbox(INBOX)
|
|
||||||
INBOX.close()
|
|
||||||
else:
|
|
||||||
print("ERROR: Unable to open mailbox ", rv)
|
|
||||||
|
|
||||||
INBOX.logout()
|
|
||||||
sys.exit(0)
|
|
||||||
Loading…
Reference in New Issue
Block a user