github-public-mirror/gulag
1
0
Fork 0
mirror of https://github.com/chillout2k/gulag.git synced 2025-12-13 16:00:18 +00:00
Code Issues Projects Releases Wiki Activity

* build dependent http-proxy as ARG

Browse Source 
* extended exception handling now meets the requirements of REST
* README cosmetics
This commit is contained in:
Dominik Chilla 2018-12-31 00:51:11 +01:00
parent 259aec5a96
commit ebe8742776
9 changed files with 271 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
README.md
View File

@ -1,2 +1,48 @@
# gulag
Gulag quarantine
## curl examples
### get all QuarMail metadata
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails | jq
```
### get all QuarMail metadata + RFC822 messages
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails?rfc822_message=1 | jq
```
### get QuarMails metadata by (jqgrid-style) filter
```curl -v -s -G --data-urlencode 'filters={"groupOp":"OR","rules":[{"field":"hdr_subject","op":"eq","data":"996 test from quar mit sync xyz"}]}' http://127.0.0.1:9090/api/v1/quarmails | jq
```
### delete a QuarMail by ID
```curl -v -s -X DELETE http://127.0.0.1:9090/api/v1/quarmails/141 | jq
```
### get a QuarMail´s metadata by ID
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136 | jq
```
### get a QuarMail´s metadata by ID + RFC822 message
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136?rfc822_message=1 | jq
```
### get all URIs of a QuarMail
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136/uris | jq
```
### get an URI of a QuarMail by ID
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136/uris/249 | jq
```
### get all attachments metadata of a QuarMail
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136/attachments | jq
```
### get an attachments metadata of a QuarMail by ID
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136/attachments/71 | jq
```
### get an attachments metadata of a QuarMail by ID + attachment data
```curl -v -s http://127.0.0.1:9090/api/v1/quarmails/136/attachments/71?data=1 | jq
```

46
app/Gulag.py
View File

@ -1,6 +1,8 @@
import json, sys,os,logging,re,magic
import email,email.header,email.message
from GulagDB import GulagDB,GulagDBException
from GulagDB import (
GulagDB,GulagDBException,GulagDBNotFoundException,GulagDBBadInputException
)
from GulagMailbox import IMAPmailbox,IMAPmailboxException
from GulagUtils import whoami,extract_uris,extract_fqdn
import ssdeep, hashlib
@ -10,6 +12,16 @@ class GulagException(Exception):
def __init__(self,message):
self.message = message
class GulagNotFoundException(Exception):
message = None
def __init__(self,message):
self.message = message
class GulagBadInputException(Exception):
message = None
def __init__(self,message):
self.message = message
class Gulag:
version = None
config = None
@ -231,6 +243,8 @@ class Gulag:
try:
self.check_fields('QuarMails',args)
qms_db = self.db.get_quarmails(args)
except GulagDBBadInputException as e:
raise GulagBadInputException(whoami(self) + e.message) from e
except(GulagException,GulagDBException) as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -278,6 +292,8 @@ class Gulag:
qm_db = None
try:
qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -287,6 +303,8 @@ class Gulag:
mailbox = None
try:
mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -306,12 +324,16 @@ class Gulag:
qm_db = None
try:
qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
mailbox = None
try:
mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -341,7 +363,7 @@ class Gulag:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
imap_mb.close()
return True
return
def get_quarmail_attachments(self,args):
try:
@ -356,6 +378,8 @@ class Gulag:
qmat_db = self.db.get_quarmail_attachment(
args['quarmail_id'],args['attachment_id']
)
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -365,6 +389,8 @@ class Gulag:
mailbox = None
try:
mailbox = self.db.get_mailbox(qmat_db['mailbox_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -384,6 +410,8 @@ class Gulag:
at_db = None
try:
at_db = self.db.get_attachment({"id": args['id']})
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
raise GulagException(whoami(self) + e.message) from e
if 'data' not in args:
@ -399,12 +427,16 @@ class Gulag:
qm_db = None
try:
qm_db = self.db.get_quarmail({"id": args['quarmail_id']})
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
mailbox = None
try:
mailbox = self.db.get_mailbox(qm_db['mailbox_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
@ -424,10 +456,20 @@ class Gulag:
logging.warning(whoami(self) + e.message)
raise GulagException(whoami(self) + e.message) from e
def get_quarmail_uri(self,args):
try:
return self.db.get_quarmail_uri(args['quarmail_id'],args['uri_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
raise GulagException(whoami(self) + e.message) from e
def rspamd_http2imap(self,args):
mailbox = None
try:
mailbox = self.db.get_mailbox(args['mailbox_id'])
except GulagDBNotFoundException as e:
raise GulagNotFoundException(whoami(self) + e.message) from e
except GulagDBException as e:
raise GulagException(whoami(self) + e.message) from e
# check if the request comes really from rspamd´s metadata_exporter

101
app/GulagDB.py
View File

@ -11,6 +11,16 @@ class GulagDBException(Exception):
def __init__(self,message):
self.message = str(message)
class GulagDBNotFoundException(Exception):
message = None
def __init__(self,message):
self.message = str(message)
class GulagDBBadInputException(Exception):
message = None
def __init__(self,message):
self.message = str(message)
class GulagDB:
conn = None
uri_prefixes = None
@ -51,8 +61,8 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self)
+ "describe " + table_name + " failed!"
raise GulagDBNotFoundException(whoami(self)
+ "describe " + table_name + " failed: got no fields!"
)
desc = cursor.description
cursor.close()
@ -70,21 +80,25 @@ class GulagDB:
try:
int(args['query_offset'])
except ValueError:
raise GulagDBException(whoami(self) + "query_offset must be numeric!")
raise GulagDBBadInputException(whoami(self) +
"query_offset must be numeric!"
)
try:
int(args['query_limit'])
except ValueError:
raise GulagDBException(whoami(self) + "query_limit must be numeric!")
raise GulagDBBadInputException(whoami(self) +
"query_limit must be numeric!"
)
return "limit "+args['query_offset']+","+args['query_limit']
elif('query_offset' in args and 'query_limit' not in args):
raise GulagDBException(whoami(self) +
raise GulagDBBadInputException(whoami(self) +
"query_offset without query_limit is useless!"
)
elif('query_limit' in args and 'query_offset' not in args):
try:
int(args['query_limit'])
except ValueError:
raise GulagDBException(whoami(self) + "query_limit must be numeric!")
raise GulagDBBadInputException(whoami(self) + "query_limit must be numeric!")
return "limit " + args['query_limit']
else:
return ""
@ -110,21 +124,31 @@ class GulagDB:
def get_where_clause_from_filters(self,filters):
# {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]}
if 'rules' not in filters:
raise GulagDBException(whoami(self) + "no 'rules' found in filters!")
raise GulagDBBadInputException(whoami(self) +
"no 'rules' found in filters!"
)
if 'groupOp' not in filters:
raise GulagDBException(whoami(self) + "'groupOp' not found in filters!")
raise GulagDBBadInputException(whoami(self) +
"'groupOp' not found in filters!"
)
if filters['groupOp'] != 'AND' and filters['groupOp'] != 'OR':
raise GulagDBException(whoami(self) +
raise GulagDBBadInputException(whoami(self) +
"invalid 'groupOp': " + filters['groupOp']
)
where_clause = ""
for rule in filters['rules']:
if 'field' not in rule:
raise GulagDBException(whoami(self) + "'field' not found in rule!")
raise GulagDBBadInputException(whoami(self) +
"'field' not found in rule!"
)
if 'op' not in rule:
raise GulagDBException(whoami(self) + "'op' not found in rule!")
raise GulagDBBadInputException(whoami(self) +
"'op' not found in rule!"
)
if 'data' not in rule:
raise GulagDBException(whoami(self) + "'data' not found in rule!")
raise GulagDBBadInputException(whoami(self) +
"'data' not found in rule!"
)
field_op_data = None
if(rule['op'] == 'eq'):
field_op_data = rule['field'] + "='" + rule['data'] + "'"
@ -141,7 +165,9 @@ class GulagDB:
elif(rule['op'] == 'lt'):
field_op_data = rule['field'] + " < '" + rule['data'] + "'"
if(field_op_data == None):
raise GulagDBException(whoami(self) + "invalid rule-op: " + rule['op'])
raise GulagDBBadInputException(whoami(self) +
"invalid rule-op: " + rule['op']
)
if(len(filters['rules']) == 1 or len(where_clause) == 0):
if rule['field'] in self.vcols:
where_clause = "having " + field_op_data
@ -158,7 +184,7 @@ class GulagDB:
results = []
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self) + "No mailboxes found in DB!")
return results
desc = cursor.description
for tuple in data:
dict = {}
@ -182,7 +208,7 @@ class GulagDB:
)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self)
raise GulagDBNotFoundException(whoami(self)
+ "Mailbox '" + mailbox_id + "' does not exist!"
)
desc = cursor.description
@ -259,6 +285,8 @@ class GulagDB:
dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
results.append(QuarMail(dict).__dict__)
return results
except GulagDBBadInputException as e:
raise GulagDBBadInputException(whoami(self) + e.message) from e
except GulagDBException as e:
raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e:
@ -275,7 +303,7 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self)
raise GulagDBNotFoundException(whoami(self)
+ "Quarmail with id '"+ str(args['id']) + "' does not exist!"
)
desc = cursor.description
@ -288,10 +316,6 @@ class GulagDB:
else:
dict[name[0]] = value
dict['href'] = self.uri_prefixes['quarmails'] + str(dict['id'])
# try:
# dict['attachments'] = self.get_attachments_by_quarmail(args['id'])
# except GulagDBException as e:
# pass
return QuarMail(dict).__dict__
except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e
@ -363,7 +387,7 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self)
raise GulagDBNotFoundException(whoami(self)
+ "Attachment("+ str(args['id']) +") does not exist!"
)
desc = cursor.description
@ -413,8 +437,9 @@ class GulagDB:
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBException(whoami(self) + "QuarMail("+ str(quarmail_id) +") "
+ "has no attachment (" + str(attachment_id) + ")!"
raise GulagDBNotFoundException(whoami(self) +
"QuarMail("+ str(quarmail_id) +") " +
"has no attachment (" + str(attachment_id) + ")!"
)
desc = cursor.description
tuple = data[0]
@ -497,6 +522,36 @@ class GulagDB:
except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e
def get_quarmail_uri(self,quarmail_id,uri_id):
try:
query = "select URIs.*"
query += " from QuarMail2URI"
query += " left join QuarMails ON QuarMails.id = QuarMail2URI.quarmail_id"
query += " left join URIs ON URIs.id = QuarMail2URI.uri_id"
query += " where QuarMails.id = " + str(quarmail_id)
query += " and URIs.id = " + str(uri_id) + ";"
cursor = self.conn.cursor()
cursor.execute(query)
data = cursor.fetchall()
if not data:
raise GulagDBNotFoundException(whoami(self) +
"QuarMail("+ str(quarmail_id) +")" +
" has no uri (" + str(uri_id) + ")!"
)
desc = cursor.description
tuple = data[0]
dict = {}
for (name, value) in zip(desc, tuple):
dict[name[0]] = value
dict['href'] = self.uri_prefixes['quarmails'] + str(quarmail_id)
dict['href'] += "/uris/" + str(dict['id'])
try:
return URI(dict).__dict__
except URIException as e:
raise GulagDBException(whoami(self) + e.message) from e
except mariadb.Error as e:
raise GulagDBException(whoami(self) + str(e)) from e
def delete_quarmail_uris(self, quarmail_id):
cursor = None
try:

67
app/Resources.py
View File

@ -1,7 +1,9 @@
from flask import request
from flask import request, Response
from flask_restful import Resource, abort, reqparse
from Gulag import GulagException
import json
from Gulag import (
GulagException,GulagNotFoundException,GulagBadInputException
)
class GulagResource(Resource):
gulag = None
@ -37,7 +39,7 @@ class ResMailboxes(GulagResource):
try:
return self.gulag.get_mailboxes()
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResMailbox(GulagResource):
def get(self,id):
@ -50,11 +52,13 @@ class ResQuarMails(GulagResource):
try:
args['filters'] = json.loads(args['filters'])
except json.JSONDecodeError as e:
abort(400, message=whoami(self) + "JSON parse error: " + e.msg)
abort(400, message="Invalid filters: " + e.msg)
try:
return self.gulag.get_quarmails(args)
except GulagException as e:
except GulagBadInputException as e:
abort(400, message=e.message)
except GulagException as e:
abort(500, message=e.message)
class ResQuarMail(GulagResource):
def get(self,quarmail_id):
@ -63,14 +67,19 @@ class ResQuarMail(GulagResource):
if(request.args.get('rfc822_message')):
args['rfc822_message'] = True
return self.gulag.get_quarmail(args)
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
def delete(self,quarmail_id):
args = {"quarmail_id": quarmail_id}
try:
return self.gulag.delete_quarmail(args)
self.gulag.delete_quarmail(args)
return Response(response=None,status=202,mimetype=None)
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResQuarMailAttachments(GulagResource):
def get(self,quarmail_id):
@ -80,7 +89,7 @@ class ResQuarMailAttachments(GulagResource):
try:
return self.gulag.get_quarmail_attachments(args)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResQuarMailAttachment(GulagResource):
def get(self,quarmail_id,attachment_id):
@ -92,8 +101,10 @@ class ResQuarMailAttachment(GulagResource):
args['data'] = True
try:
return self.gulag.get_quarmail_attachment(args)
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResQuarMailURIs(GulagResource):
def get(self,quarmail_id):
@ -101,11 +112,24 @@ class ResQuarMailURIs(GulagResource):
"quarmail_id": quarmail_id
}
if(request.args.get('from_rfc822_message')):
args['from_rfc822_message'] = True
args['from_rfc822_message'] = True
try:
return self.gulag.get_quarmail_uris(args)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResQuarMailURI(GulagResource):
def get(self,quarmail_id,uri_id):
args = {
"quarmail_id": quarmail_id,
"uri_id": uri_id
}
try:
return self.gulag.get_quarmail_uri(args)
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagException as e:
abort(500, message=e.message)
class ResAttachments(GulagResource):
def get(self):
@ -116,8 +140,10 @@ class ResAttachment(GulagResource):
args = {"id": attachment_id}
try:
return self.gulag.get_attachment(args)
except GulagNotFoundException as e:
abort(404, message=e.message)
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)
class ResRSPAMDImporter(GulagResource):
def post(self,mailbox_id):
@ -127,8 +153,15 @@ class ResRSPAMDImporter(GulagResource):
"req_headers": request.headers,
"rfc822_message": request.get_data(as_text=True)
})
# TODO: Response mit Location-Header?
# https://stackoverflow.com/a/22707491
return {"resource: ": "HTTP2IMAP for RSPAMD"}
return {}
# response = Response(
# response=json.dumps(resp),
# status=201,
# mimetype='application/json',
# headers=Headers([
# ('Location', 'https://invalid.local/api/v1/blablabla/123')
# ])
# )
# return response
except GulagException as e:
abort(400, message=e.message)
abort(500, message=e.message)

8
app/gulag_server.py
View File

@ -7,7 +7,7 @@ from Gulag import Gulag,GulagException
from Resources import (ResRoot,ResMailboxes,
ResQuarMails,ResQuarMail,ResQuarMailAttachments,
ResQuarMailAttachment,ResAttachments,ResAttachment,
ResRSPAMDImporter,ResQuarMailURIs
ResRSPAMDImporter,ResQuarMailURIs,ResQuarMailURI
)
parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help="Path to config file")
@ -19,6 +19,8 @@ try:
except GulagException as e:
raise Exception(e.message) from e
app = Flask(__name__)
# https://github.com/flask-restful/flask-restful/issues/780#issuecomment-434588559
app.config['ERROR_404_HELP'] = False
api = Api(app, catch_all_404s=True)
api.add_resource(ResRoot,
'/api/v1/',
@ -48,6 +50,10 @@ try:
'/api/v1/quarmails/<int:quarmail_id>/uris',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResQuarMailURI,
'/api/v1/quarmails/<int:quarmail_id>/uris/<int:uri_id>',
resource_class_kwargs={'gulag_object': gulag}
)
api.add_resource(ResAttachments,
'/api/v1/attachments',
resource_class_kwargs={'gulag_object': gulag}

7
docker-build.sh
View File

@ -22,7 +22,11 @@ fi
IMAGES="gulag-server gulag-db"
for IMAGE in ${IMAGES}; do
/usr/bin/docker build -t "${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}" -f "docker/${IMAGE}/${BASEOS}/Dockerfile" .
/usr/bin/docker build \
--build-arg http_proxy=http://wprx-zdf.zwackl.local:3128 \
--build-arg https_proxy=http://wprx-zdf.zwackl.local:3128 \
-t "${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}" \
-f "docker/${IMAGE}/${BASEOS}/Dockerfile" .
# /usr/bin/docker tag "${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}" "${REGISTRY}/${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}"
done
@ -30,4 +34,3 @@ done
#for IMAGE in ${IMAGES}; do
# /bin/echo "/usr/bin/docker push ${REGISTRY}/${IMAGE}/${BASEOS}:${VERSION}_${BRANCH}"
#done

2
docker/gulag-db/debian/Dockerfile
View File

@ -1,3 +1,5 @@
ARG http_proxy
ARG https_proxy
FROM debian
RUN apt update && \
apt -yq --no-install-recommends install procps net-tools mariadb-server mariadb-client

8
docker/gulag-server/debian/Dockerfile
View File

@ -1,17 +1,19 @@
ARG http_proxy
ARG https_proxy
FROM debian
LABEL maintainer="Dominik Chilla"
ENV DEBIAN_FRONTEND=noninteractive \
TZ=Europe/Berlin
RUN set -ex ; \
RUN env; set -ex ; \
apt-get -qq update \
&& apt-get -qq --no-install-recommends install \
uwsgi-plugin-python3 python3-setuptools python3-flask \
python3-flask-restful python3-mysql.connector \
uwsgi uwsgi-plugin-python3 procps net-tools \
python3-pip libmagic1 python3-ssdeep \
&& pip3 install python-magic
&& pip3 install python-magic \
&& /bin/mkdir /config /socket /app
RUN /bin/mkdir /config /socket /app
COPY app/*.py /app/

41
gulag-openapi-2.0.yaml
View File

@ -1,6 +1,6 @@
swagger: '2.0'
info:
description: Gulag quarantine REST API
description: https://app.swaggerhub.com/apis/DC-IT-Consulting/Gulag/1.0.0#/
version: "1.0.0"
title: Gulag quarantine REST API
contact:
@ -173,10 +173,10 @@ paths:
required: true
type: integer
responses:
200:
202:
description: quarantined email deleted
400:
description: bad input parameter
404:
description: not found
500:
description: server error
@ -228,11 +228,11 @@ paths:
required: false
responses:
200:
description: quarantined email object
description: attachment object
schema:
$ref: '#/definitions/Attachment'
400:
description: bad input parameter
404:
description: not found
500:
description: server error
@ -265,6 +265,33 @@ paths:
500:
description: server error
/quarmails/{quarmail_id}/uris/{uri_id}:
get:
summary: "returns an URI by ID"
operationId: get_quarmail_uri
produces:
- application/json
parameters:
- in: path
name: quarmail_id
description: unique id of quarantined email
required: true
type: string
- in: path
name: uri_id
description: id of URI to fetch
required: true
type: string
responses:
200:
description: URI object
schema:
$ref: '#/definitions/URI'
404:
description: not found
500:
description: server error
definitions:
QuarMail:
type: object