diff --git a/app/Gulag.py b/app/Gulag.py index ce089b7..f514d8d 100644 --- a/app/Gulag.py +++ b/app/Gulag.py @@ -3,6 +3,7 @@ import email,email.header,email.message from GulagDB import GulagDB,GulagDBException from GulagMailbox import IMAPmailbox,IMAPmailboxException from GulagUtils import whoami,extract_uris,extract_fqdn +import ssdeep, hashlib class GulagException(Exception): message = None @@ -154,6 +155,14 @@ class Gulag: else: # filename isn´t encoded filename = filename[0][0] + logging.info(whoami(self) + + "SSDEEP: " + ssdeep.hash(part.get_payload(decode=True)) + ) + logging.info(whoami(self) + + "SHA256 " + hashlib.sha256( + part.get_payload(decode=True) + ).hexdigest() + ) attach_magic = None try: attach_magic = magic.from_buffer(part.get_payload(decode=True)) @@ -224,9 +233,7 @@ class Gulag: qms_db = self.db.get_quarmails(args) except(GulagException,GulagDBException) as e: logging.warning(whoami(self) + e.message) - raise GulagException( - whoami(self) + e.message - ) from e + raise GulagException(whoami(self) + e.message) from e if 'rfc822_message' not in args: return { 'quarmails': qms_db, diff --git a/app/GulagDB.py b/app/GulagDB.py index abb08bf..d7e5a51 100644 --- a/app/GulagDB.py +++ b/app/GulagDB.py @@ -5,7 +5,6 @@ from Entities import( AttachmentException,URI,URIException ) from GulagUtils import whoami -import json class GulagDBException(Exception): message = None @@ -108,18 +107,17 @@ class GulagDB: cnt += 1 return where_clause - def get_where_clause_from_filters(self,filters_json): + def get_where_clause_from_filters(self,filters): # {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]} - filters = None - where_clause = "" - try: - filters = json.loads(filters_json) - except json.JSONDecodeError as e: - raise GulagDBException(whoami(self) + "JSON parse error: " + e.msg) from e if 'rules' not in filters: raise GulagDBException(whoami(self) + "no 'rules' found in filters!") if 'groupOp' not in filters: raise GulagDBException(whoami(self) + "'groupOp' not found in filters!") + if filters['groupOp'] != 'AND' and filters['groupOp'] != 'OR': + raise GulagDBException(whoami(self) + + "invalid 'groupOp': " + filters['groupOp'] + ) + where_clause = "" for rule in filters['rules']: if 'field' not in rule: raise GulagDBException(whoami(self) + "'field' not found in rule!") @@ -138,6 +136,10 @@ class GulagDB: field_op_data = rule['field'] + " like '%" + rule['data'] + "%'" elif(rule['op'] == 'ne'): field_op_data = rule['field'] + " <>'" + rule['data'] + "'" + elif(rule['op'] == 'gt'): + field_op_data = rule['field'] + " > '" + rule['data'] + "'" + elif(rule['op'] == 'lt'): + field_op_data = rule['field'] + " < '" + rule['data'] + "'" if(field_op_data == None): raise GulagDBException(whoami(self) + "invalid rule-op: " + rule['op']) if(len(filters['rules']) == 1 or len(where_clause) == 0): diff --git a/app/Resources.py b/app/Resources.py index affe269..7f95c0e 100644 --- a/app/Resources.py +++ b/app/Resources.py @@ -1,6 +1,7 @@ from flask import request from flask_restful import Resource, abort, reqparse from Gulag import GulagException +import json class GulagResource(Resource): gulag = None @@ -36,7 +37,7 @@ class ResMailboxes(GulagResource): try: return self.gulag.get_mailboxes() except GulagException as e: - abort(500, message=e.message) + abort(400, message=e.message) class ResMailbox(GulagResource): def get(self,id): @@ -44,8 +45,14 @@ class ResMailbox(GulagResource): class ResQuarMails(GulagResource): def get(self): + args = request.args.to_dict() + if 'filters' in args: + try: + args['filters'] = json.loads(args['filters']) + except json.JSONDecodeError as e: + abort(400, message=whoami(self) + "JSON parse error: " + e.msg) try: - return self.gulag.get_quarmails(request.args.to_dict()) + return self.gulag.get_quarmails(args) except GulagException as e: abort(400, message=e.message) diff --git a/client/python/libgulag.py b/client/python/libgulag.py index a56011b..8df4056 100644 --- a/client/python/libgulag.py +++ b/client/python/libgulag.py @@ -31,7 +31,7 @@ class GulagClient: def get_quarmails(self,args): if 'filters' in args: try: - # jqgrid-style filters must be JSON-encoded + # jqgrid-style filters must be JSON-encoded before transport args['filters'] = json.dumps(args['filters']) except TypeError as e: raise GulagClientException(e.__str__) diff --git a/client/python/libgulag_test.py b/client/python/libgulag_test.py index f4b6a82..e1dd2f2 100644 --- a/client/python/libgulag_test.py +++ b/client/python/libgulag_test.py @@ -6,17 +6,27 @@ try: 'api_key': 'NotImplemented' }) quarmails = gulag.get_quarmails({ - 'filters': {"groupOp":"AND","rules":[ - {"field":"uri_count","op":"eq","data":"2"} + 'filters': {"groupOp":"OR","rules":[ + # {"field":"uri_count","op":"eq","data":"2"}, + # {"field":"attach_count","op":"ne","data":"0"} + {"field":"uri_count","op":"lt","data":"2"} ]}, - 'rfc822_message': 'ja, ich will', - 'query_limit': 2 + #'rfc822_message': 'ja, ich will', + #'query_limit': 2, + #'query_offset': 1 }) for qm in quarmails['quarmails']: print( "ID: " + str(qm['id']) + "\n Subject: " + qm['hdr_subject'] + "\n ctime: " + qm['ctime'] + + "\n attach_count: " + str(qm['attach_count']) + + "\n uri_count: " + str(qm['uri_count']) + + "\n imap_uid: " + str(qm['imap_uid']) + + "\n env_rcpt: " + qm['env_rcpt'] + + "\n env_from: " + qm['env_from'] + + "\n hdr_from: " + qm['hdr_from'] + + "\n hdr_msgid: " + str(qm['hdr_msgid']) ) except libgulag.GulagClientException as e: print("ERROR: " + e.message) diff --git a/docker/gulag-server/debian/Dockerfile b/docker/gulag-server/debian/Dockerfile index b2ee86d..49cea62 100644 --- a/docker/gulag-server/debian/Dockerfile +++ b/docker/gulag-server/debian/Dockerfile @@ -10,7 +10,7 @@ RUN set -ex ; \ uwsgi-plugin-python3 python3-setuptools python3-flask \ python3-flask-restful python3-mysql.connector \ uwsgi uwsgi-plugin-python3 procps net-tools \ - python3-pip libmagic1 \ + python3-pip libmagic1 python3-ssdeep \ && pip3 install python-magic RUN /bin/mkdir /config /socket /app