Attachment hashing and other fixes

app/Gulag.py

@@ -3,6 +3,7 @@ import email,email.header,email.message
 from GulagDB import GulagDB,GulagDBException
 from GulagMailbox import IMAPmailbox,IMAPmailboxException
 from GulagUtils import whoami,extract_uris,extract_fqdn
+import ssdeep, hashlib
 
 class GulagException(Exception):
   message = None
@@ -154,6 +155,14 @@ class Gulag:
             else:
               # filename isn´t encoded
               filename = filename[0][0]
+            logging.info(whoami(self) +
+              "SSDEEP: " + ssdeep.hash(part.get_payload(decode=True))
+            )
+            logging.info(whoami(self) +
+              "SHA256 " + hashlib.sha256(
+                part.get_payload(decode=True)
+              ).hexdigest()
+            )
             attach_magic = None
             try:
               attach_magic = magic.from_buffer(part.get_payload(decode=True))
@@ -224,9 +233,7 @@ class Gulag:
       qms_db = self.db.get_quarmails(args)
     except(GulagException,GulagDBException) as e:
       logging.warning(whoami(self) + e.message)
-      raise GulagException(
+      raise GulagException(whoami(self) + e.message) from e
-        whoami(self) + e.message
-      ) from e
     if 'rfc822_message' not in args:
       return {
         'quarmails': qms_db,

app/GulagDB.py

@@ -5,7 +5,6 @@ from Entities import(
   AttachmentException,URI,URIException
 )
 from GulagUtils import whoami
-import json
 
 class GulagDBException(Exception):
   message = None
@@ -108,18 +107,17 @@ class GulagDB:
       cnt += 1
     return where_clause
 
-  def get_where_clause_from_filters(self,filters_json):
+  def get_where_clause_from_filters(self,filters):
     # {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]}
-    filters = None
-    where_clause = ""
-    try:
-      filters = json.loads(filters_json)
-    except json.JSONDecodeError as e:
-      raise GulagDBException(whoami(self) + "JSON parse error: " + e.msg) from e
     if 'rules' not in filters:
       raise GulagDBException(whoami(self) + "no 'rules' found in filters!")
     if 'groupOp' not in filters:
       raise GulagDBException(whoami(self) + "'groupOp' not found in filters!")
+    if filters['groupOp'] != 'AND' and filters['groupOp'] != 'OR':
+      raise GulagDBException(whoami(self) +
+        "invalid 'groupOp': " + filters['groupOp']
+      )
+    where_clause = ""
     for rule in filters['rules']:
       if 'field' not in rule:
         raise GulagDBException(whoami(self) + "'field' not found in rule!")
@@ -138,6 +136,10 @@ class GulagDB:
         field_op_data = rule['field'] + " like '%" + rule['data'] + "%'"
       elif(rule['op'] == 'ne'):
         field_op_data = rule['field'] + " <>'" + rule['data'] + "'"
+      elif(rule['op'] == 'gt'):
+        field_op_data = rule['field'] + " > '" + rule['data'] + "'"
+      elif(rule['op'] == 'lt'):
+        field_op_data = rule['field'] + " < '" + rule['data'] + "'"
       if(field_op_data == None):
         raise GulagDBException(whoami(self) + "invalid rule-op: " + rule['op'])
       if(len(filters['rules']) == 1 or len(where_clause) == 0):

app/Resources.py

@@ -1,6 +1,7 @@
 from flask import request
 from flask_restful import Resource, abort, reqparse
 from Gulag import GulagException
+import json
 
 class GulagResource(Resource):
   gulag = None
@@ -36,7 +37,7 @@ class ResMailboxes(GulagResource):
     try:
       return self.gulag.get_mailboxes()
     except GulagException as e:
-      abort(500, message=e.message)
+      abort(400, message=e.message)
 
 class ResMailbox(GulagResource):
   def get(self,id):
@@ -44,8 +45,14 @@ class ResMailbox(GulagResource):
 
 class ResQuarMails(GulagResource):
   def get(self):
+    args = request.args.to_dict()
+    if 'filters' in args:
       try:
-      return self.gulag.get_quarmails(request.args.to_dict())
+        args['filters'] = json.loads(args['filters'])
+      except json.JSONDecodeError as e:
+        abort(400, message=whoami(self) + "JSON parse error: " + e.msg)
+    try:
+      return self.gulag.get_quarmails(args)
     except GulagException as e:
       abort(400, message=e.message)
 

client/python/libgulag.py

@@ -31,7 +31,7 @@ class GulagClient:
   def get_quarmails(self,args):
     if 'filters' in args:
       try:
-        # jqgrid-style filters must be JSON-encoded
+        # jqgrid-style filters must be JSON-encoded before transport
         args['filters'] = json.dumps(args['filters'])
       except TypeError as e:
         raise GulagClientException(e.__str__)

client/python/libgulag_test.py

@@ -6,17 +6,27 @@ try:
     'api_key': 'NotImplemented'
   })
   quarmails = gulag.get_quarmails({
-    'filters': {"groupOp":"AND","rules":[
+    'filters': {"groupOp":"OR","rules":[
-      {"field":"uri_count","op":"eq","data":"2"}
+    #  {"field":"uri_count","op":"eq","data":"2"},
+    #  {"field":"attach_count","op":"ne","data":"0"}
+      {"field":"uri_count","op":"lt","data":"2"}
     ]},
-    'rfc822_message': 'ja, ich will',
+    #'rfc822_message': 'ja, ich will',
-    'query_limit': 2
+    #'query_limit': 2,
+    #'query_offset': 1
   })
   for qm in quarmails['quarmails']:
     print(
       "ID: " + str(qm['id'])
       + "\n  Subject: " + qm['hdr_subject']
       + "\n  ctime: " + qm['ctime']
+      + "\n  attach_count: " + str(qm['attach_count'])
+      + "\n  uri_count: " + str(qm['uri_count'])
+      + "\n  imap_uid: " + str(qm['imap_uid'])
+      + "\n  env_rcpt: " + qm['env_rcpt']
+      + "\n  env_from: " + qm['env_from']
+      + "\n  hdr_from: " + qm['hdr_from']
+      + "\n  hdr_msgid: " + str(qm['hdr_msgid'])
     )
 except libgulag.GulagClientException as e:
   print("ERROR: " + e.message)

docker/gulag-server/debian/Dockerfile

@@ -10,7 +10,7 @@ RUN set -ex ; \
       uwsgi-plugin-python3 python3-setuptools python3-flask \
       python3-flask-restful python3-mysql.connector \
       uwsgi uwsgi-plugin-python3 procps net-tools \
-      python3-pip libmagic1 \
+      python3-pip libmagic1 python3-ssdeep \
  && pip3 install python-magic
 
 RUN /bin/mkdir /config /socket /app