too much :)

app/Entities.py

@@ -1,13 +1,56 @@
 import re
 
+class MailrelayException(Exception):
+  message = None
+  def __init__(self,message):
+    self.message = message
+
+class Mailrelay:
+  id = None
+  smtp_server = None
+  smtp_port = None
+  smtp_security = None
+  smtp_user = None
+  smtp_pass = None
+  comment = None
+  href = None
+
+  def __init__(self,mr_ref):
+    if 'id' not in mr_ref:
+      raise MailrelayException("'id' is mandatory!")
+    self.id = mr_ref['id']
+    if 'smtp_server' not in mr_ref:
+      raise MailrelayException("'smtp_server' is mandatory!")
+    self.smtp_server = mr_ref['smtp_server']
+    if 'smtp_security' in mr_ref:
+      if re.match("^(plain|starttls|tls)$",mr_ref['smtp_security']) is not None:
+        self.smtp_security = mr_ref['smtp_security']
+      else:
+        raise MailrelayException('smtp_security: {} is invalid! '+
+          'Valid values: plain,starttls,tls'.format(mr_ref['smtp_security'])
+        )
+    else:
+      raise MailrelayException("'smtp_security' is a mandatory!")
+    if 'smtp_port' in mr_ref:
+      self.smtp_port = mr_ref['smtp_port']
+    if 'smtp_user' not in mr_ref:
+      raise MailrelayException("'smtp_user' is mandatory!")
+    self.smtp_user = mr_ref['smtp_user']
+    if 'smtp_pass' not in mr_ref:
+      raise MailrelayException("'smtp_pass' is mandatory!")
+    self.smtp_pass = mb_ref['imap_pass']
+    if 'comment' in mr_ref:
+      self.comment = mr_ref['comment']
+    if 'href' in mr_ref:
+      self.href = mr_ref['href']
+
 class MailboxException(Exception):
   message = None
   def __init__(self,message):
     self.message = message
 
 class Mailbox:
-  email_address = None
+  id = None
-  name = None
   imap_server = None
   imap_port = None
   imap_security = None
@@ -16,13 +59,14 @@ class Mailbox:
   imap_mailbox = None
   imap_mailbox_fp = None
   imap_separator = None
+  mailrelay_id = None
   comment = None
   href = None
 
   def __init__(self,mb_ref):
-    if 'email_address' not in mb_ref:
+    if 'id' not in mb_ref:
-      raise MailboxException("'email_address' is mandatory!")
+      raise MailboxException("'id' is mandatory!")
-    self.email_address = mb_ref['email_address']
+    self.id = mb_ref['id']
     if 'name' not in mb_ref:
       raise MailboxException("'name' is mandatory!")
     self.name = mb_ref['name']
@@ -85,6 +129,7 @@ class QuarMail:
   uri_count = None
   source_id = None
   ssdeep = None
+  release_time = None
 
   def __init__(self,qm_ref):
     if 'id' not in qm_ref:
@@ -135,6 +180,8 @@ class QuarMail:
     if 'ssdeep' not in qm_ref:
       raise QuarMailException("'ssdeep' is mandatory!")
     self.ssdeep = qm_ref['ssdeep']
+    if 'release_time' not in qm_ref:
+      raise QuarMailException("'release_time' is mandatory!")
 
 class AttachmentException(Exception):
   message = None

app/Gulag.py

@@ -57,28 +57,37 @@ class Gulag:
     try:
       self.db = GulagDB(self.config['db'],self.config['uri_prefixes'])
       self.fields['Mailboxes'] = self.db.get_fields('Mailboxes')
+      self.fields['Mailrelays'] = self.db.get_fields('Mailrelays')
       self.fields['QuarMails'] = self.db.get_fields('QuarMails')
       self.fields['Attachments'] = self.db.get_fields('Attachments')
     except GulagDBException as e:
       logging.warning(whoami(self) + e.message)
       raise GulagException(whoami(self) + e.message) from e
-    logging.info('Gulag core initialized by ' + os.path.basename(__file__))
 
-  def check_fields(self,fields_target,args):
+  def check_filters(self,fields_target,filters):
     if fields_target not in self.fields:
-      raise GulagException(
+      raise GulagBadInputException(
         whoami(self) + fields_target + " not found in Gulag.fields!"
       )
-    for arg in args:
+    if 'rules' not in filters:
-      if(arg == 'query_offset' or arg == 'query_limit'
+      raise GulagBadInputException(whoami(self) +
-         or arg == 'sort_index' or arg == 'sort_order'
+        "no 'rules' found in filters!"
-         or arg == 'rfc822_message' or arg == 'filters'
+      )
-         or arg in self.db.vcols):
+    if 'groupOp' not in filters:
+      raise GulagBadInputException(whoami(self) +
+        "'groupOp' not found in filters!"
+      )
+    if filters['groupOp'] != 'AND' and filters['groupOp'] != 'OR':
+      raise GulagBadInputException(whoami(self) +
+        "invalid 'groupOp': " + filters['groupOp']
+      )
+    # {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]}
+    for rule in filters['rules']:
+      if(rule['field'] in self.db.vcols):
         continue
-      if arg not in self.fields[fields_target]:
+      if rule['field'] not in self.fields[fields_target]:
-        raise GulagException(
+        raise GulagBadInputException(whoami(self) +
-          whoami(self) + arg + " is not a valid field of "
+          rule['field'] + " is not a valid field of " + fields_target + "!"
-          + fields_target + "!"
         )
 
   # Iterate through all mailboxes, extract metadata
@@ -86,12 +95,14 @@ class Gulag:
   def import_quarmails(self):
     for mailbox in self.db.get_mailboxes():
       imap_mb = None
+      messages = []
       try:
         imap_mb = IMAPmailbox(mailbox)
+        messages = imap_mb.get_unseen_messages()
       except IMAPmailboxException as e:
         logging.warning(whoami(self) + e.message)
         continue
-      for unseen in imap_mb.get_unseen_messages():
+      for unseen in messages:
         quarmail_ids = []
         attachments = []
         uris = {}
@@ -162,7 +173,9 @@ class Gulag:
           except GulagDBException as e:
             logging.warn(whoami(self) + e.message)
             raise GulagException(whoami(self) + e.message) from e
-          logging.info(whoami(self) + "QuarMail (%s) imported" % (quarmail_id))
+          logging.info(whoami(self) +
+            "QuarMail(%s)@Mailbox(%s) imported" % (quarmail_id,mailbox['id'])
+          )
           quarmail_ids.append(quarmail_id)
         # Ende for rcpts
         # Alle MIME-Parts durchiterieren und Attachments
@@ -242,7 +255,8 @@ class Gulag:
   def get_quarmails(self,args):
     qms_db = None
     try:
-      self.check_fields('QuarMails',args)
+      if 'filters' in args:
+        self.check_filters('QuarMails',args['filters'])
       qms_db = self.db.get_quarmails(args)
     except GulagDBBadInputException as e:
       raise GulagBadInputException(whoami(self) + e.message) from e
@@ -321,6 +335,20 @@ class Gulag:
       logging.warning(whoami(self) + e.message)
       raise GulagException(whoami(self) + e.message) from e
 
+  def release_quarmail(self,args):
+    try:
+      quarmail = self.get_quarmail({
+        "quarmail_id": args['quarmail_id'],
+        "rfc822_message": True
+      })
+      #
+      # TODO: re-send quarmail to original env_rcpt
+      # TODO: self.delete_quarmail() if arg['purge']
+    except GulagNotFoundException as e:
+      raise GulagNotFoundException(whoami(self) + e.message) from e
+    except GulagException as e:
+      raise GulagException(whoami(self) + e.message) from e
+
   def delete_quarmail(self, args):
     qm_db = None
     try:
@@ -610,6 +638,3 @@ class Gulag:
       imap_mb.close()
     except IMAPmailboxException as e:
       raise GulagException(whoami(self) + e.message) from e
-
-  def release_quarmail(self,args):
-    pass

app/GulagDB.py

@@ -2,7 +2,8 @@ import mysql.connector as mariadb
 from Entities import(
   Mailbox,MailboxException,QuarMail,
   QuarMailException,Attachment,
-  AttachmentException,URI,URIException
+  AttachmentException,URI,URIException,
+  Mailrelay,MailrelayException
 )
 from GulagUtils import whoami
 
@@ -47,7 +48,7 @@ class GulagDB:
     except mariadb.Error as e:
       raise GulagDBException(whoami(self) + str(e.msg)) from e
     self.uri_prefixes = uri_prefixes
-    # virtual columns cannot not be stated in where-clause
+    # virtual columns
     self.vcols['attach_count'] = {}
     self.vcols['uri_count'] = {}
 
@@ -62,7 +63,7 @@ class GulagDB:
       data = cursor.fetchall()
       if not data:
         raise GulagDBNotFoundException(whoami(self)
-          + "describe " + table_name + " failed: got no fields!"
+          + "describe ''" + table_name + "'' failed: got no fields!"
         )
       desc = cursor.description
       cursor.close()
@@ -81,60 +82,32 @@ class GulagDB:
         int(args['query_offset'])
       except ValueError:
         raise GulagDBBadInputException(whoami(self) +
-          "query_offset must be numeric!"
+          "'query_offset' must be numeric!"
         )
+      try:
+        int(args['query_limit'])
+      except ValueError:
+        raise GulagDBBadInputException(whoami(self) +
+          "'query_limit' must be numeric!"
+        )
+      return "limit "+args['query_offset']+","+args['query_limit']
+    elif('query_offset' in args and 'query_limit' not in args):
+      raise GulagDBBadInputException(whoami(self) +
+        "'query_offset' without 'query_limit' is useless!"
+      )
+    elif('query_limit' in args and 'query_offset' not in args):
       try:
         int(args['query_limit'])
       except ValueError:
         raise GulagDBBadInputException(whoami(self) +
           "query_limit must be numeric!"
         )
-      return "limit "+args['query_offset']+","+args['query_limit']
-    elif('query_offset' in args and 'query_limit' not in args):
-      raise GulagDBBadInputException(whoami(self) +
-        "query_offset without query_limit is useless!"
-      )
-    elif('query_limit' in args and 'query_offset' not in args):
-      try:
-        int(args['query_limit'])
-      except ValueError:
-        raise GulagDBBadInputException(whoami(self) + "query_limit must be numeric!")
       return "limit " + args['query_limit']
     else:
       return ""
 
-  def get_where_clause(self,args):
-    where_clause = ""
-    cnt = 0
-    for arg in args:
-      if(arg == 'query_offset' or arg == 'query_limit'
-         or arg == 'sort_index' or arg == 'sort_order'
-         or arg == 'rfc822_message'):
-        continue
-      if(cnt == 0):
-        if arg in self.vcols:
-          where_clause += "having " + arg + "='" + args[arg] + "' "
-        else:
-          where_clause += "where " + arg + "='" + args[arg] + "' "
-      else:
-        where_clause += "and " + arg + "='" + args[arg] + "' "
-      cnt += 1
-    return where_clause
-
   def get_where_clause_from_filters(self,filters):
     # {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]}
-    if 'rules' not in filters:
-      raise GulagDBBadInputException(whoami(self) +
-        "no 'rules' found in filters!"
-      )
-    if 'groupOp' not in filters:
-      raise GulagDBBadInputException(whoami(self) +
-        "'groupOp' not found in filters!"
-      )
-    if filters['groupOp'] != 'AND' and filters['groupOp'] != 'OR':
-      raise GulagDBBadInputException(whoami(self) +
-        "invalid 'groupOp': " + filters['groupOp']
-      )
     where_clause = ""
     for rule in filters['rules']:
       if 'field' not in rule:
@@ -177,6 +150,65 @@ class GulagDB:
         where_clause += " " + filters['groupOp'] + " " + field_op_data
     return where_clause
 
+  def add_mailrelay(self,args):
+    pass
+
+  def delete_mailrelay(self,mailrelay_id):
+    pass
+
+  def get_mailrelays(self):
+    try:
+      cursor = self.conn.cursor()
+      cursor.execute("select * from Mailrelays;")
+      results = []
+      data = cursor.fetchall()
+      if not data:
+        return results
+      desc = cursor.description
+      for tuple in data:
+        dict = {}
+        for (name, value) in zip(desc, tuple):
+          dict[name[0]] = value
+        dict['href'] = self.uri_prefixes['mailrelays'] + dict['id']
+        try:
+          results.append(Mailrelay(dict).__dict__)
+        except MailboxException as e:
+          print("MailrelayException: " + e.message)
+          continue
+      return results
+    except mariadb.Error as e:
+      raise GulagDBException(whoami(self) + str(e.msg)) from e
+
+  def get_mailrelay(self,mailbox_id):
+    try:
+      cursor = self.conn.cursor()
+      cursor.execute(
+        "select * from Mailrelays where id='" + mailrelay_id + "' limit 1;"
+      )
+      data = cursor.fetchall()
+      if not data:
+        raise GulagDBNotFoundException(whoami(self)
+          + "Mailrelay '" + mailrelay_id + "' does not exist!"
+        )
+      desc = cursor.description
+      tuple = data[0]
+      dict = {}
+      for (name, value) in zip(desc, tuple):
+        dict[name[0]] = value
+      dict['href'] = self.uri_prefixes['mailrelays'] + dict['id']
+      try:
+        return Mailrelay(dict).__dict__
+      except MailrelayException as e:
+        raise GulagDBException(whoami(self) + e.message) from e
+    except mariadb.Error as e:
+      raise GulagDBException(whoami(self) + str(e.msg)) from e
+
+  def add_mailbox(self,args):
+    pass
+
+  def delete_mailbox(self,mailbox_id):
+    pass
+
   def get_mailboxes(self):
     try:
       cursor = self.conn.cursor()
@@ -190,7 +222,7 @@ class GulagDB:
         dict = {}
         for (name, value) in zip(desc, tuple):
           dict[name[0]] = value
-        dict['href'] = self.uri_prefixes['mailboxes'] + dict['email_address']
+        dict['href'] = self.uri_prefixes['mailboxes'] + dict['id']
         try:
           results.append(Mailbox(dict).__dict__)
         except MailboxException as e:
@@ -204,7 +236,7 @@ class GulagDB:
     try:
       cursor = self.conn.cursor()
       cursor.execute(
-        "select * from Mailboxes where email_address='" + mailbox_id + "' limit 1;"
+        "select * from Mailboxes where id='" + mailbox_id + "' limit 1;"
       )
       data = cursor.fetchall()
       if not data:
@@ -216,7 +248,7 @@ class GulagDB:
       dict = {}
       for (name, value) in zip(desc, tuple):
         dict[name[0]] = value
-      dict['href'] = self.uri_prefixes['mailboxes'] + dict['email_address']
+      dict['href'] = self.uri_prefixes['mailboxes'] + dict['id']
       try:
         return Mailbox(dict).__dict__
       except MailboxException as e:
@@ -260,8 +292,6 @@ class GulagDB:
       where_clause = ""
       if 'filters' in args:
         where_clause = self.get_where_clause_from_filters(args['filters'])
-      else:
-        where_clause = self.get_where_clause(args)
       cursor = self.conn.cursor()
       query = "select *,(select count(*) from QuarMail2Attachment"
       query += " where QuarMails.id=QuarMail2Attachment.quarmail_id) as attach_count,"

app/GulagMailbox.py

@@ -11,7 +11,7 @@ class IMAPmailboxException(Exception):
     self.message = str(message)
 
 class IMAPmailbox:
-  email_address = None
+  id = None
   imap_server = None
   imap_user = None
   imap_pass = None
@@ -19,7 +19,7 @@ class IMAPmailbox:
   mailbox = None
 
   def __init__(self, mb_ref):
-    self.email_address = mb_ref['email_address']
+    self.id = mb_ref['id']
     self.imap_server = mb_ref['imap_server']
     self.imap_user = mb_ref['imap_user']
     self.imap_pass = mb_ref['imap_pass']
@@ -54,8 +54,9 @@ class IMAPmailbox:
     for uid in data[0].split():
       rv, data = self.mailbox.uid('FETCH', uid, '(RFC822)')
       if rv != 'OK':
-        print("ERROR getting message", str(uid))
+        raise IMAPmailboxException(whoami(self) +
-        continue
+          str(data) + ", IMAP_UID: " + str(uid)
+        )
       results.append({
         'imap_uid': uid,
         'msg': data[0][1]
@@ -114,14 +115,13 @@ class IMAPmailbox:
         else:
           # not encoded
           part_fn = part_fn[0][0]
-        print("C-T-E: " + str(part['Content-Transfer-Encoding']))
         if(part_fn == filename):
           return part.get_payload(decode=False)
       # End if part.get_filename()
     # End msg.walk() loop
     raise IMAPmailboxException(whoami(self) +
       "Attachment ("+ str(filename) +")@IMAP UID(" + str(imap_uid) + ")@"
-      + str(self.email_address) + " not found!"
+      + str(self.id) + " not found!"
     )
 
   def get_main_parts(self,imap_uid):
@@ -134,5 +134,5 @@ class IMAPmailbox:
     if(len(mparts) > 0):
       return mparts
     raise IMAPmailboxException(whoami(self) +
-      "IMAP_UID(" + str(imap_uid)+")@"+str(self.email_address)+" has no main parts!"
+      "IMAP_UID(" + str(imap_uid)+")@"+str(self.id)+" has no main parts!"
     )

app/gulag_helpers.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import argparse,sys,os,time,signal
+import argparse,sys,os,time,signal,logging
 from Gulag import Gulag,GulagException
 
 parser = argparse.ArgumentParser()
@@ -16,13 +16,14 @@ if(importer_pid == 0):
   except GulagException as e:
     print(e.message)
     sys.exit(1)
+  logging.info("Gulag-Importer: starting")
   while True:
     try:
       gulag.import_quarmails()
     except GulagException as e:
-      print("Importer-Exception: " + e.message, file=sys.stderr)
+      logging.error("Gulag-Importer-Exception: " + e.message)
     except:
-      print("Importer-Exception: " + str(sys.exc_info()),file=sys.stderr)
+      logging.error("Gulag-Importer-Exception: " + str(sys.exc_info()))
     time.sleep(gulag.config['importer']['interval'])
 
 cleaner_pid = os.fork()
@@ -31,28 +32,27 @@ if(cleaner_pid == 0):
   try:
     gulag = Gulag(args.config)
   except GulagException as e:
-    print(e.message)
+    logging.info("Gulag-Cleaner-Exception: " + e.message)
     sys.exit(1)
+  logging.info("Gulag-Cleaner: starting")
   while True:
     try:
       gulag.cleanup_quarmails()
     except GulagException as e:
-      print("Cleaner-Exception: " + e.message)
+      logging.info("Cleaner-Exception: " + e.message)
     except:
-      print("Cleaner-Exception: " + str(sys.exc_info()),file=sys.stderr)
+      logging.info("Cleaner-Exception: " + str(sys.exc_info()))
     time.sleep(gulag.config['cleaner']['interval'])
 
 # Parent
 child_pids.append(importer_pid)
 child_pids.append(cleaner_pid)
 try:
-  print("Entered helpers main loop...")
   while True:
     time.sleep(10)
 except:
-  print("Helpers MAIN-EXCEPTION: " + str(sys.exc_info()))
+  logging.info("Helpers MAIN-EXCEPTION: " + str(sys.exc_info()))
   # Destroy childs
   for child_pid in child_pids:
-    print("Killing child pid: %s", child_pid)
+    logging.info("Helpers parent: Killing child pid: %s", child_pid)
     os.kill(child_pid, signal.SIGTERM)
-

app/gulag_server.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import argparse,sys
+import argparse,sys,logging
 from flask import Flask
 from flask_restful import Api
 from Gulag import Gulag,GulagException
@@ -8,7 +8,7 @@ from Resources import (ResRoot,ResMailboxes,
   ResQuarMails,ResQuarMail,ResQuarMailAttachments,
   ResQuarMailAttachment,ResAttachments,ResAttachment,
   ResRspamd2Mailbox,ResQuarMailURIs,ResQuarMailURI,
-  ResMailradar2Mailbox
+  ResMailradar2Mailbox,ResQuarMailRelease
 )
 parser = argparse.ArgumentParser()
 parser.add_argument('--config', required=True, help="Path to config file")
@@ -19,6 +19,7 @@ try:
     gulag = Gulag(args.config)
   except GulagException as e:
     raise Exception(e.message) from e
+  logging.info("Gulag-Server: starting")
   app = Flask(__name__)
   # https://github.com/flask-restful/flask-restful/issues/780#issuecomment-434588559
   app.config['ERROR_404_HELP'] = False
@@ -39,6 +40,10 @@ try:
     '/api/v1/quarmails/<int:quarmail_id>',
     resource_class_kwargs={'gulag_object': gulag}
   )
+  api.add_resource(ResQuarMailRelease,
+    '/api/v1/quarmails/<int:quarmail_id>/release',
+    resource_class_kwargs={'gulag_object': gulag}
+  )
   api.add_resource(ResQuarMailAttachments,
     '/api/v1/quarmails/<int:quarmail_id>/attachments',
     resource_class_kwargs={'gulag_object': gulag}
@@ -80,4 +85,4 @@ try:
     gulag.db.close()
     sys.exit(0)
 except:
-  print("MAIN-EXCEPTION: " + str(sys.exc_info()))
+  logging.error("Gulag-Server-Exception: " + str(sys.exc_info()))

bindings/python/libgulag_test.py

@@ -9,7 +9,7 @@ try:
     'filters': {"groupOp":"OR","rules":[
     #  {"field":"uri_count","op":"eq","data":"2"},
     #  {"field":"attach_count","op":"ne","data":"0"}
-      {"field":"uri_count","op":"lt","data":"2"}
+    #  {"field":"uri_count","op":"gt","data":"2"}
     ]},
     #'rfc822_message': 'ja, ich will',
     #'query_limit': 2,

config/gulag-config.json

@@ -24,6 +24,7 @@
   },
   "uri_prefixes": {
     "root": "http://127.0.0.1:9090/api/v1/",
+    "mailrelays": "http://127.0.0.1:9090/api/v1/mailrelays/",
     "mailboxes": "http://127.0.0.1:9090/api/v1/mailboxes/",
     "quarmails": "http://127.0.0.1:9090/api/v1/quarmails/",
     "attachments": "http://127.0.0.1:9090/api/v1/attachments/"

db/gulag.sql

@@ -3,7 +3,7 @@ create database Gulag;
 use Gulag;
 
 create table Mailrelays(
-  id varchar(64) not null primary key,
+  id varchar(128) not null primary key,
   smtp_server varchar(256) default '127.0.0.1' collate 'ascii_general_ci',
   smtp_port smallint unsigned not null default 25,
   smtp_security varchar(32) not null default 'plain' collate 'ascii_general_ci',
@@ -11,10 +11,11 @@ create table Mailrelays(
   smtp_pass varchar(1024) default null collate 'ascii_general_ci',
   comment varchar(256) default null
 )ENGINE = InnoDB;
+insert into Mailrelays (id) values ('default_local_mailrelay');
 
 create table Mailboxes(
-  email_address varchar(767) not null primary key collate 'ascii_general_ci',
+  id varchar(128) not null primary key,
-  name varchar(256) not null,
+  name varchar(512) not null,
   imap_server varchar(256) not null default '127.0.0.1' collate 'ascii_general_ci',
   imap_port smallint unsigned not null default 143,
   imap_security varchar(32) not null default 'plain' collate 'ascii_general_ci',
@@ -23,10 +24,12 @@ create table Mailboxes(
   imap_mailbox varchar(256) not null default 'INBOX',
   imap_mailbox_fp varchar(256) not null default 'false-positives',
   imap_separator varchar(4) not null default '/',
+  mailrelay_id varchar(128) not null,
+  foreign key (mailrelay_id) references Mailrelays (id) on update cascade on delete restrict,
   comment varchar(256) default null
 )ENGINE = InnoDB;
-insert into Mailboxes (email_address,name,imap_user,imap_pass)
+insert into Mailboxes (id,name,imap_user,imap_pass,mailrelay_id)
-  values('quarantine@example.org','E-Mail inbound quarantine','quarantine','quarantine_secure_password');
+  values('quarantine@example.org','E-Mail inbound quarantine','quarantine','quarantine_secure','default_local_mailrelay');
 
 create table Sources (
   id varchar(32) not null collate 'ascii_general_ci' primary key
@@ -37,7 +40,7 @@ insert into Sources (id) values ('mailradar');
 
 create table QuarMails (
   id int unsigned auto_increment primary key,
-  ctime TIMESTAMP,
+  ctime TIMESTAMP default CURRENT_TIMESTAMP,
   mx_queue_id varchar(64) not null collate 'ascii_general_ci',
   env_from varchar(256) not null,
   env_rcpt varchar(256) not null,
@@ -47,13 +50,14 @@ create table QuarMails (
   hdr_msgid varchar(512) default null,
   hdr_date varchar(128) default null collate 'ascii_general_ci',
   cf_meta TEXT default null,
-  imap_uid int unsigned not null,
-  mailbox_id varchar(256) not null collate 'ascii_general_ci',
-  foreign key (mailbox_id) references Mailboxes (email_address) on update cascade on delete cascade,
-  source_id varchar(32) not null collate 'ascii_general_ci',
-  foreign key (source_id) references Sources (id) on update cascade on delete cascade,
   msg_size int unsigned not null,
-  ssdeep varchar(592) not null collate 'ascii_general_ci'
+  ssdeep varchar(592) not null collate 'ascii_general_ci',
+  release_time TIMESTAMP default 0,
+  imap_uid int unsigned not null,
+  mailbox_id varchar(128) not null,
+  foreign key (mailbox_id) references Mailboxes (id) on update cascade on delete restrict,
+  source_id varchar(32) not null collate 'ascii_general_ci',
+  foreign key (source_id) references Sources (id) on update cascade on delete restrict
 )ENGINE = InnoDB;
 
 create table Attachments (

gulag-openapi-2.0.yaml

@@ -8,12 +8,10 @@ info:
   license:
     name: Apache 2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html
-
 host: gulag.example.com
 basePath: "/api/v1"
 schemes:
   - https
-
 paths:
   /quarmails:
     get:
@@ -25,11 +23,6 @@ paths:
       produces:
       - application/json
       parameters:
-      - in: query
-        name: query_offset
-        description: number of records to skip for pagination
-        required: false
-        type: string
       - in: query
         name: query_limit
         description: number of records to retrieve
@@ -37,6 +30,11 @@ paths:
         format: int32
         minimum: 0
         required: false
+      - in: query
+        name: query_offset
+        description: number of records to skip for pagination
+        required: false
+        type: string
       - in: query
         name: sort_index
         description: field used to sort results
@@ -47,72 +45,6 @@ paths:
         description: order used to sort results
         type: string
         required: false
-      - in: query
-        name: id
-        description: unique id of a quarantined email
-        type: integer
-        format: int32
-        minimum: 0
-        required: false
-      - in: query
-        name: ctime
-        description: create timestamp of quarantined email
-        type: string
-        required: false
-      - in: query
-        name: mx_queueid
-        type: string
-        required: false
-      - in: query
-        name: env_from
-        type: string
-        description: RFC5321 envelope sender
-        required: false
-      - in: query
-        name: env_rcpt
-        type: string
-        description: RFC5321 envelope recipient
-        required: false
-      - in: query
-        name: hdr_cf
-        type: string
-        description: content scanner header
-        required: false
-      - in: query
-        name: hdr_from
-        type: string
-        description: RFC5322 From header
-        required: false
-      - in: query
-        name: hdr_subject
-        type: string
-        description: RFC5322 Subject header
-        required: false
-      - in: query
-        name: hdr_msgid
-        type: string
-        description: RFC5322 Message-ID header
-        required: false
-      - in: query
-        name: hdr_date
-        type: string
-        description: RFC5322 Date header
-        required: false
-      - in: query
-        name: cf_meta
-        type: string
-        description: content filter meta data
-        required: false
-      - in: query
-        name: mailbox_id
-        description: IMAP mailbox identifier, e.g. quarantine@example.org
-        type: string
-        required: false
-      - in: query
-        name: imap_uid
-        description: IMAP UID of a quarantined email
-        type: string
-        required: false
       - in: query
         name: rfc822_message
         type: string
@@ -122,7 +54,7 @@ paths:
         name: filters
         type: string
         required: false
-        description: jqgrid-style search filter
+        description: 'jqgrid-style filters, e.g. {"groupOp":"AND","rules":[{"field":"uri_count","op":"eq","data":"3"}]}'
       responses:
         200:
           description: search results matching criteria
@@ -159,6 +91,8 @@ paths:
             $ref: '#/definitions/QuarMail'
         400:
           description: bad input parameter
+        404:
+          description: not found
         500:
           description: server error
     delete:
@@ -201,6 +135,8 @@ paths:
               $ref: '#/definitions/Attachment'
         400:
           description: bad input parameter
+        404:
+          description: not found
         500:
           description: server error
 
@@ -262,6 +198,8 @@ paths:
               $ref: '#/definitions/URI'
         400:
           description: bad input parameter
+        404:
+          description: not found
         500:
           description: server error
 
@@ -310,6 +248,7 @@ definitions:
     - mailbox_id
     - imap_uid
     - msg_size
+    - ssdeep
     - href
     properties:
       id:
@@ -365,6 +304,10 @@ definitions:
       uri_count:
         type: integer
         description: number of uris
+      ssdeep:
+        type: string
+        description: Context triggered piecewise hash (CTPH)
+        example: '6:lWRUFiWwx5QHD2Q2/NNsj90YzrWPpsj7v:lWiEQHD2Q+sj90aKsjr'
       rfc822_message:
         type: string
         description: full RFC822 email message
@@ -378,6 +321,8 @@ definitions:
     - magic
     - mailbox_id
     - imap_uid
+    - ssdeep
+    - sha256
     - href
     properties:
       id:
@@ -409,9 +354,18 @@ definitions:
       imap_uid:
         type: integer
         description: IMAP unique id of the quarantined email
+        example: 12345
       data:
         type: string
         description: raw/encoded (see content_encoding) attachment payload
+      ssdeep:
+        type: string
+        description: Context Triggered Piecewise Hash (CTPH)
+        example: '6:lWRUFiWwx5QHD2Q2/NNsj90YzrWPpsj7v:lWiEQHD2Q+sj90aKsjr'
+      sha256:
+        type: string
+        description: SHA256 digest
+        example: '658cb334f4ab3d747e77fdfceaa1ff3c2477ccc8500c4d8f4552ac0471089b60'
   URI:
     type: object
     required: