Merge pull request #2 from chillout2k/devel

Devel

README.md

@@ -1,38 +1,68 @@
 # ldap-acl-milter
+A lightweight, fast and thread-safe python3 [milter](http://www.postfix.org/MILTER_README.html) on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for basic Access Control (ACL) scenarios. The milter consumes policies from LDAP based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt).
+
+In the case, one already has a LDAP server running with the [amavis schema](https://www.ijs.si/software/amavisd/LDAP.schema.txt), the 'amavisWhitelistSender' attribute could be reused. The filtering direction (inbound or outbound) can be simply controlled by swapping the %from% and %rcpt% placeholders within the LDAP query template. Please have a look at the docker-compose.yml example.
+
+The connection to the LDAP server is always persistent: one LDAP-bind is shared among all milter-threads, which makes it more efficient due to less communication overhead (which also implies transport encryption with TLS). Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). On the one hand, this aproach eliminates network round trip times while reading from a UNIX-socket, and on the other, performance bottlenecks on a shared, centralized and (heavy) utilized LDAP server.
+
+### Deployment paradigm
+The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not really a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage.
 
 ### docker-compose.yml
+The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-capable MTA and OpenLDAP as local LDAP replica.
 
 ```
 version: '3'
 
 volumes:
   lam_socket:
+  openldap_spool:
+  openldap_socket:
 
 services:
+  openldap:
+    image: "your/favorite/openldap/image"
+    restart: unless-stopped
+    hostname: openldap
+    volumes:
+    - "./config/openldap:/etc/openldap:rw"
+    - "openldap_spool:/var/openldap-data:rw"
+    - "openldap_socket:/socket:rw"
+
   ldap-acl-milter:
-    image: "ldap-acl-milter/debian:19.02_devel"
+    depends_on:
+    - openldap
+    image: "ldap-acl-milter/debian:19.02_master"
     restart: unless-stopped
     environment:
-      LDAP_SERVER: ldap://ldap-slave.example.org:389
+      #LDAP_SERVER: ldap://ldap-slave.example.local:389
-      LDAP_BINDDN: uid=lam,ou=apps,dc=example,dc=org
+      LDAP_SERVER: ldapi:///socket//slapd//slapd
-      LDAP_BINDPW: TopSecret1!
+      LDAP_BINDDN: uid=lam,ou=applications,dc=example,dc=org
+      LDAP_BINDPW: TopSecret123!%&
       LDAP_BASE: ou=users,dc=example,dc=org
-      LDAP_QUERY: (&(mail=%rcpt%)(whitelistSender=%from%))
+      # This example LDAP query is for inbound filtering
-      # Socket default: /socket/ldap-acl-milter
+      # where the 'mail' attribute equals to the recipient
+      # and the 'amavisWhitelistSender' attribute the eligible sender
+      LDAP_QUERY: (&(mail=%rcpt%)(amavisWhitelistSender=%from%))
+      # Default: UNIX-socket located under /socket/ldap-acl-milter
+      # https://pythonhosted.org/pymilter/namespacemilter.html#a266a6e09897499d8b1ae0e20f0d2be73
       #MILTER_SOCKET: inet6:8020
-      MILTER_REJECT_MESSAGE: Rejected due to security policy violation
+      MILTER_REJECT_MESSAGE: Message rejected due to security policy
     hostname: ldap-acl-milter
     volumes:
     - "lam_socket:/socket/:rw"
+    - "openldap_socket:/socket/slapd:ro"
+
   postfix:
     depends_on:
     - ldap-acl-milter
-    image: "postfix/alpine/amd64"
+    image: "your/favorite/postfix/image"
     restart: unless-stopped
     hostname: postfix
     ports:
     - "25:25"
     volumes:
     - "./config/postfix:/etc/postfix:rw"
-    - "lam_socket:/socket/:rw"
+    - "lam_socket:/socket/ldap-acl-milter/:rw"
+    - "openldap_socket:/socket/slapd:ro"
 ```

app/ldap-acl-milter.py

@@ -1,11 +1,14 @@
 import Milter
-from ldap3 import Server,Connection,NONE,LDAPOperationResult
+from ldap3 import (
+  Server,ServerPool,Connection,NONE,LDAPOperationResult
+)
 import sys
 import os
 import logging
-from multiprocessing import Process,Queue
+import string
+import random
+from timeit import default_timer as timer
 
-log_queue = Queue(maxsize=32)
 g_milter_name = 'ldap-acl-milter'
 g_milter_socket = '/socket/' + g_milter_name
 g_milter_reject_message = 'Absender/Empfaenger passen nicht!'
@@ -22,12 +25,18 @@ logging.basicConfig(
 )
 
 class LdapAclMilter(Milter.Base):
-  def __init__(self):  # A new instance with each new connection.
+  # Each new connection is handled in an own thread
+  def __init__(self):
+    self.time_start = timer()
     self.id = Milter.uniqueID()
     self.ldap_conn = g_ldap_conn
     self.R = []
+    # https://stackoverflow.com/a/2257449
+    self.mconn_id = ''.join(
+      random.choice(string.ascii_lowercase + string.digits) for _ in range(8)
+    )
 
-  # Not registered callbacks
+  # Not registered/used callbacks
   @Milter.nocallback
   def connect(self, IPname, family, hostaddr):
     return self.CONTINUE
@@ -46,7 +55,9 @@ class LdapAclMilter(Milter.Base):
 
   def envfrom(self, mailfrom, *str):
     if mailfrom == '<>':
-      self.setreply('550','5.7.1','Envelope null-sender not allowed!')
+      ex = str(self.mconn_id + '/FROM Envelope null-sender not allowed!')
+      logging.error(ex)
+      self.setreply('550','5.7.1',ex)
       Milter.REJECT
     mailfrom = mailfrom.replace("<","")
     mailfrom = mailfrom.replace(">","")
@@ -54,54 +65,75 @@ class LdapAclMilter(Milter.Base):
     return Milter.CONTINUE
 
   def envrcpt(self, to, *str):
+    time_start = timer()
     to = to.replace("<","")
     to = to.replace(">","")
+    time_end = None
     try:
       query = g_ldap_query.replace("%rcpt%",to)
       query = query.replace("%from%", self.F)
       self.ldap_conn.search(g_ldap_base, query)
+      time_end = timer()
       if len(self.ldap_conn.entries) == 0:
-        self.R.append({"rcpt": to, "reason": g_milter_reject_message})
+        self.R.append({
-        self.setreply('550','5.7.1',g_milter_reject_message)
+          "rcpt": to, "reason": g_milter_reject_message,
+          "time_start":time_start, "time_end":time_end
+        })
+        self.setreply('550','5.7.1',
+          'Sender does not comply with recipients policy!'
+        )
+        logging.info(self.mconn_id + "/RCPT " + g_milter_reject_message)
         return Milter.REJECT
     except LDAPOperationResult as e:
-      self.log("LDAP Exception (envrcpt): " + str(e))
+      logging.warn(self.mconn_id + "/RCPT LDAP Exception (envrcpt): " + str(e))
-      self.setreply('451','4.7.1', str(e))
+      self.setreply('451','4.7.1',
+        'Service temporarily not available! Please try again later.'
+      )
       return Milter.TEMPFAIL
-    self.R.append({"rcpt": to, "reason":'pass'})
+    self.R.append({
+      "rcpt": to, "reason":'pass',"time_start":time_start,"time_end":time_end
+    })
     return Milter.CONTINUE
 
   def data(self):
-    # This callback is used only to log with queue-id
+    # A queue-id will be generated after the first accepted RCPT TO
+    # and therefore not available until DATA command
+    self.queue_id = self.getsymval('i')
+    try:
       for rcpt in self.R:
-      self.log(self.getsymval('i') +
+        duration = rcpt['time_end'] - rcpt['time_start']
+        logging.info(self.mconn_id + "/DATA " + self.queue_id +
           ": 5321.from=<" + self.F + "> 5321.rcpt=<" +
-        rcpt['rcpt'] + "> reason: " + rcpt['reason']
+          rcpt['rcpt'] + "> reason: " + rcpt['reason'] +
+          " duration: " + str(duration) + " sec."
         )
+    except:
+      ex = str(self.mconn_id + "/DATA " + self.queue_id +
+        ": Exception (data): " + sys.exc_info()
+      )
+      logging.warn(ex)
+      self.setreply('451','4.7.1', ex)
+      return Milter.TEMPFAIL
     return Milter.CONTINUE
 
   def eom(self):
-    # EOM will always be called - non-optional
+    # EOM is not optional and thus, always called by MTA
+    time_end = timer()
+    duration = time_end - self.time_start
+    logging.info(self.mconn_id + "/EOM " + self.queue_id +
+      " processing: " + str(duration) + " sec."
+    )
     return Milter.CONTINUE
 
   def abort(self):
-    # client disconnected prematurely
+    # Client disconnected prematurely
     return Milter.CONTINUE
 
   def close(self):
-    # always called, even when abort is called.  Clean up
+    # Always called, even when abort is called.
-    # any external resources here.
+    # Clean up any external resources here.
     return Milter.CONTINUE
 
-  def log(self,msg):
-    log_queue.put(msg)
-
-def logger_proc_loop():
-  while True:
-    qmsg = log_queue.get()
-    if not qmsg: break
-    logging.info(qmsg)
-
 if __name__ == "__main__":
   try:
     if 'LDAP_SERVER' not in os.environ:
@@ -124,7 +156,12 @@ if __name__ == "__main__":
       g_milter_socket = os.environ['MILTER_SOCKET']
     if 'MILTER_REJECT_MESSAGE' in os.environ:
       g_milter_reject_message = os.environ['MILTER_REJECT_MESSAGE']
+    #server_pool = ServerPool(None, pool_strategy='ROUND_ROBIN', active=False, exhaust=False)
     server = Server(g_ldap_server, get_info=NONE)
+    #server_pool.add(server)
+    #server2 = Server('ldap://ldap-master-zdf.zwackl.local:389', get_info=NONE)
+    #server_pool.add(server2)
+    #g_ldap_conn = Connection(server_pool,
     g_ldap_conn = Connection(server,
       g_ldap_binddn, g_ldap_bindpw,
       auto_bind=True, raise_exceptions=True,
@@ -135,18 +172,14 @@ if __name__ == "__main__":
     logging.error("LDAP Exception: " + str(e))
     sys.exit(1)
   try:
-    logger_proc = Process(target=logger_proc_loop)
-    logger_proc.start()
     timeout = 600
     # Register to have the Milter factory create instances of your class:
     Milter.factory = LdapAclMilter
     # Tell the MTA which features we use
     flags = Milter.ADDHDRS
     Milter.set_flags(flags)
-    logging.info("Startup " + g_milter_name + "@"+ g_milter_socket)
+    logging.info("Startup " + g_milter_name + "@socket: " + g_milter_socket)
     Milter.runmilter(g_milter_name,g_milter_socket,timeout,True)
-    log_queue.put(None)
-    logger_proc.join()
     logging.info("Shutdown " + g_milter_name)
   except:
     logging.error("MAIN-EXCEPTION: " + str(sys.exc_info()))