From f2d57629dc3abf2728557a1cad86b0636c5ff537 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:07:55 +0100 Subject: [PATCH 01/12] markup test --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 897e5ed..d9996f5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # ldap-acl-milter +A fast and lightweight python3 milter (on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter)) for Access ControL (ACL) scenarios. The milter consumes policies from a LDAP server based on custom queries with trivial templating (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt) support. + +The connection to the LDAP server is always persistent: one TCP-Session, one LDAP-bind -> less overhead ### docker-compose.yml From 8e3b83397a068227c67bd0169df9284f5357e706 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:16:23 +0100 Subject: [PATCH 02/12] docs, docs, docs --- app/ldap-acl-milter.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/app/ldap-acl-milter.py b/app/ldap-acl-milter.py index f2f218a..620e046 100644 --- a/app/ldap-acl-milter.py +++ b/app/ldap-acl-milter.py @@ -3,9 +3,10 @@ from ldap3 import Server,Connection,NONE,LDAPOperationResult import sys import os import logging +from timeit import default_timer as timer from multiprocessing import Process,Queue -log_queue = Queue(maxsize=32) +log_queue = Queue(maxsize=4) g_milter_name = 'ldap-acl-milter' g_milter_socket = '/socket/' + g_milter_name g_milter_reject_message = 'Absender/Empfaenger passen nicht!' @@ -23,6 +24,7 @@ logging.basicConfig( class LdapAclMilter(Milter.Base): def __init__(self): # A new instance with each new connection. + self.time_start = timer() self.id = Milter.uniqueID() self.ldap_conn = g_ldap_conn self.R = [] @@ -56,32 +58,50 @@ class LdapAclMilter(Milter.Base): def envrcpt(self, to, *str): to = to.replace("<","") to = to.replace(">","") + time_start = timer() + time_end = None try: query = g_ldap_query.replace("%rcpt%",to) query = query.replace("%from%", self.F) self.ldap_conn.search(g_ldap_base, query) + time_end = timer() if len(self.ldap_conn.entries) == 0: - self.R.append({"rcpt": to, "reason": g_milter_reject_message}) + self.R.append({ + "rcpt": to, "reason": g_milter_reject_message, + "time_start":time_start, "time_end":time_end + }) self.setreply('550','5.7.1',g_milter_reject_message) return Milter.REJECT except LDAPOperationResult as e: self.log("LDAP Exception (envrcpt): " + str(e)) self.setreply('451','4.7.1', str(e)) return Milter.TEMPFAIL - self.R.append({"rcpt": to, "reason":'pass'}) + self.R.append({ + "rcpt": to, "reason":'pass',"time_start":time_start,"time_end":time_end + }) return Milter.CONTINUE def data(self): # This callback is used only to log with queue-id - for rcpt in self.R: - self.log(self.getsymval('i') + - ": 5321.from=<" + self.F + "> 5321.rcpt=<" + - rcpt['rcpt'] + "> reason: " + rcpt['reason'] - ) + try: + for rcpt in self.R: + duration = rcpt['time_end'] - rcpt['time_start'] + self.log(self.getsymval('i') + + ": 5321.from=<" + self.F + "> 5321.rcpt=<" + + rcpt['rcpt'] + "> reason: " + rcpt['reason'] + + " duration: " + str(duration) + " sec." + ) + except: + self.log("Exception (data): " + str(sys.exc_info())) + self.setreply('451','4.7.1', str(sys.exc_info())) + return Milter.TEMPFAIL return Milter.CONTINUE def eom(self): # EOM will always be called - non-optional + time_end = timer() + duration = time_end - self.time_start + self.log(self.getsymval('i') + " processing: " + str(duration) + " sec.") return Milter.CONTINUE def abort(self): From 21aaaefb6757bd5edc2e59653f0a439aa59e6ca8 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:17:07 +0100 Subject: [PATCH 03/12] docs, docs, docs --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9996f5..fc5ddc1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # ldap-acl-milter -A fast and lightweight python3 milter (on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter)) for Access ControL (ACL) scenarios. The milter consumes policies from a LDAP server based on custom queries with trivial templating (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt) support. +A fast and lightweight and thread-safe python3 milter on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for Access ControL (ACL) scenarios. The milter consumes policies from a LDAP server based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). Please have a look at the docker-compose.yml example. + +So, if you already have a LDAP server running with e.g. amavis-schema, you may reuse the 'amavisWhitelistSender'/'amavisBlacklistSender' attributes. Please have a look at the docker-compose.yml example. The connection to the LDAP server is always persistent: one TCP-Session, one LDAP-bind -> less overhead @@ -20,7 +22,7 @@ services: LDAP_BINDDN: uid=lam,ou=apps,dc=example,dc=org LDAP_BINDPW: TopSecret1! LDAP_BASE: ou=users,dc=example,dc=org - LDAP_QUERY: (&(mail=%rcpt%)(whitelistSender=%from%)) + LDAP_QUERY: (&(mail=%rcpt%)(amavisWhitelistSender=%from%)) # Socket default: /socket/ldap-acl-milter # MILTER_SOCKET: inet6:8020 MILTER_REJECT_MESSAGE: Rejected due to security policy violation From f27e87c833fee2e6b0d15a246568d537a80c3158 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:17:38 +0100 Subject: [PATCH 04/12] . --- app/ldap-acl-milter.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/app/ldap-acl-milter.py b/app/ldap-acl-milter.py index f2f218a..620e046 100644 --- a/app/ldap-acl-milter.py +++ b/app/ldap-acl-milter.py @@ -3,9 +3,10 @@ from ldap3 import Server,Connection,NONE,LDAPOperationResult import sys import os import logging +from timeit import default_timer as timer from multiprocessing import Process,Queue -log_queue = Queue(maxsize=32) +log_queue = Queue(maxsize=4) g_milter_name = 'ldap-acl-milter' g_milter_socket = '/socket/' + g_milter_name g_milter_reject_message = 'Absender/Empfaenger passen nicht!' @@ -23,6 +24,7 @@ logging.basicConfig( class LdapAclMilter(Milter.Base): def __init__(self): # A new instance with each new connection. + self.time_start = timer() self.id = Milter.uniqueID() self.ldap_conn = g_ldap_conn self.R = [] @@ -56,32 +58,50 @@ class LdapAclMilter(Milter.Base): def envrcpt(self, to, *str): to = to.replace("<","") to = to.replace(">","") + time_start = timer() + time_end = None try: query = g_ldap_query.replace("%rcpt%",to) query = query.replace("%from%", self.F) self.ldap_conn.search(g_ldap_base, query) + time_end = timer() if len(self.ldap_conn.entries) == 0: - self.R.append({"rcpt": to, "reason": g_milter_reject_message}) + self.R.append({ + "rcpt": to, "reason": g_milter_reject_message, + "time_start":time_start, "time_end":time_end + }) self.setreply('550','5.7.1',g_milter_reject_message) return Milter.REJECT except LDAPOperationResult as e: self.log("LDAP Exception (envrcpt): " + str(e)) self.setreply('451','4.7.1', str(e)) return Milter.TEMPFAIL - self.R.append({"rcpt": to, "reason":'pass'}) + self.R.append({ + "rcpt": to, "reason":'pass',"time_start":time_start,"time_end":time_end + }) return Milter.CONTINUE def data(self): # This callback is used only to log with queue-id - for rcpt in self.R: - self.log(self.getsymval('i') + - ": 5321.from=<" + self.F + "> 5321.rcpt=<" + - rcpt['rcpt'] + "> reason: " + rcpt['reason'] - ) + try: + for rcpt in self.R: + duration = rcpt['time_end'] - rcpt['time_start'] + self.log(self.getsymval('i') + + ": 5321.from=<" + self.F + "> 5321.rcpt=<" + + rcpt['rcpt'] + "> reason: " + rcpt['reason'] + + " duration: " + str(duration) + " sec." + ) + except: + self.log("Exception (data): " + str(sys.exc_info())) + self.setreply('451','4.7.1', str(sys.exc_info())) + return Milter.TEMPFAIL return Milter.CONTINUE def eom(self): # EOM will always be called - non-optional + time_end = timer() + duration = time_end - self.time_start + self.log(self.getsymval('i') + " processing: " + str(duration) + " sec.") return Milter.CONTINUE def abort(self): From 15ff0eeafd29712941551e92491be5868367f4b5 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:30:14 +0100 Subject: [PATCH 05/12] docs, docs, docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fc5ddc1..6f8f816 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ So, if you already have a LDAP server running with e.g. amavis-schema, you may r The connection to the LDAP server is always persistent: one TCP-Session, one LDAP-bind -> less overhead +The intention of this project is to deploy the milter ALWAYS AND ONLY as a docker container. The main reason ist that I´m not so familiar with/interested in building distribution packages (rpm, deb, ...). Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments: get the code, compile and finaly install the results 'somewhere' in the filesystem. In term of CI/CD docker gives us wonderful possibilities I don´t want to miss anymore... + ### docker-compose.yml ``` From 8c954f0e0f83fef64592d8fe4bca801c0fd4cdb3 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:42:30 +0100 Subject: [PATCH 06/12] docs, docs, docs --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6f8f816..f6e3c65 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # ldap-acl-milter -A fast and lightweight and thread-safe python3 milter on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for Access ControL (ACL) scenarios. The milter consumes policies from a LDAP server based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). Please have a look at the docker-compose.yml example. +A lightweight, fast and thread-safe python3 milter on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for basic Access Control (ACL) scenarios. The milter consumes policies from LDAP based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). So, if you already have a LDAP server running with e.g. amavis-schema, you may reuse the 'amavisWhitelistSender'/'amavisBlacklistSender' attributes. Please have a look at the docker-compose.yml example. Of course one is free to write an own LDAP schema for his/her case ;) -So, if you already have a LDAP server running with e.g. amavis-schema, you may reuse the 'amavisWhitelistSender'/'amavisBlacklistSender' attributes. Please have a look at the docker-compose.yml example. +The LDAP-connection is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it less overhead. Thus, LDAP interactions with 3 msec. and less are realistic, depending on your environment like network round-trip-times, the load of your LDAP server, ... -The connection to the LDAP server is always persistent: one TCP-Session, one LDAP-bind -> less overhead +The milter base ([sdgathman/pymilter](https://github.com/sdgathman/pymilter)) is able to 'spawn' hundreds of threads with a wink ;) The intention of this project is to deploy the milter ALWAYS AND ONLY as a docker container. The main reason ist that I´m not so familiar with/interested in building distribution packages (rpm, deb, ...). Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments: get the code, compile and finaly install the results 'somewhere' in the filesystem. In term of CI/CD docker gives us wonderful possibilities I don´t want to miss anymore... From ab91b8b5cb0f0be2908601e6d56f8f3a2c78baa5 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 11:44:09 +0100 Subject: [PATCH 07/12] docs, docs, docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6e3c65..580cda5 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # ldap-acl-milter A lightweight, fast and thread-safe python3 milter on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for basic Access Control (ACL) scenarios. The milter consumes policies from LDAP based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). So, if you already have a LDAP server running with e.g. amavis-schema, you may reuse the 'amavisWhitelistSender'/'amavisBlacklistSender' attributes. Please have a look at the docker-compose.yml example. Of course one is free to write an own LDAP schema for his/her case ;) -The LDAP-connection is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it less overhead. Thus, LDAP interactions with 3 msec. and less are realistic, depending on your environment like network round-trip-times, the load of your LDAP server, ... +The LDAP-connection is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it less overhead. Thus, LDAP interactions with 3 msec. and less are realistic, depending on your environment like network round-trip-times, the load of your LDAP server, ... -The milter base ([sdgathman/pymilter](https://github.com/sdgathman/pymilter)) is able to 'spawn' hundreds of threads with a wink ;) +The milter base ([sdgathman/pymilter](https://github.com/sdgathman/pymilter)) is able to 'spawn' hundreds of threads within a wink ;) The intention of this project is to deploy the milter ALWAYS AND ONLY as a docker container. The main reason ist that I´m not so familiar with/interested in building distribution packages (rpm, deb, ...). Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments: get the code, compile and finaly install the results 'somewhere' in the filesystem. In term of CI/CD docker gives us wonderful possibilities I don´t want to miss anymore... From de11c995e6f1c72e3f285515ff23952953d71d31 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 23:06:15 +0100 Subject: [PATCH 08/12] docs, docs, docs --- README.md | 49 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 580cda5..615e0aa 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,68 @@ # ldap-acl-milter -A lightweight, fast and thread-safe python3 milter on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for basic Access Control (ACL) scenarios. The milter consumes policies from LDAP based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). So, if you already have a LDAP server running with e.g. amavis-schema, you may reuse the 'amavisWhitelistSender'/'amavisBlacklistSender' attributes. Please have a look at the docker-compose.yml example. Of course one is free to write an own LDAP schema for his/her case ;) +A lightweight, fast and thread-safe python3 [milter](http://www.postfix.org/MILTER_README.html) on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter) for basic Access Control (ACL) scenarios. The milter consumes policies from LDAP based on custom queries with trivial templating support (%from% = RFC5321.from; %rcpt% = RFC5321.rcpt). -The LDAP-connection is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it less overhead. Thus, LDAP interactions with 3 msec. and less are realistic, depending on your environment like network round-trip-times, the load of your LDAP server, ... +In the case, one already has a LDAP server running with the [amavis schema](https://www.ijs.si/software/amavisd/LDAP.schema.txt), the 'amavisWhitelistSender' attribute could be reused. The filtering direction (inbound or outbound) can be simply controlled by swapping the %from% and %rcpt% placeholders within the LDAP query template. Please have a look at the docker-compose.yml example. -The milter base ([sdgathman/pymilter](https://github.com/sdgathman/pymilter)) is able to 'spawn' hundreds of threads within a wink ;) +The connection to the LDAP server is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it more efficient due to less communication overhead. Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). This aproach eliminates network round trip times while reading operations as well as race conditions on a shared, centralized and (heavy) utilized LDAP server. -The intention of this project is to deploy the milter ALWAYS AND ONLY as a docker container. The main reason ist that I´m not so familiar with/interested in building distribution packages (rpm, deb, ...). Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments: get the code, compile and finaly install the results 'somewhere' in the filesystem. In term of CI/CD docker gives us wonderful possibilities I don´t want to miss anymore... +### Deployment paradigm +The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org)) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. ### docker-compose.yml +The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-capable MTA and OpenLDAP as local LDAP replica. ``` version: '3' volumes: lam_socket: + openldap_spool: + openldap_socket: services: + openldap: + image: "your/favorite/openldap/image" + restart: unless-stopped + hostname: openldap + volumes: + - "./config/openldap:/etc/openldap:rw" + - "openldap_spool:/var/openldap-data:rw" + - "openldap_socket:/socket:rw" + ldap-acl-milter: - image: "ldap-acl-milter/debian:19.02_devel" + depends_on: + - openldap + image: "ldap-acl-milter/debian:19.02_master" restart: unless-stopped environment: - LDAP_SERVER: ldap://ldap-slave.example.org:389 - LDAP_BINDDN: uid=lam,ou=apps,dc=example,dc=org - LDAP_BINDPW: TopSecret1! + #LDAP_SERVER: ldap://ldap-slave.example.local:389 + LDAP_SERVER: ldapi:///socket//slapd//slapd + LDAP_BINDDN: uid=lam,ou=applications,dc=example,dc=org + LDAP_BINDPW: TopSecret123!%& LDAP_BASE: ou=users,dc=example,dc=org + # This example LDAP query is for inbound filtering + # where the 'mail' attribute equals to the recipient + # and the 'amavisWhitelistSender' attribute the eligible sender LDAP_QUERY: (&(mail=%rcpt%)(amavisWhitelistSender=%from%)) - # Socket default: /socket/ldap-acl-milter - # MILTER_SOCKET: inet6:8020 - MILTER_REJECT_MESSAGE: Rejected due to security policy violation + # Default: UNIX-socket located under /socket/ldap-acl-milter + # https://pythonhosted.org/pymilter/namespacemilter.html#a266a6e09897499d8b1ae0e20f0d2be73 + #MILTER_SOCKET: inet6:8020 + MILTER_REJECT_MESSAGE: Message rejected due to security policy hostname: ldap-acl-milter volumes: - "lam_socket:/socket/:rw" + - "openldap_socket:/socket/slapd:ro" + postfix: depends_on: - ldap-acl-milter - image: "postfix/alpine/amd64" + image: "your/favorite/postfix/image" restart: unless-stopped hostname: postfix ports: - "25:25" volumes: - "./config/postfix:/etc/postfix:rw" - - "lam_socket:/socket/:rw" + - "lam_socket:/socket/ldap-acl-milter/:rw" + - "openldap_socket:/socket/slapd:ro" ``` From fddf593beb6ff3db309920fe6a26cf3ca5f290b7 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 23:09:51 +0100 Subject: [PATCH 09/12] docs, docs, docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 615e0aa..ae48ae2 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ A lightweight, fast and thread-safe python3 [milter](http://www.postfix.org/MILT In the case, one already has a LDAP server running with the [amavis schema](https://www.ijs.si/software/amavisd/LDAP.schema.txt), the 'amavisWhitelistSender' attribute could be reused. The filtering direction (inbound or outbound) can be simply controlled by swapping the %from% and %rcpt% placeholders within the LDAP query template. Please have a look at the docker-compose.yml example. -The connection to the LDAP server is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it more efficient due to less communication overhead. Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). This aproach eliminates network round trip times while reading operations as well as race conditions on a shared, centralized and (heavy) utilized LDAP server. +The connection to the LDAP server is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it more efficient due to less communication overhead. Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). This aproach eliminates network round trip times while reading from a UNIX-socket as well as performance bottlenecks on a shared, centralized and (heavy) utilized LDAP server. ### Deployment paradigm -The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org)) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. +The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. ### docker-compose.yml The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-capable MTA and OpenLDAP as local LDAP replica. From 4e273d911b721fd516d5afc72aef7e37990e15db Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 23:19:29 +0100 Subject: [PATCH 10/12] docs, docs, docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae48ae2..4bab833 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A lightweight, fast and thread-safe python3 [milter](http://www.postfix.org/MILT In the case, one already has a LDAP server running with the [amavis schema](https://www.ijs.si/software/amavisd/LDAP.schema.txt), the 'amavisWhitelistSender' attribute could be reused. The filtering direction (inbound or outbound) can be simply controlled by swapping the %from% and %rcpt% placeholders within the LDAP query template. Please have a look at the docker-compose.yml example. -The connection to the LDAP server is always persistent: one TCP-Session/one LDAP-bind shared among all milter-threads, which makes it more efficient due to less communication overhead. Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). This aproach eliminates network round trip times while reading from a UNIX-socket as well as performance bottlenecks on a shared, centralized and (heavy) utilized LDAP server. +The connection to the LDAP server is always persistent: one LDAP-bind is shared among all milter-threads, which makes it more efficient due to less communication overhead (which also implies transport encryption with TLS). Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). On the one hand, this aproach eliminates network round trip times while reading from a UNIX-socket, and on the other, performance bottlenecks on a shared, centralized and (heavy) utilized LDAP server. ### Deployment paradigm The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. From a977424d19abf1682ce6346e489b62bb1420bf90 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 23:21:31 +0100 Subject: [PATCH 11/12] docs, docs, docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4bab833..c04c499 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ In the case, one already has a LDAP server running with the [amavis schema](http The connection to the LDAP server is always persistent: one LDAP-bind is shared among all milter-threads, which makes it more efficient due to less communication overhead (which also implies transport encryption with TLS). Thus, LDAP interactions with 2 msec. and less are realistic, depending on your environment like network round-trip-times or the load of your LDAP server. A very swag LDAP setup is to use a local read-only LDAP replica, which syncs over network with a couple of LDAP masters: [OpenLDAP does it for free!](https://www.openldap.org/doc/admin24/replication.html). On the one hand, this aproach eliminates network round trip times while reading from a UNIX-socket, and on the other, performance bottlenecks on a shared, centralized and (heavy) utilized LDAP server. ### Deployment paradigm -The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not realy a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. +The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not really a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. ### docker-compose.yml The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-capable MTA and OpenLDAP as local LDAP replica. From 683a9803a6456b92029124f905cbb09356c251a2 Mon Sep 17 00:00:00 2001 From: Dominik Chilla Date: Tue, 12 Feb 2019 23:26:36 +0100 Subject: [PATCH 12/12] milter logs directly to stdout; No logging-background process anymore --- app/ldap-acl-milter.py | 79 ++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/app/ldap-acl-milter.py b/app/ldap-acl-milter.py index 620e046..b44afd3 100644 --- a/app/ldap-acl-milter.py +++ b/app/ldap-acl-milter.py @@ -1,12 +1,14 @@ import Milter -from ldap3 import Server,Connection,NONE,LDAPOperationResult +from ldap3 import ( + Server,ServerPool,Connection,NONE,LDAPOperationResult +) import sys import os import logging +import string +import random from timeit import default_timer as timer -from multiprocessing import Process,Queue -log_queue = Queue(maxsize=4) g_milter_name = 'ldap-acl-milter' g_milter_socket = '/socket/' + g_milter_name g_milter_reject_message = 'Absender/Empfaenger passen nicht!' @@ -23,13 +25,18 @@ logging.basicConfig( ) class LdapAclMilter(Milter.Base): - def __init__(self): # A new instance with each new connection. + # Each new connection is handled in an own thread + def __init__(self): self.time_start = timer() self.id = Milter.uniqueID() self.ldap_conn = g_ldap_conn self.R = [] + # https://stackoverflow.com/a/2257449 + self.mconn_id = ''.join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(8) + ) - # Not registered callbacks + # Not registered/used callbacks @Milter.nocallback def connect(self, IPname, family, hostaddr): return self.CONTINUE @@ -48,7 +55,9 @@ class LdapAclMilter(Milter.Base): def envfrom(self, mailfrom, *str): if mailfrom == '<>': - self.setreply('550','5.7.1','Envelope null-sender not allowed!') + ex = str(self.mconn_id + '/FROM Envelope null-sender not allowed!') + logging.error(ex) + self.setreply('550','5.7.1',ex) Milter.REJECT mailfrom = mailfrom.replace("<","") mailfrom = mailfrom.replace(">","") @@ -56,9 +65,9 @@ class LdapAclMilter(Milter.Base): return Milter.CONTINUE def envrcpt(self, to, *str): + time_start = timer() to = to.replace("<","") to = to.replace(">","") - time_start = timer() time_end = None try: query = g_ldap_query.replace("%rcpt%",to) @@ -70,11 +79,16 @@ class LdapAclMilter(Milter.Base): "rcpt": to, "reason": g_milter_reject_message, "time_start":time_start, "time_end":time_end }) - self.setreply('550','5.7.1',g_milter_reject_message) + self.setreply('550','5.7.1', + 'Sender does not comply with recipients policy!' + ) + logging.info(self.mconn_id + "/RCPT " + g_milter_reject_message) return Milter.REJECT except LDAPOperationResult as e: - self.log("LDAP Exception (envrcpt): " + str(e)) - self.setreply('451','4.7.1', str(e)) + logging.warn(self.mconn_id + "/RCPT LDAP Exception (envrcpt): " + str(e)) + self.setreply('451','4.7.1', + 'Service temporarily not available! Please try again later.' + ) return Milter.TEMPFAIL self.R.append({ "rcpt": to, "reason":'pass',"time_start":time_start,"time_end":time_end @@ -82,46 +96,44 @@ class LdapAclMilter(Milter.Base): return Milter.CONTINUE def data(self): - # This callback is used only to log with queue-id + # A queue-id will be generated after the first accepted RCPT TO + # and therefore not available until DATA command + self.queue_id = self.getsymval('i') try: for rcpt in self.R: duration = rcpt['time_end'] - rcpt['time_start'] - self.log(self.getsymval('i') + + logging.info(self.mconn_id + "/DATA " + self.queue_id + ": 5321.from=<" + self.F + "> 5321.rcpt=<" + rcpt['rcpt'] + "> reason: " + rcpt['reason'] + " duration: " + str(duration) + " sec." ) except: - self.log("Exception (data): " + str(sys.exc_info())) - self.setreply('451','4.7.1', str(sys.exc_info())) + ex = str(self.mconn_id + "/DATA " + self.queue_id + + ": Exception (data): " + sys.exc_info() + ) + logging.warn(ex) + self.setreply('451','4.7.1', ex) return Milter.TEMPFAIL return Milter.CONTINUE def eom(self): - # EOM will always be called - non-optional + # EOM is not optional and thus, always called by MTA time_end = timer() duration = time_end - self.time_start - self.log(self.getsymval('i') + " processing: " + str(duration) + " sec.") + logging.info(self.mconn_id + "/EOM " + self.queue_id + + " processing: " + str(duration) + " sec." + ) return Milter.CONTINUE def abort(self): - # client disconnected prematurely + # Client disconnected prematurely return Milter.CONTINUE def close(self): - # always called, even when abort is called. Clean up - # any external resources here. + # Always called, even when abort is called. + # Clean up any external resources here. return Milter.CONTINUE - def log(self,msg): - log_queue.put(msg) - -def logger_proc_loop(): - while True: - qmsg = log_queue.get() - if not qmsg: break - logging.info(qmsg) - if __name__ == "__main__": try: if 'LDAP_SERVER' not in os.environ: @@ -144,7 +156,12 @@ if __name__ == "__main__": g_milter_socket = os.environ['MILTER_SOCKET'] if 'MILTER_REJECT_MESSAGE' in os.environ: g_milter_reject_message = os.environ['MILTER_REJECT_MESSAGE'] + #server_pool = ServerPool(None, pool_strategy='ROUND_ROBIN', active=False, exhaust=False) server = Server(g_ldap_server, get_info=NONE) + #server_pool.add(server) + #server2 = Server('ldap://ldap-master-zdf.zwackl.local:389', get_info=NONE) + #server_pool.add(server2) + #g_ldap_conn = Connection(server_pool, g_ldap_conn = Connection(server, g_ldap_binddn, g_ldap_bindpw, auto_bind=True, raise_exceptions=True, @@ -155,18 +172,14 @@ if __name__ == "__main__": logging.error("LDAP Exception: " + str(e)) sys.exit(1) try: - logger_proc = Process(target=logger_proc_loop) - logger_proc.start() timeout = 600 # Register to have the Milter factory create instances of your class: Milter.factory = LdapAclMilter # Tell the MTA which features we use flags = Milter.ADDHDRS Milter.set_flags(flags) - logging.info("Startup " + g_milter_name + "@"+ g_milter_socket) + logging.info("Startup " + g_milter_name + "@socket: " + g_milter_socket) Milter.runmilter(g_milter_name,g_milter_socket,timeout,True) - log_queue.put(None) - logger_proc.join() logging.info("Shutdown " + g_milter_name) except: logging.error("MAIN-EXCEPTION: " + str(sys.exc_info()))