diff --git a/README.md b/README.md index 75ffa24..9018616 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,22 @@ -# sos-milter +# SPF-on-submission-Milter - sos-milter A lightweight, fast and thread-safe python3 [milter](http://www.postfix.org/MILTER_README.html) on top of [sdgathman/pymilter](https://github.com/sdgathman/pymilter). -### Deployment paradigm -The intention of this project is to deploy the milter ALWAYS AND ONLY as an [OCI compliant](https://www.opencontainers.org) container. In this case it´s [docker](https://www.docker.com). The main reason is that I´m not interested (and familiar with) in building distribution packages like .rpm, .deb, etc.. Furthermore I´m not really a fan of 'wild and uncontrollable' software deployments like: get the code, compile it and finaly install the results 'somewhere' in the filesystem. In terms of software deployment docker provides wonderful possibilities, which I don´t want to miss anymore... No matter if in development, QA or production stage. +The main goal of the **sos-milter** is to check the SPF-policy of a senders domain in term of mail submission scenario. Especially when forwarding of messages comming from *foreign* domains with restrictive SPF-policies (-all) takes place. The milter is also designed to check the correctness of SPF-policies for *own* domains (such as customers domains). In this case the milter expects that all *own* (not foreign) domains are managed in a LDAP server so that the milter can recognize them as such. For those domains the milter enforces checks regarding the appearence of particular SPF statements (include/s, ip4, ip6, ...) in the domain name system (DNS). Herefor the milter uses a regular expression which is part of the configuration. In this way the email service provider (ESP) running the sos-milter becomes able to check if his/her customers did set SPF-TXT-records correctly (as documented/expected) on each mail submission attempt and not just during the setup phase. -### docker-compose.yml -The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-aware MTA. +Further the sos-milter can be run in `test` or `reject` mode. In `test` mode the milter only does log policy violations which may be turned into metrics and used for baselining. Thus the `test` mode is recommended for first steps in an productive environment before enabling reject mode (if ever). In `reject` mode the milter fulfills policy enforcement and rejects every email submission requests that does not meet the configured expectations (expected SPF statements as regular expression). + +### Deployment paradigm +Following the principles of [12-Factor-App](https://12factor.net/) for cloud native applications, the intention of this project is to deploy the milter as an [OCI compliant](https://www.opencontainers.org) container. + +There´s nothing wrong to deploy the milter as a +* docker-compose deployment on a stand-alone docker host or a docker-swarm cluster +* stateless Kubernetes-workload (type: `Deployment`) +* local systemd unit, which is NOT a OCI-compliant was but works too ;-) + +Please note, that according to the [3rd principle](https://12factor.net/config) of 12-Factor-App a cloud-native app is configured through environment variables, so this app does. + +### Deployment with Docker - docker-compose.yml +The following [docker-compose](https://docs.docker.com/compose/) file demonstrates how such a setup could be orchestrated on a single docker host or on a docker swarm cluster. In this context we use [postfix](http://www.postfix.org) as our milter-aware MTA which connects to the milter via an UNIX-socket. ``` version: '3' @@ -15,7 +26,7 @@ volumes: services: sos-milter: - image: "sos-milter/debian:19.06_master" + image: "sos-milter:" restart: unless-stopped environment: # default: info, possible: info, warning, error, debug @@ -55,11 +66,11 @@ services: postfix: depends_on: - sos-milter - image: "postfix/alpine/amd64" + image: "your favorite postfix image" restart: unless-stopped hostname: postfix ports: - - "1587:587" + - "465:465" volumes: - "./config/postfix:/etc/postfix:ro" - "sosm_socket:/socket/sos-milter/:rw" diff --git a/app/sos-milter.py b/app/sos-milter.py index bd90a86..b9b884f 100644 --- a/app/sos-milter.py +++ b/app/sos-milter.py @@ -8,7 +8,8 @@ import random import re import dns.resolver from ldap3 import ( - Server, Connection, NONE, set_config_parameter + Server, Connection, NONE, set_config_parameter, + SAFE_RESTARTABLE ) from ldap3.core.exceptions import LDAPException @@ -23,6 +24,9 @@ g_loglevel = logging.INFO g_milter_mode = 'test' g_ignored_next_hops = {} g_ldap_conn = None +g_ldap_server_uri = None +g_ldap_search_base = None +g_ldap_query_filter = None g_ldap_binddn = '' g_ldap_bindpw = '' @@ -74,11 +78,15 @@ class SOSMilter(Milter.Base): self.reset() self.client_ip = self.getsymval('{client_addr}') if self.client_ip is None: - logging.error(self.mconn_id + " FROM exception: could not retrieve milter-macro ({client_addr})!") + logging.error(self.mconn_id + + " FROM exception: could not retrieve milter-macro ({client_addr})!" + ) self.setreply('450','4.7.1', g_milter_tmpfail_message) return Milter.TEMPFAIL else: - logging.debug(self.mconn_id + "/FROM client_ip={0}".format(self.client_ip)) + logging.debug(self.mconn_id + + "/FROM client_ip={0}".format(self.client_ip) + ) try: # DSNs/bounces are not relevant if(mailfrom == '<>'): @@ -100,17 +108,21 @@ class SOSMilter(Milter.Base): ) # Check if env_from_domain is in ldap if(g_ldap_conn is not None): - filter = os.environ['LDAP_QUERY_FILTER'] + filter = g_ldap_query_filter filter = filter.replace("%d", self.env_from_domain) + logging.debug(self.mconn_id + "/FROM " + + "LDAP query filter: {}".format(filter) + ) try: - g_ldap_conn.search(os.environ['LDAP_SEARCH_BASE'], + _, _, ldap_response, _ = g_ldap_conn.search( + g_ldap_search_base, filter, attributes=[] ) - if len(g_ldap_conn.entries) != 0: + if len(ldap_response) != 0: self.is_env_from_domain_in_ldap = True logging.info(self.mconn_id + - "/FROM Domain {0} found in LDAP".format(self.env_from_domain) + "/FROM 5321.from_domain={0} found in LDAP".format(self.env_from_domain) ) except LDAPException: logging.error(self.mconn_id + "/FROM " + traceback.format_exc()) @@ -291,61 +303,75 @@ if __name__ == "__main__": if 'MILTER_MODE' in os.environ: if re.match(r'^test|reject$',os.environ['MILTER_MODE'], re.IGNORECASE): g_milter_mode = os.environ['MILTER_MODE'] + logging.info("ENV[MILTER_MODE]: {}".format(g_milter_mode)) if 'MILTER_NAME' in os.environ: g_milter_name = os.environ['MILTER_NAME'] + logging.info("ENV[MILTER_NAME]: {}".format(g_milter_name)) if 'MILTER_SOCKET' in os.environ: g_milter_socket = os.environ['MILTER_SOCKET'] + logging.info("ENV[MILTER_SOCKET]: {}".format(g_milter_socket)) if 'MILTER_REJECT_MESSAGE' in os.environ: g_milter_reject_message = os.environ['MILTER_REJECT_MESSAGE'] + logging.info("ENV[MILTER_REJECT_MESSAGE]: {}".format(g_milter_reject_message)) if 'MILTER_TMPFAIL_MESSAGE' in os.environ: g_milter_tmpfail_message = os.environ['MILTER_TMPFAIL_MESSAGE'] + logging.info("ENV[MILTER_TMPFAIL_MESSAGE]: {}".format(g_milter_tmpfail_message)) if 'SPF_REGEX' in os.environ: try: g_re_spf_regex = re.compile(os.environ['SPF_REGEX'], re.IGNORECASE) except: logging.error("ENV[SPF_REGEX] exception: " + traceback.format_exc()) sys.exit(1) + logging.info("ENV[SPF_REGEX]: {}".format(g_re_spf_regex)) if 'IGNORED_NEXT_HOPS' in os.environ: try: tmp_hops = os.environ['IGNORED_NEXT_HOPS'].split(',') for next_hop in tmp_hops: g_ignored_next_hops[next_hop] = 'ignore' - logging.debug("next-hops: " + str(g_ignored_next_hops)) except: logging.error("ENV[IGNORED_NEXT_HOPS] exception: " + traceback.format_exc()) sys.exit(1) + logging.info("ENV[IGNORED_NEXT_HOPS]: {}".format(g_ignored_next_hops)) if 'LDAP_ENABLED' in os.environ: if 'LDAP_SERVER_URI' not in os.environ: logging.error("ENV[LDAP_SERVER_URI] is mandatory!") sys.exit(1) + g_ldap_server_uri = os.environ['LDAP_SERVER_URI'] + logging.info("ENV[LDAP_SERVER_URI]: {}".format(g_ldap_server_uri)) if 'LDAP_BINDDN' not in os.environ: logging.info("ENV[LDAP_BINDDN] not set! Continue...") else: g_ldap_binddn = os.environ['LDAP_BINDDN'] + logging.info("ENV[LDAP_BINDDN]: {}".format("***")) if 'LDAP_BINDPW' not in os.environ: logging.info("ENV[LDAP_BINDPW] not set! Continue...") else: g_ldap_bindpw = os.environ['LDAP_BINDPW'] + logging.info("ENV[LDAP_BINDPW]: {}".format("***")) if 'LDAP_SEARCH_BASE' not in os.environ: logging.error("ENV[LDAP_SEARCH_BASE] is mandatory!") sys.exit(1) + g_ldap_search_base = os.environ['LDAP_SEARCH_BASE'] + logging.info("ENV[LDAP_SEARCH_BASE]: {}".format(g_ldap_search_base)) if 'LDAP_QUERY_FILTER' not in os.environ: logging.error("ENV[LDAP_QUERY_FILTER] is mandatory!") sys.exit(1) + g_ldap_query_filter = os.environ['LDAP_QUERY_FILTER'] + logging.info("ENV[LDAP_QUERY_FILTER]: {}".format(g_ldap_query_filter)) try: set_config_parameter("RESTARTABLE_SLEEPTIME", 2) set_config_parameter("RESTARTABLE_TRIES", 2) set_config_parameter('DEFAULT_SERVER_ENCODING', 'utf-8') set_config_parameter('DEFAULT_CLIENT_ENCODING', 'utf-8') - server = Server(os.environ['LDAP_SERVER_URI'], get_info=NONE) + server = Server(g_ldap_server_uri, get_info=NONE) g_ldap_conn = Connection(server, g_ldap_binddn, g_ldap_bindpw, auto_bind=True, raise_exceptions=True, - client_strategy='RESTARTABLE' + client_strategy=SAFE_RESTARTABLE ) - logging.info("LDAP-Connection established. PID: " + str(os.getpid())) + logging.info("LDAP connection established. PID: " + str(os.getpid())) except LDAPException as e: print("LDAP-Exception: " + traceback.format_exc()) sys.exit(1) diff --git a/tests/miltertest_dsn.lua b/tests/miltertest_dsn.lua new file mode 100644 index 0000000..0a13701 --- /dev/null +++ b/tests/miltertest_dsn.lua @@ -0,0 +1,48 @@ +-- https://mopano.github.io/sendmail-filter-api/constant-values.html#com.sendmail.milter.MilterConstants +-- http://www.opendkim.org/miltertest.8.html + +-- socket must be defined as miltertest global variable (-D) +conn = mt.connect(socket) +if conn == nil then + error "mt.connect() failed" +end + +mt.set_timeout(3) + +-- 5321.FROM + MACROS +mt.macro(conn, SMFIC_MAIL, '{client_addr}', "127.128.129.130") +if mt.mailfrom(conn, "<>") ~= nil then + error "mt.mailfrom() failed" +end +if mt.getreply(conn) ~= SMFIR_CONTINUE then + error "mt.mailfrom() unexpected reply" +end + +-- 5321.RCPT + MACROS +mt.macro(conn, SMFIC_RCPT, "i", "TestQueueId-1",'{rcpt_host}', "test.next-hostx") +if mt.rcptto(conn, "some@recipient.somewhere") ~= nil then + error "mt.rcptto() failed" +end +if mt.getreply(conn) ~= SMFIR_CONTINUE then + error "mt.rcptto() unexpected reply" +end + +-- EOM +if mt.eom(conn) ~= nil then + error "mt.eom() failed" +end +mt.echo("EOM: " .. mt.getreply(conn)) +if mt.getreply(conn) == SMFIR_CONTINUE then + mt.echo("EOM-continue") +elseif mt.getreply(conn) == SMFIR_REPLYCODE then + mt.echo("EOM-reject") +end + +if not mt.eom_check(conn, MT_HDRADD, "X-SOS-Milter") then + mt.echo("no header added") +else + mt.echo("X-SOS-Milter header added -> LDAP-Domain with broken SPF") +end + +-- DISCONNECT +mt.disconnect(conn) \ No newline at end of file