LDAP policy backend, take 1

LDAP/README.md

@@ -0,0 +1 @@
+# ExOTA-Milter with LDAP backend

LDAP/exota-milter.schema

@@ -0,0 +1,41 @@
+# https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers
+# DC IT-Consulting
+# Dominik Chilla
+#
+# OID prefix: 1.3.6.1.4.1.53501
+#
+# ExOTA-Milter: 1.3.6.1.4.1.53501.3
+# Attributes:   1.3.6.1.4.1.53501.3.1
+# Objects:      1.3.6.1.4.1.53501.3.2
+
+### Attributes
+attributetype ( 1.3.6.1.4.1.53501.3.1.1
+  NAME 'exotaMilterTenantId'
+  DESC 'ExOTA-Milter Tenant-ID'
+  EQUALITY uuidMatch
+  ORDERING uuidOrderingMatch
+  SYNTAX 1.3.6.1.1.16.1
+  SINGLE-VALUE )
+
+attributetype ( 1.3.6.1.4.1.53501.3.1.2
+  NAME 'exotaMilterDkimEnabled'
+  DESC 'ExOTA-Milter DKIM enabled flag'
+  EQUALITY booleanMatch
+  SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 
+  SINGLE-VALUE )
+
+attributetype ( 1.3.6.1.4.1.53501.3.1.3
+  NAME 'exotaMilterDkimAlignmentRequired'
+  DESC 'ExOTA-Milter DKIM alignment required flag'
+  EQUALITY booleanMatch
+  SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 
+  SINGLE-VALUE )
+
+### Objects
+objectclass ( 1.3.6.1.4.1.53501.3.2.1
+  NAME 'exotaMilterPolicy' AUXILIARY
+  DESC 'ExOTA-Milter policy object'
+  SUP top
+  MAY ( exotaMilterTenantId $ 
+        exotaMilterDkimEnabled $ 
+        exotaMilterDkimAlignmentRequired ))

README.md

@@ -80,10 +80,20 @@ Authentication-Results: trusted.dkim.validating.relay;
 [...]
 ```
 
-## More authentic with DKIM alignment and message forwarding
+## More authentic message forwarding with DKIM alignment
-From the point of view of a postmaster, message forwarding is a kind of nightmare. If DKIM alignment is enabled the DKIM SDID (Signers Domain ID = `header.d` field of *Authentication-Results* header) must be equivalent to the RFC5322.from_domain. In this mode the **EXOTA-Milter** operates in the most secure way, but with limitations in terms of usability. With DKIM alignment enabled [traditional email forwarding](https://docs.microsoft.com/de-de/microsoft-365/admin/email/configure-email-forwarding?view=o365-worldwide) cannot be guaranteed. In this case the exchange online system preserves the original RFC5322.from header and signs the forwarded email with the main tenants SDID, e.g. *tenantdomain.onmicrosoft.com*. An email that was forwarded in that way cannot pass the DKIM alignment, because the RFC5322.from_domain will never match the DKIM SDID. Further there is no policy for the RFC5322.from_domain!
 
-Nevertheless, don´t put your head in the sand, there is a way out of this dilemma! Just use outlook rules to forward messages, which is described [here](https://support.microsoft.com/en-us/office/use-rules-to-automatically-forward-messages-45aa9664-4911-4f96-9663-ece42816d746). In this case the original content gets forwarded within a new message, that carries the correct RFC5322.from as well as the correct DKIM SDID! Messages forwarded in such way will always pass DKIM alignment -> mission accomplished! Don´t forget to tell your end users ;)
+From the point of view of a postmaster, message forwarding is a kind of nightmare. If DKIM alignment requirement is enabled (`ENV[MILTER_DKIM_ALIGNMENT_REQUIRED]`) the DKIM SDID (Signers Domain ID = `header.d` field of *Authentication-Results* header) must be equivalent to the RFC5322.from_domain. In this mode the **ExOTA-Milter** operates in the most secure way, but with limitations in terms of usability. With DKIM alignment enabled [traditional email forwarding](https://docs.microsoft.com/de-de/microsoft-365/admin/email/configure-email-forwarding?view=o365-worldwide) cannot be guaranteed to work. In this case the exchange online system preserves the original RFC5322.from header (e.g. *someone@gmail.com*) and signs the forwarded message with the main tenants SDID, e.g. *tenantdomain.onmicrosoft.com*. An email that was forwarded in that way cannot pass the DKIM alignment, because the RFC5322.from_domain (e.g. *gmail.com*) will never match the DKIM SDID (e.g. *tenantdomain.onmicrosoft.com*). Further there is no policy match for the RFC5322.from_domain!
+
+Nevertheless, don´t put your head in the sand, there is a way out of this dilemma! Just use **outlook rules** instead of the classic forwarding feature to forward messages, which is described [here](https://support.microsoft.com/en-us/office/use-rules-to-automatically-forward-messages-45aa9664-4911-4f96-9663-ece42816d746). In this case the original content gets forwarded within a new message, that carries the correct RFC5322.from as well as the correct DKIM SDID! Messages forwarded in such way will always pass DKIM alignment -> mission accomplished! Don´t forget to tell your end users how to correctly set up their forwarding in outlook ;)
+
+By the way, the global setting `ENV[MILTER_DKIM_ALIGNMENT_REQUIRED]` can be overriden per policy! Just add the following `dkim_alignment_required` key with the value `false` to the appropriate policy:
+```
+"yad.onmicrosoft.com": {
+  "tenant_id": "1234abcd-18c5-45e8-88de-123456789abc",
+  "dkim_enabled": true,
+  "dkim_alignment_required": false
+}
+```
 
 ## X-MS-Exchange-CrossTenant-Id header (policy binding)
 Further each Microsoft Exchange-Online tenant has a unique tenant-ID in form of a UUID ([RFC 4122](https://tools.ietf.org/html/rfc4122)). **ExOTA-Milter** extracts the tenant-ID from the *X-MS-Exchange-CrossTenant-Id* email header and uses it as a *mandatory* authentication factor.
@@ -97,7 +107,8 @@ At last the **ExOTA-Milter** needs an additional policy (JSON file), that provid
 {
   "yad.onmicrosoft.com": {
     "tenant_id": "1234abcd-18c5-45e8-88de-123456789abc",
-    "dkim_enabled": true
+    "dkim_enabled": true,
+    "dkim_alignment_required": true
   }
 }
 ```

app/exota-milter.py

@@ -11,8 +11,13 @@ import authres
 import json
 from policy import (
   ExOTAPolicyException, ExOTAPolicyNotFoundException, 
-  ExOTAPolicyBackendJSON, ExOTAPolicy
+  ExOTAPolicyBackendJSON, ExOTAPolicyBackendLDAP, ExOTAPolicy,
+  ExOTAPolicyInvalidException, ExOTAPolicyBackendException
 )
+from ldap3 import (
+  Server, Connection, NONE, set_config_parameter
+)
+from ldap3.core.exceptions import LDAPException
 
 # Globals with defaults. Can/should be modified by ENV-variables on startup.
 # ENV[MILTER_NAME]
@@ -45,10 +50,28 @@ g_milter_x509_ip_whitelist = ['127.0.0.1','::1']
 g_milter_add_header = False
 # ENV[MILTER_AUTHSERVID]
 g_milter_authservid = None
+# ENV[MILTER_LDAP_SERVER_URI]
+g_milter_ldap_server_uri = ''
+# ENV[MILTER_LDAP_BINDDN]
+g_milter_ldap_binddn = ''
+# ENV[MILTER_LDAP_BINDPW]
+g_milter_ldap_bindpw = ''
+# ENV[MILTER_LDAP_SEARCH_BASE]
+g_milter_ldap_search_base = ''
+# ENV[MILTER_LDAP_QUERY]
+g_milter_ldap_query = ''
+# ENV[MILTER_LDAP_TENANT_ID_ATTR]
+g_milter_ldap_tenant_id_attr = 'exotaMilterTenantId'
+# ENV[MILTER_LDAP_DKIM_ENABLED_ATTR]
+g_milter_ldap_dkim_enabled_attr = 'exotaMilterDkimEnabled'
+# ENV[MILTER_LDAP_DKIM_ALIGNMENT_REQIRED_ATTR]
+g_milter_ldap_dkim_alignment_required_attr = 'exotaMilterDkimAlignmentRequired'
+
 
 # Another globals
 g_policy_backend = None
 g_re_domain = re.compile(r'^.*@(\S+)$', re.IGNORECASE)
+g_milter_ldap_conn = None
 
 class ExOTAMilter(Milter.Base):
   # Each new connection is handled in an own thread
@@ -91,6 +114,20 @@ class ExOTAMilter(Milter.Base):
     self.setreply('550','5.7.1', message)
     return Milter.REJECT
   
+  def smfir_tempfail(self, **kwargs):
+    message = g_milter_tmpfail_message
+    if 'message' in kwargs:
+      message = kwargs['message']
+    if 'queue_id' in kwargs:
+      message = "queue_id: {0} - {1}".format(kwargs['queue_id'], message)
+    if 'reason' in kwargs:
+      message = "{0} - reason: {1}".format(message, kwargs['reason'])
+    logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+      ": milter_action=tempfail message={0}".format(message)
+    )
+    self.setreply('450','4.7.1', message)
+    return Milter.TEMPFAIL
+  
   def smfir_continue(self):
     return Milter.CONTINUE
 
@@ -267,6 +304,26 @@ class ExOTAMilter(Milter.Base):
           self.hdr_from_domain
         )
       )
+    except ExOTAPolicyBackendException as e:
+      logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+        "/EOM: Policy backend problem: {0}".format(e.message)
+      )
+      return self.smfir_tempfail(
+        queue_id = self.getsymval('i'),
+        reason = "Policy backend problem"
+      )
+    except ExOTAPolicyInvalidException as e:
+      logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+        "/EOM: Invalid policy for 5322.from_domain={0}: {1}".format(
+          self.hdr_from_domain, e.message
+        )
+      )
+      return self.smfir_reject(
+        queue_id = self.getsymval('i'),
+        reason = "Invalid policy for 5322.from_domain {0}".format(
+          self.hdr_from_domain
+        )
+      )
     except (ExOTAPolicyException, ExOTAPolicyNotFoundException) as e:
       logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
         "/EOM: 5322.from: {0}".format(e.message)
@@ -286,6 +343,26 @@ class ExOTAMilter(Milter.Base):
               self.hdr_resent_from_domain
             )
           )
+        except ExOTAPolicyBackendException as e:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: Policy backend problem: {0}".format(e.message)
+          )
+          return self.smfir_tempfail(
+            queue_id = self.getsymval('i'),
+            reason = "Policy backend problem"
+          )
+        except ExOTAPolicyInvalidException as e:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: Invalid policy for 5322.resent_from_domain={0}: {1}".format(
+              self.hdr_resent_from_domain, e.message
+            )
+          )
+          return self.smfir_reject(
+            queue_id = self.getsymval('i'),
+            reason = "Invalid policy for 5322.resent_from_domain {0}".format(
+              self.hdr_resent_from_domain
+            )
+          )
         except (ExOTAPolicyException, ExOTAPolicyNotFoundException) as e:
           logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
             "/EOM: 5322.resent-from: {0}".format(e.message)
@@ -485,9 +562,6 @@ if __name__ == "__main__":
       g_milter_x509_ip_whitelist = g_milter_x509_ip_whitelist.split(',')
     logging.info("ENV[MILTER_X509_IP_WHITELIST]: {0}".format(g_milter_x509_ip_whitelist))
   logging.info("ENV[MILTER_X509_ENABLED]: {0}".format(g_milter_x509_enabled))
-  if 'MILTER_POLICY_SOURCE' in os.environ:
-    g_milter_policy_source = os.environ['MILTER_POLICY_SOURCE']
-  logging.info("ENV[MILTER_POLICY_SOURCE]: {0}".format(g_milter_policy_source))
   if 'MILTER_ADD_HEADER' in os.environ:
     g_milter_add_header = True
     if 'MILTER_AUTHSERVID' in os.environ:
@@ -499,6 +573,9 @@ if __name__ == "__main__":
       logging.error("ENV[MILTER_AUTHSERVID] is mandatory!")
       sys.exit(1)
   logging.info("ENV[MILTER_ADD_HEADER]: {0}".format(g_milter_add_header))
+  if 'MILTER_POLICY_SOURCE' in os.environ:
+    g_milter_policy_source = os.environ['MILTER_POLICY_SOURCE']
+  logging.info("ENV[MILTER_POLICY_SOURCE]: {0}".format(g_milter_policy_source))
   if g_milter_policy_source == 'file':
     if 'MILTER_POLICY_FILE' in os.environ:
       g_milter_policy_file = os.environ['MILTER_POLICY_FILE']
@@ -513,8 +590,62 @@ if __name__ == "__main__":
       logging.error("ENV[MILTER_POLICY_FILE] is mandatory!")
       sys.exit(1)
   elif g_milter_policy_source == 'ldap':
-    logging.debug("LDAP-Backend not supported yet!")
+    if 'MILTER_LDAP_SERVER_URI' not in os.environ:
-    sys.exit(1)
+      logging.error("ENV[MILTER_LDAP_SERVER_URI] is mandatory!")
+      sys.exit(1)
+    g_milter_ldap_server_uri = os.environ['MILTER_LDAP_SERVER_URI']
+    if 'MILTER_LDAP_BINDDN' not in os.environ:
+      logging.info("ENV[MILTER_LDAP_BINDDN] not set! Continue...")
+    else:
+      g_milter_ldap_binddn = os.environ['MILTER_LDAP_BINDDN']
+    if 'MILTER_LDAP_BINDPW' not in os.environ:
+      logging.info("ENV[MILTER_LDAP_BINDPW] not set! Continue...")
+    else:
+      g_milter_ldap_bindpw = os.environ['MILTER_LDAP_BINDPW']
+    if 'MILTER_LDAP_SEARCH_BASE' not in os.environ:
+      logging.error("ENV[MILTER_LDAP_SEARCH_BASE] is mandatory!")
+      sys.exit(1)
+    g_milter_ldap_search_base = os.environ['MILTER_LDAP_SEARCH_BASE']
+    if 'MILTER_LDAP_QUERY' not in os.environ:
+      logging.error("ENV[MILTER_LDAP_QUERY] is mandatory!")
+      sys.exit(1)
+    g_milter_ldap_query = os.environ['MILTER_LDAP_QUERY']
+    if 'MILTER_LDAP_TENANT_ID_ATTR' in os.environ:
+      g_milter_ldap_tenant_id_attr = os.environ['MILTER_LDAP_TENANT_ID_ATTR']
+    if 'MILTER_LDAP_DKIM_ENABLED_ATTR' in os.environ:
+      g_milter_ldap_dkim_enabled_attr = os.environ['MILTER_LDAP_DKIM_ENABLED_ATTR']
+    if 'MILTER_LDAP_DKIM_ALIGNMENT_REQUIRED_ATTR' in os.environ:
+      g_milter_ldap_dkim_alignment_required_attr = os.environ['MILTER_LDAP_DKIM_ALIGNMENT_REQUIRED_ATTR']
+    try:
+      set_config_parameter("RESTARTABLE_SLEEPTIME", 2)
+      set_config_parameter("RESTARTABLE_TRIES", True)
+      set_config_parameter('DEFAULT_SERVER_ENCODING', 'utf-8')
+      set_config_parameter('DEFAULT_CLIENT_ENCODING', 'utf-8')
+      server = Server(g_milter_ldap_server_uri, get_info=NONE)
+      g_milter_ldap_conn = Connection(server,
+        g_milter_ldap_binddn,
+        g_milter_ldap_bindpw,
+        auto_bind=True, 
+        raise_exceptions=True,
+        client_strategy='RESTARTABLE'
+      )
+      logging.info("LDAP-Connection established")
+      try:
+        g_policy_backend = ExOTAPolicyBackendLDAP({
+          'ldap_conn': g_milter_ldap_conn,
+          'ldap_search_base': g_milter_ldap_search_base,
+          'ldap_query': g_milter_ldap_query,
+          'ldap_tenant_id_attr': g_milter_ldap_tenant_id_attr,
+          'ldap_dkim_enabled_attr': g_milter_ldap_dkim_enabled_attr,
+          'ldap_dkim_alignment_required_attr': g_milter_ldap_dkim_alignment_required_attr
+        })
+        logging.info("LDAP policy backend initialized")
+      except ExOTAPolicyException as e:
+        logging.error("Policy backend error: {0}".format(e.message))
+        sys.exit(1)
+    except LDAPException as e:
+      print("LDAP-Exception: " + traceback.format_exc())
+      sys.exit(1)
   else:
     logging.debug("Unsupported backend: {0}!".format(g_milter_policy_source))
     sys.exit(1)

app/policy.py

@@ -2,6 +2,7 @@ import json
 import traceback
 import re
 from uuid import UUID
+from ldap3.core.exceptions import LDAPException
 
 class ExOTAPolicyException(Exception):
   def __init__(self, message):
@@ -13,6 +14,10 @@ class ExOTAPolicyNotFoundException(ExOTAPolicyException):
 class ExOTAPolicyInvalidException(ExOTAPolicyException):
   pass
 
+class ExOTAPolicyBackendException(Exception):
+  def __init__(self, message):
+    self.message = message
+
 class ExOTAPolicy():
   def __init__(self, policy_dict):
     self.tenant_id = policy_dict['tenant_id']
@@ -120,4 +125,57 @@ class ExOTAPolicyBackendJSON(ExOTAPolicyBackend):
 ########## LDAP
 class ExOTAPolicyBackendLDAP(ExOTAPolicyBackendJSON):
   type = 'ldap'
-  pass
+  def __init__(self, ldap_config):
+    try:
+      self.conn = ldap_config['ldap_conn']
+      self.search_base = ldap_config['ldap_search_base']
+      self.query = ldap_config['ldap_query']
+      self.tenant_id_attr = ldap_config['ldap_tenant_id_attr']
+      self.dkim_enabled_attr = ldap_config['ldap_dkim_enabled_attr']
+      self.dkim_alignment_required_attr = ldap_config['ldap_dkim_alignment_required_attr']
+    except Exception as e:
+      raise ExOTAPolicyException(
+        "An error occured while initializing LDAP backend: " + traceback.format_exc()
+      ) from e
+
+  def get(self, from_domain):
+    self.query = self.query.replace('%d', from_domain)
+    try:
+      self.conn.search(
+        self.search_base,
+        self.query,
+        attributes=[
+          self.tenant_id_attr,
+          self.dkim_enabled_attr,
+          self.dkim_alignment_required_attr
+        ]
+      )
+      if len(self.conn.entries) == 1:
+        entry = self.conn.entries[0]
+        policy_dict = {}
+        if self.tenant_id_attr in entry:
+          policy_dict['tenant_id'] = entry[self.tenant_id_attr].value
+        if self.dkim_enabled_attr in entry:
+          if entry[self.dkim_enabled_attr].value == 'TRUE':
+            policy_dict['dkim_enabled'] = True
+          else:
+            policy_dict['dkim_enabled'] = False
+        if self.dkim_alignment_required_attr in entry:
+          if entry[self.dkim_alignment_required_attr].value == 'TRUE':
+            policy_dict['dkim_alignment_required'] = True
+          else:
+            policy_dict['dkim_alignment_required'] = False
+        ExOTAPolicy.check_policy(policy_dict)
+        return ExOTAPolicy(policy_dict)
+      if len(self.conn.entries) > 1:
+        raise ExOTAPolicyInvalidException(
+          "Multiple policies found for domain={0}!".format(from_domain)
+        )
+      else:
+        raise ExOTAPolicyNotFoundException(
+          "Policy for domain={0} not found".format(from_domain)
+        )
+    except LDAPException as e:
+      raise ExOTAPolicyBackendException(
+        "asdf"
+      ) from e

requirements.txt

@@ -1,2 +1,3 @@
 authres==1.2.0
 pymilter==1.0.4
+ldap3

tests/miltertest.lua

@@ -30,7 +30,10 @@ if mt.getreply(conn) ~= SMFIR_CONTINUE then
 end
 
 -- HEADER
-if mt.header(conn, "fRoM", '"Blah Blubb" <O365ConnectorValidation@yad.onmicrosoft.com>') ~= nil then
+--if mt.header(conn, "fRoM", '"Blah Blubb" <O365ConnectorValidation@yad.onmicrosoft.com>') ~= nil then
+--  error "mt.header(From) failed"  
+--end
+if mt.header(conn, "fRoM", '"Blah Blubb" <O365ConnectorValidation@chillout2k.de>') ~= nil then
   error "mt.header(From) failed"  
 end
 if mt.header(conn, "resent-fRoM", '"Blah Blubb" <blah@yad.onmicrosoft.COM>') ~= nil then
@@ -54,6 +57,9 @@ end
 if mt.header(conn, "Authentication-RESULTS", "my-auth-serv-id;\n  dkim=pass header.d=yad.onmicrosoft.comx header.s=selector1-yad-onmicrosoft-com header.b=mmmjFpv8") ~= nil then
   error "mt.header(Subject) failed"  
 end
+if mt.header(conn, "Authentication-RESULTS", "my-auth-serv-id;\n  dkim=pass header.d=chillout2k.dex header.s=selector1-yad-onmicrosoft-com header.b=mmmjFpv8") ~= nil then
+  error "mt.header(Subject) failed"  
+end
 if mt.header(conn, "Authentication-Results", "my-auth-serv-id;\n  dkim=fail header.d=yad.onmicrosoft.com header.s=selector2-asdf header.b=mmmjFpv8") ~= nil then
   error "mt.header(Subject) failed"  
 end

tests/policy.json

@@ -2,7 +2,7 @@
   "yad.onmicrosoft.com": {
     "tenant_id": "1234abcd-18c5-45e8-88de-123456789abc",
     "dkim_enabled": true,
-    "dkim_alignment_required": true
+    "dkim_alignment_required": false
   },
   "example.com": {
     "tenant_id": "abcd1234-18c5-45e8-88de-987654321cba",