Merge pull request #23 from chillout2k/forwarding_patch

Processing of forwarded messages

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '39 12 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/ossar-analysis.yml

@@ -0,0 +1,49 @@
+# This workflow integrates a collection of open source static analysis tools
+# with GitHub code scanning. For documentation, or to provide feedback, visit
+# https://github.com/github/ossar-action
+name: OSSAR
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  OSSAR-Scan:
+    # OSSAR runs on windows-latest.
+    # ubuntu-latest and macos-latest support coming soon
+    runs-on: windows-latest
+
+    steps:
+      # Checkout your code repository to scan
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Ensure a compatible version of dotnet is installed.
+    # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201.
+    # A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action.
+    # GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped.
+    # For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action:
+    # - name: Install .NET
+    #   uses: actions/setup-dotnet@v1
+    #   with:
+    #     dotnet-version: '3.1.x'
+
+      # Run open source static analysis tools
+    - name: Run OSSAR
+      uses: github/ossar-action@v1
+      id: ossar
+
+      # Upload results to the Security tab
+    - name: Upload OSSAR results
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: ${{ steps.ossar.outputs.sarifFile }}

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "python.pythonPath": "/home/dominik/src/git/ExOTA-Milter/venv/bin/python3"
+    "python.pythonPath": "/home/dominik/src/github/ExOTA-Milter/venv/bin/python3"
 }

OCI/README.md

@@ -47,6 +47,7 @@ services:
       MILTER_TRUSTED_AUTHSERVID: 'my-auth-serv-id'
       MILTER_X509_ENABLED: 'some_value'
       MILTER_X509_TRUSTED_CN: 'mail.protection.outlook.com'
+      MILTER_X509_IP_WHITELIST='127.0.0.1,::1'
       MILTER_ADD_HEADER: 'some_value'
       MILTER_AUTHSERVID: 'my-auth-serv-id'
     volumes:

activity_policy.puml

@@ -8,7 +8,7 @@ start
 note left: From, Authentication-Results, X-MS-Exchange-CrossTenant-Id
 
 :HDR: Recognising sender domain;
-note left: Taken from RFC5322 From-header. RFC5321.mail (envelope) is NOT relevant!
+note left: Taken from RFC5322.From header and/or RFC5322.Resent-From header. RFC5321.mail (envelope) is NOT relevant!
 
 :EOM: Looking up policy in backend;
 note left: Based on RFC5322.from domain

app/exota-milter.py

@@ -37,6 +37,8 @@ g_milter_policy_file = '/data/policy.json'
 g_milter_x509_enabled = False
 # ENV[MILTER_X509_TRUSTED_CN]
 g_milter_x509_trusted_cn = 'mail.protection.outlook.com'
+# ENV[MILTER_X509_IP_WHITELIST]
+g_milter_x509_ip_whitelist = ['127.0.0.1','::1']
 # ENV[MILTER_ADD_HEADER]
 g_milter_add_header = False
 # ENV[MILTER_AUTHSERVID]
@@ -57,9 +59,15 @@ class ExOTAMilter(Milter.Base):
     self.conn_reused = False
     self.hdr_from = None
     self.hdr_from_domain = None
+    self.hdr_resent_from = None
+    self.hdr_resent_from_domain = None
+    self.forwarded = False
     self.hdr_tenant_id = None
     self.hdr_tenant_id_count = 0
+    self.x509_whitelisted = False
     self.dkim_valid = False
+    self.passed_dkim_results = []
+    self.dkim_aligned = False
     self.xar_hdr_count = 0
     # https://stackoverflow.com/a/2257449
     self.mconn_id = g_milter_name + ': ' + ''.join(
@@ -76,7 +84,7 @@ class ExOTAMilter(Milter.Base):
     if 'reason' in kwargs:
       message = "{0} - reason: {1}".format(message, kwargs['reason'])
     logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-      ": milter_action=reject"
+      ": milter_action=reject message={0}".format(message)
     )
     self.setreply('550','5.7.1', message)
     return Milter.REJECT
@@ -138,6 +146,23 @@ class ExOTAMilter(Milter.Base):
           self.hdr_from, self.hdr_from_domain
         )
       )
+    
+    # Parse RFC-5322-Resent-From header (Forwarded)
+    if(name.lower() == "Resent-From".lower()):
+      hdr_5322_resent_from = email.utils.parseaddr(hval)
+      self.hdr_resent_from = hdr_5322_resent_from[1].lower()
+      m = re.match(g_re_domain, self.hdr_resent_from)
+      if m is None:
+        logging.error(self.mconn_id  + "/" + str(self.getsymval('i')) + "/HDR " +
+          "Could not determine domain-part of 5322.resent_from=" + self.hdr_resent_from
+        )
+      else:
+        self.hdr_resent_from_domain = m.group(1).lower()
+        logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+          "/HDR: 5322.resentfrom={0}, 5322.resent_from_domain={1}".format(
+            self.hdr_resent_from, self.hdr_resent_from_domain
+          )
+        )
 
     # Parse non-standardized X-MS-Exchange-CrossTenant-Id header
     elif(name.lower() == "X-MS-Exchange-CrossTenant-Id".lower()):
@@ -159,6 +184,12 @@ class ExOTAMilter(Milter.Base):
             for ar_result in ar.results:
               if ar_result.method == 'dkim':
                 if ar_result.result == 'pass':
+                  self.passed_dkim_results.append({
+                    "sdid": ar_result.header_d
+                  })
+                  logging.debug(self.mconn_id + "/" + str(self.getsymval('i')) +
+                    "/HDR: DKIM passed SDID {0}".format(ar_result.header_d)
+                  )
                   self.dkim_valid = True
           else:
             logging.debug(self.mconn_id + "/" + str(self.getsymval('i')) +
@@ -183,33 +214,42 @@ class ExOTAMilter(Milter.Base):
 
     # Check if client certificate CN matches trusted CN
     if g_milter_x509_enabled:
-      cert_subject = self.getsymval('{cert_subject}') 
-      if cert_subject is None:
-        logging.info(self.mconn_id + "/" + str(self.getsymval('i')) 
-          + "/EOM: No trusted x509 client CN found - action=reject"
-        )
-        return self.smfir_reject(
-          queue_id = self.getsymval('i'),
-          reason = 'No trusted x509 client CN found'
-        )
-      else:
-        if g_milter_x509_trusted_cn.lower() == cert_subject.lower():
-          self.x509_client_valid = True
-          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-            "/EOM: Trusted x509 client CN {0}".format(cert_subject)
+      for whitelisted_client_ip in g_milter_x509_ip_whitelist:
+        if self.client_ip == whitelisted_client_ip:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) 
+            + "/EOM: x509 CN check: client-IP '{0}' is whitelisted".format(
+              whitelisted_client_ip
+            )
           )
-        else:
-          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-            "/EOM Untrusted x509 client CN {0} - action=reject".format(cert_subject)
+          self.x509_whitelisted = True
+      if not self.x509_whitelisted:
+        cert_subject = self.getsymval('{cert_subject}') 
+        if cert_subject is None:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) 
+            + "/EOM: No trusted x509 client CN found - action=reject"
           )
           return self.smfir_reject(
             queue_id = self.getsymval('i'),
-            reason = "Untrusted x509 client CN: {0}".format(cert_subject)
+            reason = 'No trusted x509 client CN found'
           )
+        else:
+          if g_milter_x509_trusted_cn.lower() == cert_subject.lower():
+            self.x509_client_valid = True
+            logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+              "/EOM: Trusted x509 client CN {0}".format(cert_subject)
+            )
+          else:
+            logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+              "/EOM: Untrusted x509 client CN {0} - action=reject".format(cert_subject)
+            )
+            return self.smfir_reject(
+              queue_id = self.getsymval('i'),
+              reason = "Untrusted x509 client CN: {0}".format(cert_subject)
+            )
 
     if self.hdr_from is None:
       logging.error(self.mconn_id + "/" + str(self.getsymval('i')) +
-        "/EOM exception: could not determine 5322.from header - action=reject"
+        "/EOM: exception: could not determine 5322.from header - action=reject"
       )
       return self.smfir_reject(
         queue_id = self.getsymval('i'),
@@ -221,20 +261,48 @@ class ExOTAMilter(Milter.Base):
     try:
       policy = g_policy_backend.get(self.hdr_from_domain)
       logging.debug(self.mconn_id + "/" + str(self.getsymval('i')) +
-        "/EOM Policy for 5322.from_domain={0} fetched from backend".format(self.hdr_from_domain)
+        "/EOM: Policy for 5322.from_domain={0} fetched from backend".format(
+          self.hdr_from_domain
+        )
       )
     except (ExOTAPolicyException, ExOTAPolicyNotFoundException) as e:
       logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-        "/EOM {0}".format(e.message)
-      )
-      return self.smfir_reject(
-        queue_id = self.getsymval('i'),
-        reason = "No policy for {0}".format(self.hdr_from_domain)
+        "/EOM: 5322.from: {0}".format(e.message)
       )
+      # Forwarded message? Maybe the Resent-From header domain matches.
+      if self.hdr_resent_from_domain is not None:
+        try:
+          policy = g_policy_backend.get(self.hdr_resent_from_domain)
+          logging.debug(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: Policy for 5322.resent_from_domain={0} fetched from backend".format(
+              self.hdr_resent_from_domain
+            )
+          )
+          self.forwarded = True
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: Forwarded message -> Policy for 5322.resent_from_domain={0} found.".format(
+              self.hdr_resent_from_domain
+            )
+          )
+        except (ExOTAPolicyException, ExOTAPolicyNotFoundException) as e:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: 5322.resent-from: {0}".format(e.message)
+          )
+          return self.smfir_reject(
+            queue_id = self.getsymval('i'),
+            reason = "No policy for 5322.resent_from_domain {0}".format(
+              self.hdr_resent_from_domain
+            )
+          ) 
+      else:
+        return self.smfir_reject(
+          queue_id = self.getsymval('i'),
+          reason = "No policy for 5322.from_domain {0}".format(self.hdr_from_domain)
+        )
 
     if self.hdr_tenant_id is None:
       logging.error(self.mconn_id + "/" + str(self.getsymval('i')) +
-        "/EOM exception: could not determine X-MS-Exchange-CrossTenant-Id - action=reject"
+        "/EOM: exception: could not determine X-MS-Exchange-CrossTenant-Id - action=reject"
       )
       return self.smfir_reject(
         queue_id = self.getsymval('i'),
@@ -268,15 +336,23 @@ class ExOTAMilter(Milter.Base):
       )
       if self.dkim_valid:
         logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-          "/EOM: Found valid DKIM authentication result for 5322.from_domain={0}".format(
-            self.hdr_from_domain
-          )
+          "/EOM: Valid DKIM signatures found"
         )
+        for passed_dkim_result in self.passed_dkim_results:
+          if self.hdr_from_domain == passed_dkim_result['sdid']:
+            logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+              "/EOM: Found aligned DKIM signature for SDID: {0}".format(
+                passed_dkim_result['sdid']
+              ) 
+            )
+            self.dkim_aligned = True
+        if not self.dkim_aligned:
+          logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
+            "/EOM: No aligned DKIM signatures found!"
+          )
       else:
         logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-          "/EOM: No valid DKIM authentication result found for 5322.from_domain={0}".format(
-            self.hdr_from_domain
-          )
+          "/EOM: No valid DKIM authentication result found"
         )
         return self.smfir_reject(
           queue_id = self.getsymval('i'),
@@ -297,12 +373,16 @@ class ExOTAMilter(Milter.Base):
 
     if g_milter_add_header:
       try:
-        self.addheader("X-ExOTA-Authentication-Results", 
-          "{0};\n  auth=pass header.d={1} dkim={2} x509_client_trust={3}".format(
-            g_milter_authservid, self.hdr_from_domain, policy.is_dkim_enabled(),
-            g_milter_x509_enabled
-          )
+        addhdr_value = str(
+          "{0};\n" + 
+          "  auth=pass 5322_from_domain={1} dkim={2} dkim_aligned={3} " + 
+          "x509_client_trust={4} forwarded={5}"
+        ).format(
+          g_milter_authservid, self.hdr_from_domain, policy.is_dkim_enabled(),
+          self.dkim_aligned, g_milter_x509_enabled, self.forwarded
         )
+        logging.debug(addhdr_value)
+        self.addheader("X-ExOTA-Authentication-Results", addhdr_value)
         logging.debug(self.mconn_id + "/" + str(self.getsymval('i')) +
           "/EOM: AR-header added"
         )
@@ -313,8 +393,8 @@ class ExOTAMilter(Milter.Base):
 
     if g_milter_dkim_enabled:
       logging.info(self.mconn_id + "/" + str(self.getsymval('i')) +
-        "/EOM: Tenant successfully authorized (dkim_enabled={0})".format(
-          str(policy.is_dkim_enabled())
+        "/EOM: Tenant successfully authorized (dkim_enabled={0} dkim_aligned={1})".format(
+          policy.is_dkim_enabled(), self.dkim_aligned
         )
       )
     else:
@@ -375,6 +455,10 @@ if __name__ == "__main__":
     if 'MILTER_X509_TRUSTED_CN' in os.environ:
       g_milter_x509_trusted_cn = os.environ['MILTER_X509_TRUSTED_CN']
     logging.info("ENV[MILTER_X509_TRUSTED_CN]: {0}".format(g_milter_x509_trusted_cn))
+    if 'MILTER_X509_IP_WHITELIST' in os.environ:
+      g_milter_x509_ip_whitelist = "".join(os.environ['MILTER_X509_IP_WHITELIST'].split())
+      g_milter_x509_ip_whitelist = g_milter_x509_ip_whitelist.split(',')
+    logging.info("ENV[MILTER_X509_IP_WHITELIST]: {0}".format(g_milter_x509_ip_whitelist))
   logging.info("ENV[MILTER_X509_ENABLED]: {0}".format(g_milter_x509_enabled))
   if 'MILTER_POLICY_SOURCE' in os.environ:
     g_milter_policy_source = os.environ['MILTER_POLICY_SOURCE']

app/policy.py

@@ -89,7 +89,7 @@ class ExOTAPolicyBackendJSON(ExOTAPolicyBackend):
       return ExOTAPolicy(self.policies[from_domain])
     except KeyError as e:
       raise ExOTAPolicyNotFoundException(
-        "Policy for from_domain={0} not found".format(from_domain)
+        "Policy for domain={0} not found".format(from_domain)
       ) from e
     except Exception as e:
       raise ExOTAPolicyException(

tests/README.md

@@ -19,6 +19,7 @@ export MILTER_DKIM_ENABLED=yepp
 export MILTER_TRUSTED_AUTHSERVID=my-auth-serv-id
 export MILTER_X509_ENABLED=yepp
 export MILTER_X509_TRUSTED_CN=mail.protection.outlook.com
+export MILTER_X509_IP_WHITELIST='127.0.0.1,::1'
 export MILTER_ADD_HEADER=yepp
 export MILTER_AUTHSERVID=my-auth-serv-id
 ```

tests/miltertest.lua

@@ -6,6 +6,9 @@ conn = mt.connect(socket)
 if conn == nil then
   error "mt.connect() failed"
 end
+if mt.conninfo(conn, "localhost", "::1") ~= nil then
+  error "mt.conninfo() failed"
+end
 
 mt.set_timeout(3)
 
@@ -18,7 +21,7 @@ if mt.getreply(conn) ~= SMFIR_CONTINUE then
 end
 
 -- 5321.RCPT+MACROS
-mt.macro(conn, SMFIC_RCPT, '{client_addr}', "127.128.129.130", "i", "4CgSNs5Q9sz7SllQ", '{cert_subject}', "mail.protection.outlook.com")
+mt.macro(conn, SMFIC_RCPT, "i", "4CgSNs5Q9sz7SllQ", '{cert_subject}', "mail.protection.outlook.comx")
 if mt.rcptto(conn, "<envelope.recipient@example.com>") ~= nil then
   error "mt.rcptto() failed"
 end
@@ -30,6 +33,9 @@ end
 if mt.header(conn, "fRoM", '"Blah Blubb" <O365ConnectorValidation@yad.onmicrosoft.com>') ~= nil then
   error "mt.header(From) failed"  
 end
+if mt.header(conn, "aaa-resent-fRoM", '"Blah Blubb" <blah@yad.onmicrosoft.COMa>') ~= nil then
+  error "mt.header(From) failed"  
+end
 if mt.header(conn, "x-mS-EXCHANGE-crosstenant-id", "1234abcd-18c5-45e8-88de-123456789abc") ~= nil then
   error "mt.header(Subject) failed"  
 end
@@ -45,7 +51,7 @@ end
 if mt.header(conn, "Authentication-Results", "my-auth-serv-id;\n  exota=pass") ~= nil then
   error "mt.header(Subject) failed"  
 end
-if mt.header(conn, "Authentication-RESULTS", "my-auth-serv-id;\n  dkim=pass header.d=yad.onmicrosoft.com header.s=selector1-yad-onmicrosoft-com header.b=mmmjFpv8") ~= nil then
+if mt.header(conn, "Authentication-RESULTS", "my-auth-serv-id;\n  dkim=pass header.d=yad.onmicrosoft.comx header.s=selector1-yad-onmicrosoft-com header.b=mmmjFpv8") ~= nil then
   error "mt.header(Subject) failed"  
 end
 if mt.header(conn, "Authentication-Results", "my-auth-serv-id;\n  dkim=fail header.d=yad.onmicrosoft.com header.s=selector2-asdf header.b=mmmjFpv8") ~= nil then