API bug fixes and security issue resolution (#1)

* auth bug fix

* added release notes

* security updates

* security fix

* ignore mitmproxy CA

* support for custom CA

* support for custom CA

* redact auth token from logs

Author: nleach999

.gitignore

@@ -4,7 +4,7 @@ run/*
 **/cxone_oauth_client_secret
 **/cxone_tenant
 env
-
+mitm*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,11 +1,14 @@
-FROM python:3.12
+FROM ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/checkmarx-ts/cxone-scan-scheduler
 LABEL org.opencontainers.image.vendor Checkmarx Professional Services
 LABEL org.opencontainers.image.title Checkmarx One Scan Scheduler
 LABEL org.opencontainers.image.description Schedules scans for projects in Checkmarx One
 
+USER root
 
-RUN apt-get update && apt-get install -y cron && apt-get clean && \
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \
+    apt-get install -y cron python3.12 python3-pip python3-debugpy bash && \
     usermod -s /bin/bash nobody && \
     mkdir -p /opt/cxone && \
     mkfifo /opt/cxone/logfifo && \
@@ -14,8 +17,12 @@ RUN apt-get update && apt-get install -y cron && apt-get clean && \
 
 WORKDIR /opt/cxone
 COPY *.txt /opt/cxone
-RUN pip install debugpy && pip install -r requirements.txt
 
+RUN pip install -r requirements.txt --no-cache-dir --break-system-packages && \
+    apt-get remove -y perl && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    dpkg --purge $(dpkg --get-selections | grep deinstall | cut -f1)
 
 COPY cxone_api /opt/cxone/cxone_api
 COPY logic /opt/cxone/logic
@@ -28,6 +35,5 @@ COPY *.json /opt/cxone
 RUN ln -s scheduler.py scheduler && \
     ln -s scheduler.py audit
 
-# ENTRYPOINT ["python", "-Xfrozen_modules=off", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client"]
 CMD ["scheduler"]
 ENTRYPOINT ["/opt/cxone/entrypoint.sh"]

README.md

@@ -144,6 +144,25 @@ The Scan Scheduler runs as a container.  At startup, it crawls the tenant's proj
 checks periodically for any schedule changes and updates 
 the scan schedules accordingly.
 
+### Add Optional Trusted CA Certificates
+
+While the CheckmarxOne system uses TLS certificates signed by a public CA, it is possible that corporate
+proxies use certificates signed by a private CA.  If so, it is possible to import custom CA certificates
+when the scheduler starts.
+
+The custom certificates must meet the following criteria:
+
+* Must be in the PEM format.
+* Must be in a file ending with the extension `.crt`.
+* Only one certificate is in the file.
+* Must be mapped to the container path `/usr/local/share/ca-certificates`.
+
+As an example, if using Docker, it is possible to map a local file to a file in the container with this
+mapping option added to the container execution command line:
+
+`-v $(pwd)/custom-ca.pem:/usr/local/share/ca-certificates/custom-ca.crt`
+
+
 ### Required Secrets
 
 Docker secrets are used to securely store secrets needed during runtime.

RELEASE_NOTES.md

@@ -0,0 +1,13 @@
+# Release Notes
+
+## v1.1
+
+* Bug fix for auth token not refreshing properly.
+* Added log output to indicate number of scheduled scans on start and schedule update.
+* Added support for loading custom CA certificates at startup.
+
+## v1.0
+
+Initial release
+
+

cxone_api/__init__.py

@@ -1,4 +1,4 @@
-import asyncio, uuid, requests, urllib, datetime
+import asyncio, uuid, requests, urllib, datetime, re
 from requests.compat import urljoin
 from pathlib import Path
 
@@ -8,8 +8,25 @@ class AuthException(BaseException):
     pass
 
 class CommunicationException(BaseException):
-    pass
 
+    @staticmethod
+    def __clean(content):
+        if type(content) is list:
+            return [CommunicationException.__clean(x) for x in content]
+        elif type(content) is tuple:
+            return (CommunicationException.__clean(x) for x in content)
+        elif type(content) is dict:
+            return {k:CommunicationException.__clean(v) for k,v in content.items()}
+        elif type(content) is str:
+            if re.match("^Bearer.*", content):
+                return "REDACTED"
+            else:
+                return content
+        else:
+            return content
+
+    def __init__(self, op, *args, **kwargs):
+        BaseException.__init__(self, f"Operation: {op.__name__} args: [{CommunicationException.__clean(args)}] kwargs: [{CommunicationException.__clean(kwargs)}]")
 
 
 class CxOneAuthEndpoint:
@@ -172,37 +189,61 @@ async def paged_api(coro, array_element, offset_field='offset', **kwargs):
         yield buf.pop()
 
 
-        
-
 class CxOneClient:
     __AGENT_NAME = 'CxOne PyClient'
 
-    def __init__(self, oauth_id, oauth_secret, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True):
 
+    def __common__init(self, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify):
         with open(Path(__file__).parent / "version.txt", "rt") as version:
             self.__version = version.readline().rstrip()
 
         self.__agent = f"{agent_name}/{agent_version}/({CxOneClient.__AGENT_NAME}/{self.__version})"
         self.__proxy = proxy
         self.__ssl_verify = ssl_verify
-
         self.__auth_lock = asyncio.Lock()
         self.__corelation_id = str(uuid.uuid4())
 
         self.__auth_endpoint = tenant_auth_endpoint
         self.__api_endpoint = api_endpoint
         self.__timeout = timeout
         self.__retries = retries
-        
-        
-        self.__auth_content = urllib.parse.urlencode( {
+
+        self.__auth_endpoint = tenant_auth_endpoint
+        self.__api_endpoint = api_endpoint
+        self.__timeout = timeout
+        self.__retries = retries
+
+        self.__auth_result = None
+
+
+
+
+
+    @staticmethod
+    def create_with_oauth(oauth_id, oauth_secret, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True):
+        inst = CxOneClient()
+        inst.__common__init(agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify)
+
+        inst.__auth_content = urllib.parse.urlencode( {
             "grant_type" : "client_credentials",
             "client_id" : oauth_id,
             "client_secret" : oauth_secret
         })
-        
-        self.__auth_result = None
 
+        return inst
+
+    @staticmethod
+    def create_with_api_key(api_key, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True):
+        inst = CxOneClient()
+        inst.__common__init(agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify)
+
+        inst.__auth_content = urllib.parse.urlencode( {
+            "grant_type" : "refresh_token",
+            "client_id" : "ast-app",
+            "refresh_token" : api_key
+        })
+
+        return inst
 
     @property
     def auth_endpoint(self):
@@ -259,14 +300,22 @@ async def __exec_request(self, op, *args, **kwargs):
         kwargs['verify'] = self.__ssl_verify
 
         for _ in range(0, self.__retries):
+            auth_headers = await self.__get_request_headers()
+
+            if 'headers' in kwargs.keys():
+                for h in auth_headers.keys():
+                    kwargs['headers'][h] = auth_headers[h]
+            else:
+                kwargs['headers'] = auth_headers
+            
             response = await asyncio.to_thread(op, *args, **kwargs)
 
             if response.status_code == 401:
                 await self.__do_auth()
             else:
                 return response
 
-        raise CommunicationException(f"{str(op)}{str(args)}{str(kwargs)}")
+        raise CommunicationException(op, *args, **kwargs)
 
 
     @staticmethod
@@ -288,47 +337,91 @@ def __join_query_dict(url, querydict):
 
         return urljoin(url, f"?{'&'.join(query)}" if len(query) > 0 else '')
 
-   
+    @dashargs("tags-keys", "tags-values")
+    async def get_applications(self, **kwargs):
+        url = urljoin(self.api_endpoint, "applications")
+        url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
+
+    async def get_application(self, id, **kwargs):
+        url = urljoin(self.api_endpoint, f"applications/{id}")
+        url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
+
     @dashargs("repo-url", "name-regex", "tags-keys", "tags-values")
     async def get_projects(self, **kwargs):
         url = urljoin(self.api_endpoint, "projects")
-
         url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
+
 
-        return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() )
+    async def get_project(self, projectid):
+        url = urljoin(self.api_endpoint, f"projects/{projectid}")
+        return await self.__exec_request(requests.get, url)
 
-    async def get_project(self, id):
-        url = urljoin(self.api_endpoint, f"projects/{id}")
-        return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() )
+    async def get_project_configuration(self, projectid):
+        url = urljoin(self.api_endpoint, f"configuration/project?project-id={projectid}")
+        return await self.__exec_request(requests.get, url)
 
-    async def get_project_configuration(self, id):
-        url = urljoin(self.api_endpoint, f"configuration/project?project-id={id}")
-        return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() )
+    async def get_tenant_configuration(self):
+        url = urljoin(self.api_endpoint, f"configuration/tenant")
+        return await self.__exec_request(requests.get, url)
 
     @dashargs("from-date", "project-id", "project-ids", "scan-ids", "project-names", "source-origin", "source-type", "tags-keys", "tags-values", "to-date")
     async def get_scans(self, **kwargs):
         url = urljoin(self.api_endpoint, "scans")
-
         url = CxOneClient.__join_query_dict(url, kwargs)
-
-        return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() )
+        return await self.__exec_request(requests.get, url)
 
+    @dashargs("scan-ids")
+    async def get_sast_scans_metadata(self, **kwargs):
+        url = urljoin(self.api_endpoint, "sast-metadata")
+        url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
 
-    async def execute_scan(self, payload, **kwargs):
+    async def get_sast_scan_metadata(self, scanid, **kwargs):
+        url = urljoin(self.api_endpoint, f"sast-metadata/{scanid}")
+        url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
+
+    @dashargs("source-node-operation", "source-node", "source-line-operation", "source-line", "source-file-operation", "source-file", \
+              "sink-node-operation", "sink-node", "sink-line-operation", "sink-line", "sink-file-operation", \
+                "sink-file", "result-ids", "preset-id", "number-of-nodes-operation", "number-of-nodes", "notes-operation", \
+                    "first-found-at-operation", "first-found-at", "apply-predicates")
+    async def get_sast_scan_aggregate_results(self, scanid, groupby_field=['SEVERITY'], **kwargs):
+        url = urljoin(self.api_endpoint, f"sast-scan-summary/aggregate")
+        url = CxOneClient.__join_query_dict(url, kwargs | { 
+            'scan-id' : scanid,
+            'group-by-field' : groupby_field
+        })
+        return await self.__exec_request(requests.get, url)
 
+    async def execute_scan(self, payload, **kwargs):
         url = urljoin(self.api_endpoint, "scans")
         url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.post, url, json=payload)
 
-        return await self.__exec_request(requests.post, url, json=payload, headers=await self.__get_request_headers() )
-
-    
     async def get_sast_scan_log(self, scanid, stream=False):
         url = urljoin(self.api_endpoint, f"logs/{scanid}/sast")
-        return await self.__exec_request(requests.get, url, stream=stream, headers=await self.__get_request_headers() )
+        response = await self.__exec_request(requests.get, url)
+
+        if response.ok and response.status_code == 307:
+            response = await self.__exec_request(requests.get, response.headers['Location'], stream=stream)
+
+        return response
+
+    async def get_groups(self, **kwargs):
+        url = CxOneClient.__join_query_dict(urljoin(self.admin_endpoint, "groups"), kwargs)
+        return await self.__exec_request(requests.get, url)
 
     async def get_groups(self, **kwargs):
         url = CxOneClient.__join_query_dict(urljoin(self.admin_endpoint, "groups"), kwargs)
-        return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() )
+        return await self.__exec_request(requests.get, url)
+
+    async def get_scan_workflow(self, scanid, **kwargs):
+        url = urljoin(self.api_endpoint, f"scans/{scanid}/workflow")
+        url = CxOneClient.__join_query_dict(url, kwargs)
+        return await self.__exec_request(requests.get, url)
 
 class ProjectRepoConfig:
 
@@ -360,4 +453,4 @@ async def primary_branch(self):
     @property
     async def repo_url(self):
         url = await self.__get_logical_repo_url()
-        return url if len(url) > 0 else None
+        return url if len(url) > 0 else None

entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+update-ca-certificates
+export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+echo "export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt" >> /etc/environment
+
 [ -n "$CXONE_REGION" ] && echo "export CXONE_REGION=$CXONE_REGION" >> /etc/environment
 [ -n "$SINGLE_TENANT_AUTH" ] && echo "export SINGLE_TENANT_AUTH=$SINGLE_TENANT_AUTH" >> /etc/environment
 [ -n "$SINGLE_TENANT_API" ] && echo "export SINGLE_TENANT_API=$SINGLE_TENANT_API" >> /etc/environment
@@ -13,4 +17,4 @@ fi
 
 service cron start > /dev/null 2>&1
 
-python $@
+python3 $@

logic/__init__.py

@@ -1,7 +1,6 @@
 import logging, asyncio, utils
 from cxone_api import paged_api, ProjectRepoConfig
 
-
 class Scheduler:
     __log = logging.getLogger("Scheduler")
 
@@ -177,6 +176,10 @@ async def refresh_schedule(self):
 
         return len(new_scheduled_projects), len(removed_projects), len(changed_schedule.keys())
 
+    @property
+    def scheduled_scans(self):
+        return len(self.__the_schedule.keys())
+        
 
     async def __load_schedule(self, bad_cb = None):
         tagged, grouped = await asyncio.gather(self.__get_tagged_project_schedule(bad_cb), self.__get_untagged_project_schedule(bad_cb))

scanner.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/usr/bin/python3
 import logging, argparse, utils, asyncio
 from cxone_api import CxOneClient
 from posix_ipc import Semaphore, BusyError, O_CREAT
@@ -38,7 +38,7 @@ async def main():
         with open("version.txt", "rt") as ver:
             version = ver.readline().strip()
 
-        client = CxOneClient(oauth_id, oauth_secret, agent, version, auth_endpoint, 
+        client = CxOneClient.create_with_oauth(oauth_id, oauth_secret, agent, version, auth_endpoint, 
                             api_endpoint, ssl_verify=ssl_verify, proxy=proxy)