MacCatalyst job fails due to python version mismatch

[kotlarmilos]

Description

Helix work items run successfully, but fail in post-processing xharness-event-processor.py on several machines due to an incompatible python version being used when loading the cryptography rust binding.

For example, dci-macm1-build-098 machine uses /etc/helix-prep/venv/bin/python3.13 and succeeds, while dci-macm2-build-072 fails, likely lower python version.

Build: https://dev.azure.com/dnceng-public/public/_build/results?buildId=1097633&view=logs&j=e574055e-630a-54e9-c3c5-795a4257d76f&t=d07609d0-328b-5d07-0155-9586c39ca3e4
Log:

+ /usr/local/bin/python3 -u /tmp/helix/working/A4EC08BA/w/A3E80924/u/xharness-event-processor.py
/etc/helix/scripts/azure/__init__.py:5: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
  import pkg_resources
Traceback (most recent call last):
  File "/tmp/helix/working/A4EC08BA/w/A3E80924/u/xharness-event-processor.py", line 8, in <module>
    from helix.public import request_reboot, request_infra_retry, send_metric, send_metrics
  File "/etc/helix/scripts/helix/public/__init__.py", line 5, in <module>
    import helix.event
  File "/etc/helix/scripts/helix/event.py", line 7, in <module>
    import helix.logs
  File "/etc/helix/scripts/helix/logs.py", line 11, in <module>
    from helix.azure_utils import get_auth_credential
  File "/etc/helix/scripts/helix/azure_utils.py", line 2, in <module>
    from azure.identity import CertificateCredential, ClientAssertionCredential, ManagedIdentityCredential, DefaultAzureCredential
  File "/etc/helix/scripts/azure/identity/__init__.py", line 10, in <module>
    from ._credentials import (
  File "/etc/helix/scripts/azure/identity/_credentials/__init__.py", line 5, in <module>
    from .authorization_code import AuthorizationCodeCredential
  File "/etc/helix/scripts/azure/identity/_credentials/authorization_code.py", line 9, in <module>
    from .._internal.aad_client import AadClient
  File "/etc/helix/scripts/azure/identity/_internal/__init__.py", line 5, in <module>
    from .aad_client import AadClient
  File "/etc/helix/scripts/azure/identity/_internal/aad_client.py", line 11, in <module>
    from .aad_client_base import AadClientBase
  File "/etc/helix/scripts/azure/identity/_internal/aad_client_base.py", line 20, in <module>
    from .aadclient_certificate import AadClientCertificate
  File "/etc/helix/scripts/azure/identity/_internal/aadclient_certificate.py", line 7, in <module>
    from cryptography import x509
  File "/etc/helix/scripts/cryptography/x509/__init__.py", line 7, in <module>
    from cryptography.x509 import certificate_transparency, verification
  File "/etc/helix/scripts/cryptography/x509/certificate_transparency.py", line 8, in <module>
    from cryptography.hazmat.bindings._rust import x509 as rust_x509
ImportError: dlopen(/etc/helix/scripts/cryptography/hazmat/bindings/_rust.abi3.so, 0x0002): symbol not found in flat namespace '_PyType_GetName'
+ /usr/local/bin/python3 /tmp/helix/working/A4EC08BA/p/reporter/run.py https://dev.azure.com/dnceng-public/ public 30019074 eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6InRzOGNhWXZ4UGNORXBpQnFhVG1FTlpwRmhZMCJ9.eyJuYW1laWQiOiJjNzczZjJjMi01MTIwLTQyMDctYWZlMi1hZmFmMzVhOGJjMGEiLCJzY3AiOiJhcHBfdG9rZW4iLCJhdWkiOiJiY2IxZjc1OS03ZGQzLTQ5NTYtOTQxOC1lMDRkMmQ2ZGJhYjgiLCJzaWQiOiJkMmUzMmY1NC1kYmE0LTRhMTMtYTcxZS1iMTg4YzVlNGZlMzQiLCJCdWlsZElkIjoiY2JiMTgyNjEtYzQ4Zi00YWJiLTg2NTEtOGNkY2I1NDc0NjQ5OzEwOTc2MzMiLCJEZWZJZCI6IjE1NCIsImpvYnJlZiI6IjNjMDVmNDU4LThmNjYtNDE4Ny05MjM1LTA1M2Y1ZDg4YzM4YzplNTc0MDU1ZS02MzBhLTU0ZTktYzNjNS03OTVhNDI1N2Q3NmYiLCJwcGlkIjoidnN0ZnM6Ly8vQnVpbGQvQnVpbGQvMTA5NzYzMyIsIm9yY2hpZCI6IjNjMDVmNDU4LThmNjYtNDE4Ny05MjM1LTA1M2Y1ZDg4YzM4Yy5idWlsZC5idWlsZF9tYWNjYXRhbHlzdF9hcm02NF9yZWxlYXNlX2FsbHN1YnNldHNfbmF0aXZlYW90X3J1bnRpbWV0ZXN0c19hcHBzYW5kYm94Ll9fZGVmYXVsdC4yIiwicmVwb0lkcyI6IiIsImlzcyI6ImFwcC52c3Rva2VuLnZpc3VhbHN0dWRpby5jb20iLCJhdWQiOiJhcHAudnN0b2tlbi52aXN1YWxzdHVkaW8uY29tfHZzbzo2ZmNjOTJlNS03M2E3LTRmODgtOGQxMy1kOTA0NWI0NWZiMjciLCJuYmYiOjE3NTI3NTY5ODIsImV4cCI6MTc1Mjc2ODk4Mn0.s4dkQdulIPl4ozV17C7gFiMPAMHQWdbpUHNJutXIiaxLhjO32_9rpKbGefVCrI8-SVwKGeLiDPekaYPwdgU4aeRftufuJu472KKG9AkfcAknhMhfsyFXXwqNbTtPLdcFmuVKAQsgUu3xs8_UJYuI_22WiNQjjC_9O1giTCuH3HRDmqdTlvpncvQ4wDMiTrtJiLUNKXAifrpvXWhmdTYxh5AbhGWU2VHy40-ydUiDu--GBxUv35nAk4dV1XCuau47XJuEbZcvhP-3JdE_YTTC5qf9FjddtH03xvB0gtqZBkxbmqvaNLZNu7hXcjHJnZpdk8Fb0vhccDHsrgEOUJiM4g
/etc/helix/scripts/azure/__init__.py:5: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
  import pkg_resources
Traceback (most recent call last):
  File "/tmp/helix/working/A4EC08BA/p/reporter/run.py", line 13, in <module>
    from test_results_reader import read_results
  File "/private/tmp/helix/working/A4EC08BA/p/reporter/test_results_reader/__init__.py", line 3, in <module>
    from helix.public import TestResult, TestResultAttachment
  File "/etc/helix/scripts/helix/public/__init__.py", line 5, in <module>
    import helix.event
  File "/etc/helix/scripts/helix/event.py", line 7, in <module>
    import helix.logs
  File "/etc/helix/scripts/helix/logs.py", line 11, in <module>
    from helix.azure_utils import get_auth_credential
  File "/etc/helix/scripts/helix/azure_utils.py", line 2, in <module>
    from azure.identity import CertificateCredential, ClientAssertionCredential, ManagedIdentityCredential, DefaultAzureCredential
  File "/etc/helix/scripts/azure/identity/__init__.py", line 10, in <module>
    from ._credentials import (
  File "/etc/helix/scripts/azure/identity/_credentials/__init__.py", line 5, in <module>
    from .authorization_code import AuthorizationCodeCredential
  File "/etc/helix/scripts/azure/identity/_credentials/authorization_code.py", line 9, in <module>
    from .._internal.aad_client import AadClient
  File "/etc/helix/scripts/azure/identity/_internal/__init__.py", line 5, in <module>
    from .aad_client import AadClient
  File "/etc/helix/scripts/azure/identity/_internal/aad_client.py", line 11, in <module>
    from .aad_client_base import AadClientBase
  File "/etc/helix/scripts/azure/identity/_internal/aad_client_base.py", line 20, in <module>
    from .aadclient_certificate import AadClientCertificate
  File "/etc/helix/scripts/azure/identity/_internal/aadclient_certificate.py", line 7, in <module>
    from cryptography import x509
  File "/etc/helix/scripts/cryptography/x509/__init__.py", line 7, in <module>
    from cryptography.x509 import certificate_transparency, verification
  File "/etc/helix/scripts/cryptography/x509/certificate_transparency.py", line 8, in <module>
    from cryptography.hazmat.bindings._rust import x509 as rust_x509
ImportError: dlopen(/etc/helix/scripts/cryptography/hazmat/bindings/_rust.abi3.so, 0x0002): symbol not found in flat namespace '_PyType_GetName'

Tasks

• List all machines from osx.14.arm64.open with python version < 3.13
• Upgrade python version

[dotnet-policy-service]

Tagging subscribers to 'os-maccatalyst': @vitek-karas, @kotlarmilos, @ivanpovazan, @steveisok, @akoeplinger
See info in area-owners.md if you want to be subscribed.

[dotnet-policy-service]

Tagging subscribers to this area: @dotnet/runtime-infrastructure
See info in area-owners.md if you want to be subscribed.