Skip to content

Commit c925d2d

Browse files
greysonlalondegreysonlalonde
authored
chore: restructure test env, cassettes, and conftest; fix flaky tests
Consolidates pytest config, standardizes env handling, reorganizes cassette layout, removes outdated VCR configs, improves sync with threading.Condition, updates event-waiting logic, ensures cleanup, regenerates Gemini cassettes, and reverts unintended test changes.
1 parent bc4e6a3 commit c925d2d

File tree

200 files changed

+2074
-1895
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

200 files changed

+2074
-1895
lines changed

‎.env.test‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
# =============================================================================
2+
# Test Environment Variables
3+
# =============================================================================
4+
# This file contains all environment variables needed to run tests locally
5+
# in a way that mimics the GitHub Actions CI environment.
6+
7+
# =============================================================================
8+
9+
# -----------------------------------------------------------------------------
10+
# LLM Provider API Keys
11+
# -----------------------------------------------------------------------------
12+
OPENAI_API_KEY=fake-api-key
13+
ANTHROPIC_API_KEY=fake-anthropic-key
14+
GEMINI_API_KEY=fake-gemini-key
15+
AZURE_API_KEY=fake-azure-key
16+
OPENROUTER_API_KEY=fake-openrouter-key
17+
18+
# -----------------------------------------------------------------------------
19+
# AWS Credentials
20+
# -----------------------------------------------------------------------------
21+
AWS_ACCESS_KEY_ID=fake-aws-access-key
22+
AWS_SECRET_ACCESS_KEY=fake-aws-secret-key
23+
AWS_DEFAULT_REGION=us-east-1
24+
AWS_REGION_NAME=us-east-1
25+
26+
# -----------------------------------------------------------------------------
27+
# Azure OpenAI Configuration
28+
# -----------------------------------------------------------------------------
29+
AZURE_ENDPOINT=https://fake-azure-endpoint.openai.azure.com
30+
AZURE_OPENAI_ENDPOINT=https://fake-azure-endpoint.openai.azure.com
31+
AZURE_OPENAI_API_KEY=fake-azure-openai-key
32+
AZURE_API_VERSION=2024-02-15-preview
33+
OPENAI_API_VERSION=2024-02-15-preview
34+
35+
# -----------------------------------------------------------------------------
36+
# Google Cloud Configuration
37+
# -----------------------------------------------------------------------------
38+
#GOOGLE_CLOUD_PROJECT=fake-gcp-project
39+
#GOOGLE_CLOUD_LOCATION=us-central1
40+
41+
# -----------------------------------------------------------------------------
42+
# OpenAI Configuration
43+
# -----------------------------------------------------------------------------
44+
OPENAI_BASE_URL=https://api.openai.com/v1
45+
OPENAI_API_BASE=https://api.openai.com/v1
46+
47+
# -----------------------------------------------------------------------------
48+
# Search & Scraping Tool API Keys
49+
# -----------------------------------------------------------------------------
50+
SERPER_API_KEY=fake-serper-key
51+
EXA_API_KEY=fake-exa-key
52+
BRAVE_API_KEY=fake-brave-key
53+
FIRECRAWL_API_KEY=fake-firecrawl-key
54+
TAVILY_API_KEY=fake-tavily-key
55+
SERPAPI_API_KEY=fake-serpapi-key
56+
SERPLY_API_KEY=fake-serply-key
57+
LINKUP_API_KEY=fake-linkup-key
58+
PARALLEL_API_KEY=fake-parallel-key
59+
60+
# -----------------------------------------------------------------------------
61+
# Exa Configuration
62+
# -----------------------------------------------------------------------------
63+
EXA_BASE_URL=https://api.exa.ai
64+
65+
# -----------------------------------------------------------------------------
66+
# Web Scraping & Automation
67+
# -----------------------------------------------------------------------------
68+
BRIGHT_DATA_API_KEY=fake-brightdata-key
69+
BRIGHT_DATA_ZONE=fake-zone
70+
BRIGHTDATA_API_URL=https://api.brightdata.com
71+
BRIGHTDATA_DEFAULT_TIMEOUT=600
72+
BRIGHTDATA_DEFAULT_POLLING_INTERVAL=1
73+
74+
OXYLABS_USERNAME=fake-oxylabs-user
75+
OXYLABS_PASSWORD=fake-oxylabs-pass
76+
77+
SCRAPFLY_API_KEY=fake-scrapfly-key
78+
SCRAPEGRAPH_API_KEY=fake-scrapegraph-key
79+
80+
BROWSERBASE_API_KEY=fake-browserbase-key
81+
BROWSERBASE_PROJECT_ID=fake-browserbase-project
82+
83+
HYPERBROWSER_API_KEY=fake-hyperbrowser-key
84+
MULTION_API_KEY=fake-multion-key
85+
APIFY_API_TOKEN=fake-apify-token
86+
87+
# -----------------------------------------------------------------------------
88+
# Database & Vector Store Credentials
89+
# -----------------------------------------------------------------------------
90+
SINGLESTOREDB_URL=mysql://fake:fake@localhost:3306/fake
91+
SINGLESTOREDB_HOST=localhost
92+
SINGLESTOREDB_PORT=3306
93+
SINGLESTOREDB_USER=fake-user
94+
SINGLESTOREDB_PASSWORD=fake-password
95+
SINGLESTOREDB_DATABASE=fake-database
96+
SINGLESTOREDB_CONNECT_TIMEOUT=30
97+
98+
SNOWFLAKE_USER=fake-snowflake-user
99+
SNOWFLAKE_PASSWORD=fake-snowflake-password
100+
SNOWFLAKE_ACCOUNT=fake-snowflake-account
101+
SNOWFLAKE_WAREHOUSE=fake-snowflake-warehouse
102+
SNOWFLAKE_DATABASE=fake-snowflake-database
103+
SNOWFLAKE_SCHEMA=fake-snowflake-schema
104+
105+
WEAVIATE_URL=http://localhost:8080
106+
WEAVIATE_API_KEY=fake-weaviate-key
107+
108+
EMBEDCHAIN_DB_URI=sqlite:///test.db
109+
110+
# Databricks Credentials
111+
DATABRICKS_HOST=https://fake-databricks.cloud.databricks.com
112+
DATABRICKS_TOKEN=fake-databricks-token
113+
DATABRICKS_CONFIG_PROFILE=fake-profile
114+
115+
# MongoDB Credentials
116+
MONGODB_URI=mongodb://fake:fake@localhost:27017/fake
117+
118+
# -----------------------------------------------------------------------------
119+
# CrewAI Platform & Enterprise
120+
# -----------------------------------------------------------------------------
121+
# setting CREWAI_PLATFORM_INTEGRATION_TOKEN causes these test to fail:
122+
#=========================== short test summary info ============================
123+
#FAILED tests/test_context.py::TestPlatformIntegrationToken::test_platform_context_manager_basic_usage - AssertionError: assert 'fake-platform-token' is None
124+
# + where 'fake-platform-token' = get_platform_integration_token()
125+
#FAILED tests/test_context.py::TestPlatformIntegrationToken::test_context_var_isolation_between_tests - AssertionError: assert 'fake-platform-token' is None
126+
# + where 'fake-platform-token' = get_platform_integration_token()
127+
#FAILED tests/test_context.py::TestPlatformIntegrationToken::test_multiple_sequential_context_managers - AssertionError: assert 'fake-platform-token' is None
128+
# + where 'fake-platform-token' = get_platform_integration_token()
129+
#CREWAI_PLATFORM_INTEGRATION_TOKEN=fake-platform-token
130+
CREWAI_PERSONAL_ACCESS_TOKEN=fake-personal-token
131+
CREWAI_PLUS_URL=https://fake.crewai.com
132+
133+
# -----------------------------------------------------------------------------
134+
# Other Service API Keys
135+
# -----------------------------------------------------------------------------
136+
ZAPIER_API_KEY=fake-zapier-key
137+
PATRONUS_API_KEY=fake-patronus-key
138+
MINDS_API_KEY=fake-minds-key
139+
HF_TOKEN=fake-hf-token
140+
141+
# -----------------------------------------------------------------------------
142+
# Feature Flags/Testing Modes
143+
# -----------------------------------------------------------------------------
144+
CREWAI_DISABLE_TELEMETRY=true
145+
OTEL_SDK_DISABLED=true
146+
CREWAI_TESTING=true
147+
CREWAI_TRACING_ENABLED=false
148+
149+
# -----------------------------------------------------------------------------
150+
# Testing/CI Configuration
151+
# -----------------------------------------------------------------------------
152+
# VCR recording mode: "none" (default), "new_episodes", "all", "once"
153+
PYTEST_VCR_RECORD_MODE=none
154+
155+
# Set to "true" by GitHub when running in GitHub Actions
156+
# GITHUB_ACTIONS=false
157+
158+
# -----------------------------------------------------------------------------
159+
# Python Configuration
160+
# -----------------------------------------------------------------------------
161+
PYTHONUNBUFFERED=1

‎.github/workflows/tests.yml‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,6 @@ on: [pull_request]
55
permissions:
66
contents: read
77

8-
env:
9-
OPENAI_API_KEY: fake-api-key
10-
PYTHONUNBUFFERED: 1
11-
BRAVE_API_KEY: fake-brave-key
12-
SNOWFLAKE_USER: fake-snowflake-user
13-
SNOWFLAKE_PASSWORD: fake-snowflake-password
14-
SNOWFLAKE_ACCOUNT: fake-snowflake-account
15-
SNOWFLAKE_WAREHOUSE: fake-snowflake-warehouse
16-
SNOWFLAKE_DATABASE: fake-snowflake-database
17-
SNOWFLAKE_SCHEMA: fake-snowflake-schema
18-
EMBEDCHAIN_DB_URI: sqlite:///test.db
19-
208
jobs:
219
tests:
2210
name: tests (${{ matrix.python-version }})
@@ -84,26 +72,20 @@ jobs:
8472
# fi
8573
8674
cd lib/crewai && uv run pytest \
87-
--block-network \
88-
--timeout=30 \
8975
-vv \
9076
--splits 8 \
9177
--group ${{ matrix.group }} \
9278
$DURATIONS_ARG \
9379
--durations=10 \
94-
-n auto \
9580
--maxfail=3
9681
9782
- name: Run tool tests (group ${{ matrix.group }} of 8)
9883
run: |
9984
cd lib/crewai-tools && uv run pytest \
100-
--block-network \
101-
--timeout=30 \
10285
-vv \
10386
--splits 8 \
10487
--group ${{ matrix.group }} \
10588
--durations=10 \
106-
-n auto \
10789
--maxfail=3
10890
10991

‎conftest.py‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
"""Pytest configuration for crewAI workspace."""
2+
3+
from collections.abc import Generator
4+
import os
5+
from pathlib import Path
6+
import tempfile
7+
from typing import Any
8+
9+
from dotenv import load_dotenv
10+
import pytest
11+
from vcr.request import Request # type: ignore[import-untyped]
12+
13+
14+
env_test_path = Path(__file__).parent / ".env.test"
15+
load_dotenv(env_test_path, override=True)
16+
load_dotenv(override=True)
17+
18+
19+
@pytest.fixture(autouse=True, scope="function")
20+
def cleanup_event_handlers() -> Generator[None, Any, None]:
21+
"""Clean up event bus handlers after each test to prevent test pollution."""
22+
yield
23+
24+
try:
25+
from crewai.events.event_bus import crewai_event_bus
26+
27+
with crewai_event_bus._rwlock.w_locked():
28+
crewai_event_bus._sync_handlers.clear()
29+
crewai_event_bus._async_handlers.clear()
30+
except Exception: # noqa: S110
31+
pass
32+
33+
34+
@pytest.fixture(autouse=True, scope="function")
35+
def setup_test_environment() -> Generator[None, Any, None]:
36+
"""Setup test environment for crewAI workspace."""
37+
with tempfile.TemporaryDirectory() as temp_dir:
38+
storage_dir = Path(temp_dir) / "crewai_test_storage"
39+
storage_dir.mkdir(parents=True, exist_ok=True)
40+
41+
if not storage_dir.exists() or not storage_dir.is_dir():
42+
raise RuntimeError(
43+
f"Failed to create test storage directory: {storage_dir}"
44+
)
45+
46+
try:
47+
test_file = storage_dir / ".permissions_test"
48+
test_file.touch()
49+
test_file.unlink()
50+
except (OSError, IOError) as e:
51+
raise RuntimeError(
52+
f"Test storage directory {storage_dir} is not writable: {e}"
53+
) from e
54+
55+
os.environ["CREWAI_STORAGE_DIR"] = str(storage_dir)
56+
os.environ["CREWAI_TESTING"] = "true"
57+
58+
try:
59+
yield
60+
finally:
61+
os.environ.pop("CREWAI_TESTING", "true")
62+
os.environ.pop("CREWAI_STORAGE_DIR", None)
63+
os.environ.pop("CREWAI_DISABLE_TELEMETRY", "true")
64+
os.environ.pop("OTEL_SDK_DISABLED", "true")
65+
os.environ.pop("OPENAI_BASE_URL", "https://api.openai.com/v1")
66+
os.environ.pop("OPENAI_API_BASE", "https://api.openai.com/v1")
67+
68+
69+
HEADERS_TO_FILTER = {
70+
"authorization": "AUTHORIZATION-XXX",
71+
"content-security-policy": "CSP-FILTERED",
72+
"cookie": "COOKIE-XXX",
73+
"set-cookie": "SET-COOKIE-XXX",
74+
"permissions-policy": "PERMISSIONS-POLICY-XXX",
75+
"referrer-policy": "REFERRER-POLICY-XXX",
76+
"strict-transport-security": "STS-XXX",
77+
"x-content-type-options": "X-CONTENT-TYPE-XXX",
78+
"x-frame-options": "X-FRAME-OPTIONS-XXX",
79+
"x-permitted-cross-domain-policies": "X-PERMITTED-XXX",
80+
"x-request-id": "X-REQUEST-ID-XXX",
81+
"x-runtime": "X-RUNTIME-XXX",
82+
"x-xss-protection": "X-XSS-PROTECTION-XXX",
83+
"x-stainless-arch": "X-STAINLESS-ARCH-XXX",
84+
"x-stainless-os": "X-STAINLESS-OS-XXX",
85+
"x-stainless-read-timeout": "X-STAINLESS-READ-TIMEOUT-XXX",
86+
"cf-ray": "CF-RAY-XXX",
87+
"etag": "ETAG-XXX",
88+
"Strict-Transport-Security": "STS-XXX",
89+
"access-control-expose-headers": "ACCESS-CONTROL-XXX",
90+
"openai-organization": "OPENAI-ORG-XXX",
91+
"openai-project": "OPENAI-PROJECT-XXX",
92+
"x-ratelimit-limit-requests": "X-RATELIMIT-LIMIT-REQUESTS-XXX",
93+
"x-ratelimit-limit-tokens": "X-RATELIMIT-LIMIT-TOKENS-XXX",
94+
"x-ratelimit-remaining-requests": "X-RATELIMIT-REMAINING-REQUESTS-XXX",
95+
"x-ratelimit-remaining-tokens": "X-RATELIMIT-REMAINING-TOKENS-XXX",
96+
"x-ratelimit-reset-requests": "X-RATELIMIT-RESET-REQUESTS-XXX",
97+
"x-ratelimit-reset-tokens": "X-RATELIMIT-RESET-TOKENS-XXX",
98+
"x-goog-api-key": "X-GOOG-API-KEY-XXX",
99+
}
100+
101+
102+
def _filter_request_headers(request: Request) -> Request: # type: ignore[no-any-unimported]
103+
"""Filter sensitive headers from request before recording."""
104+
for header_name, replacement in HEADERS_TO_FILTER.items():
105+
for variant in [header_name, header_name.upper(), header_name.title()]:
106+
if variant in request.headers:
107+
request.headers[variant] = [replacement]
108+
return request
109+
110+
111+
def _filter_response_headers(response: dict[str, Any]) -> dict[str, Any]:
112+
"""Filter sensitive headers from response before recording."""
113+
for header_name, replacement in HEADERS_TO_FILTER.items():
114+
for variant in [header_name, header_name.upper(), header_name.title()]:
115+
if variant in response["headers"]:
116+
response["headers"][variant] = [replacement]
117+
return response
118+
119+
120+
@pytest.fixture(scope="module")
121+
def vcr_cassette_dir(request: Any) -> str:
122+
"""Generate cassette directory path based on test module location.
123+
124+
Organizes cassettes to mirror test directory structure within each package:
125+
lib/crewai/tests/llms/google/test_google.py -> lib/crewai/tests/cassettes/llms/google/
126+
lib/crewai-tools/tests/tools/test_search.py -> lib/crewai-tools/tests/cassettes/tools/
127+
"""
128+
test_file = Path(request.fspath)
129+
130+
for parent in test_file.parents:
131+
if parent.name in ("crewai", "crewai-tools") and parent.parent.name == "lib":
132+
package_root = parent
133+
break
134+
else:
135+
package_root = test_file.parent
136+
137+
tests_root = package_root / "tests"
138+
test_dir = test_file.parent
139+
140+
if test_dir != tests_root:
141+
relative_path = test_dir.relative_to(tests_root)
142+
cassette_dir = tests_root / "cassettes" / relative_path
143+
else:
144+
cassette_dir = tests_root / "cassettes"
145+
146+
cassette_dir.mkdir(parents=True, exist_ok=True)
147+
148+
return str(cassette_dir)
149+
150+
151+
@pytest.fixture(scope="module")
152+
def vcr_config(vcr_cassette_dir: str) -> dict[str, Any]:
153+
"""Configure VCR with organized cassette storage."""
154+
config = {
155+
"cassette_library_dir": vcr_cassette_dir,
156+
"record_mode": os.getenv("PYTEST_VCR_RECORD_MODE", "once"),
157+
"filter_headers": [(k, v) for k, v in HEADERS_TO_FILTER.items()],
158+
"before_record_request": _filter_request_headers,
159+
"before_record_response": _filter_response_headers,
160+
"filter_query_parameters": ["key"],
161+
}
162+
163+
if os.getenv("GITHUB_ACTIONS") == "true":
164+
config["record_mode"] = "none"
165+
166+
return config

‎lib/crewai-tools/BUILDING_TOOLS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ Update the root `README.md` only if the tool introduces a new category or notabl
218218

219219
## Discovery and specs
220220

221-
Our internal tooling discovers classes whose names end with `Tool`. Keep your class exported from the module path under `crewai_tools/tools/...` to be picked up by scripts like `generate_tool_specs.py`.
221+
Our internal tooling discovers classes whose names end with `Tool`. Keep your class exported from the module path under `crewai_tools/tools/...` to be picked up by scripts like `crewai_tools.generate_tool_specs.py`.
222222

223223
---
224224

0 commit comments

Comments
 (0)