fix: tests to run under pytest

Fix tests to run fully under pytest, this includes:
- Fixing filenames, removing dots and correct typos
- Removing __init__ methods, which are not supported by pytest
- Implement parametrisation so tests can be run individually
- Added timeouts so tests can't run forever
- Replacing print and logging with assertions to prevent false successes
- Removing unused and add missing imports
- Mark tests with @pytest.mark.asyncio where appropriate
- Use http constants to avoid magic numbers
- Add type hints to improve linting and identify issues
- Use local server for API tests to improve debugging and eliminate
  docker dependency
- Call pytest in __main__ to allow running tests from command line
- Skip broken tests
- Fix out of date logic and invalid method parameters
- Re-enable disabled and commented out tests after fixing them
- Added missing test data
- Updated tests that depend on altered external css or html structure
- Automatically skip if tests if API key is not set

If you need to debug a test, which will take time, you will need to
comment out the default timeout in pyproject.toml under
[tool.pytest.ini_options].

Author: stevenh

pyproject.toml

@@ -103,6 +103,7 @@ test = [
     "matplotlib>=3.9.4",
     "nest-asyncio>=1.6.0",
     "nltk>=3.9.1",
+    "pytest-aiohttp>=1.1.0",
     "pytest-asyncio>=0.25.3",
     "pytest-cov>=6.0.0",
     "pytest-httpserver>=1.1.2",

tests/20241401/test_advanced_deep_crawl.py

@@ -1,29 +1,35 @@
-import asyncio
+import sys
 import time
 
+from httpx import codes
+import pytest
 
 from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
 from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
-from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+from crawl4ai.deep_crawling import BestFirstCrawlingStrategy
 from crawl4ai.deep_crawling.filters import FilterChain, URLPatternFilter, DomainFilter, ContentTypeFilter, ContentRelevanceFilter
 from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
-# from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+from crawl4ai.types import CrawlResult
 
 
-async def main():
+@pytest.mark.asyncio
+@pytest.mark.timeout(60)
+async def test_deep_crawl():
     """Example deep crawl of documentation site."""
     filter_chain = FilterChain([
         URLPatternFilter(patterns=["*2025*"]),
         DomainFilter(allowed_domains=["techcrunch.com"]),
         ContentRelevanceFilter(query="Use of artificial intelligence in Defence applications", threshold=1),
         ContentTypeFilter(allowed_types=["text/html","application/javascript"])
     ])
+    max_pages: int = 5
     config = CrawlerRunConfig(
         deep_crawl_strategy = BestFirstCrawlingStrategy(
             max_depth=2,
             include_external=False,
             filter_chain=filter_chain,
             url_scorer=KeywordRelevanceScorer(keywords=["anduril", "defence", "AI"]),
+            max_pages=max_pages,
         ),
         stream=False,
         verbose=True,
@@ -35,12 +41,24 @@ async def main():
         print("Starting deep crawl in streaming mode:")
         config.stream = True
         start_time = time.perf_counter()
+        result: CrawlResult
+        pages: int = 0
         async for result in await crawler.arun(
             url="https://techcrunch.com",
             config=config
         ):
+            assert result.status_code == codes.OK
+            assert result.url
+            assert result.metadata
+            assert result.metadata.get("depth", -1) >= 0
+            assert result.metadata.get("depth", -1) <= 2
+            pages += 1
             print(f"→ {result.url} (Depth: {result.metadata.get('depth', 0)})")
-        print(f"Duration: {time.perf_counter() - start_time:.2f} seconds")
+
+        print(f"Crawled {pages} pages in: {time.perf_counter() - start_time:.2f} seconds")
+        assert pages == max_pages
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    import subprocess
+
+    sys.exit(subprocess.call(["pytest", *sys.argv[1:], sys.argv[0]]))

tests/20241401/test_acyn_crawl_wuth_http_crawler_strategy.py renamed to tests/20241401/test_async_crawl_with_http_crawler_strategy.py

@@ -1,16 +1,30 @@
-import asyncio
+import sys
+
+from httpx import codes
+import pytest
+
 from crawl4ai import (
     AsyncWebCrawler,
-    CrawlerRunConfig,
-    HTTPCrawlerConfig,
     CacheMode,
+    CrawlerRunConfig,
     DefaultMarkdownGenerator,
-    PruningContentFilter
+    HTTPCrawlerConfig,
+    PruningContentFilter,
 )
 from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
 from crawl4ai.async_logger import AsyncLogger
 
-async def main():
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.com",
+        "https://httpbin.org/get",
+        "raw://<html><body>Test content</body></html>"
+    ]
+)
+async def test_async_crawl(url: str):
     # Initialize HTTP crawler strategy
     http_strategy = AsyncHTTPCrawlerStrategy(
         browser_config=HTTPCrawlerConfig(
@@ -27,30 +41,23 @@ async def main():
             cache_mode=CacheMode.BYPASS,
             markdown_generator=DefaultMarkdownGenerator(
                 content_filter=PruningContentFilter(
-                    threshold=0.48, 
-                    threshold_type="fixed", 
+                    threshold=0.48,
+                    threshold_type="fixed",
                     min_word_threshold=0
                 )
             )
         )
-        
-        # Test different URLs
-        urls = [
-            "https://example.com",
-            "https://httpbin.org/get",
-            "raw://<html><body>Test content</body></html>"
-        ]
-        
-        for url in urls:
-            print(f"\n=== Testing {url} ===")
-            try:
-                result = await crawler.arun(url=url, config=crawler_config)
-                print(f"Status: {result.status_code}")
-                print(f"Raw HTML length: {len(result.html)}")
-                if hasattr(result, 'markdown'):
-                    print(f"Markdown length: {len(result.markdown.raw_markdown)}")
-            except Exception as e:
-                print(f"Error: {e}")
+
+        result = await crawler.arun(url=url, config=crawler_config)
+        assert result.status_code == codes.OK
+        assert result.html
+        assert result.markdown
+        assert result.markdown.raw_markdown
+        print(f"Status: {result.status_code}")
+        print(f"Raw HTML length: {len(result.html)}")
+        print(f"Markdown length: {len(result.markdown.raw_markdown)}")
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    import subprocess
+
+    sys.exit(subprocess.call(["pytest", *sys.argv[1:], sys.argv[0]]))