fix: tests and associated bugs

This is a placeholder which contains all the fixes and improvements
needed to get all the existing tests running error free.

There are a small number of tests which have been marked as skipped due
to flaky behaviour or significant functionality needed to make them
work.

The intention is to split this out into many more consumable changes.

Author: stevenh

.gitignore

@@ -50,6 +50,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+tests/async/output/
 
 # Translations
 *.mo

crawl4ai/__init__.py

@@ -1,68 +1,71 @@
 # __init__.py
 import warnings
+from logging import Logger
 
-from .async_webcrawler import AsyncWebCrawler, CacheMode
 from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig
-
-from .content_scraping_strategy import (
-    ContentScrapingStrategy,
-    WebScrapingStrategy,
-    LXMLWebScrapingStrategy,
+from .async_dispatcher import (
+    BaseDispatcher,
+    CrawlerMonitor,
+    DisplayMode,
+    MemoryAdaptiveDispatcher,
+    RateLimiter,
+    SemaphoreDispatcher,
 )
 from .async_logger import (
-    AsyncLoggerBase,
     AsyncLogger,
+    AsyncLoggerBase,
 )
-from .proxy_strategy import (
-    ProxyRotationStrategy,
-    RoundRobinProxyStrategy,
-)
-from .extraction_strategy import (
-    ExtractionStrategy,
-    LLMExtractionStrategy,
-    CosineStrategy,
-    JsonCssExtractionStrategy,
-    JsonXPathExtractionStrategy,
-)
+from .async_webcrawler import AsyncWebCrawler, CacheMode
+from .browser_profiler import BrowserProfiler
 from .chunking_strategy import ChunkingStrategy, RegexChunking
-from .markdown_generation_strategy import DefaultMarkdownGenerator
 from .content_filter_strategy import (
-    PruningContentFilter,
     BM25ContentFilter,
     LLMContentFilter,
+    PruningContentFilter,
     RelevantContentFilter,
 )
-from .models import CrawlResult, MarkdownGenerationResult
-from .async_dispatcher import (
-    MemoryAdaptiveDispatcher,
-    SemaphoreDispatcher,
-    RateLimiter,
-    CrawlerMonitor,
-    DisplayMode,
-    BaseDispatcher,
+from .content_scraping_strategy import (
+    ContentScrapingStrategy,
+    LXMLWebScrapingStrategy,
+    WebScrapingStrategy,
 )
-from .docker_client import Crawl4aiDockerClient
-from .hub import CrawlerHub
-from .browser_profiler import BrowserProfiler
 from .deep_crawling import (
-    DeepCrawlStrategy,
+    BestFirstCrawlingStrategy,
     BFSDeepCrawlStrategy,
-    FilterChain,
-    URLPatternFilter,
-    DomainFilter,
-    ContentTypeFilter,
-    URLFilter,
-    FilterStats,
-    SEOFilter,
-    KeywordRelevanceScorer,
-    URLScorer,
     CompositeScorer,
+    ContentTypeFilter,
+    DeepCrawlDecorator,
+    DeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
     DomainAuthorityScorer,
+    DomainFilter,
+    FilterChain,
+    FilterStats,
     FreshnessScorer,
+    KeywordRelevanceScorer,
     PathDepthScorer,
-    BestFirstCrawlingStrategy,
-    DFSDeepCrawlStrategy,
-    DeepCrawlDecorator,
+    SEOFilter,
+    URLFilter,
+    URLPatternFilter,
+    URLScorer,
+)
+from .deep_crawling.scorers import (
+    ScoringStats,
+)
+from .docker_client import Crawl4aiDockerClient
+from .extraction_strategy import (
+    CosineStrategy,
+    ExtractionStrategy,
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+    LLMExtractionStrategy,
+)
+from .hub import CrawlerHub
+from .markdown_generation_strategy import DefaultMarkdownGenerator
+from .models import CrawlResult, MarkdownGenerationResult
+from .proxy_strategy import (
+    ProxyRotationStrategy,
+    RoundRobinProxyStrategy,
 )
 
 __all__ = [
@@ -120,6 +123,8 @@
     "Crawl4aiDockerClient",
     "ProxyRotationStrategy",
     "RoundRobinProxyStrategy",
+    "ScoringStats",
+    "Logger",
 ]