Have as many WARCBolt instances as there are workers, fix #64

Signed-off-by: Julien Nioche <[email protected]>
commoncrawl · Dec 13, 2023 · 63dafc1 · 63dafc1
1 parent 0bfccf5
commit 63dafc1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java b/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java
@@ -94,7 +94,7 @@ protected int run(String[] args) {
 
 	// take it from feed default output so that the feed files themselves
 	// don't get included - unless we want them too of course!
-	builder.setBolt("warc", warcbolt).localOrShuffleGrouping("feed");
+	builder.setBolt("warc", warcbolt, numWorkers).localOrShuffleGrouping("feed");
 
 	BoltDeclarer statusBolt = builder.setBolt("status", new StatusUpdaterBolt(), numWorkers)
 		.localOrShuffleGrouping("fetch", Constants.StatusStreamName)