Upgraded dependencies and an additional filter

build.sbt

@@ -1,12 +1,19 @@
-lazy val root = (project in file(".")).
-  settings(
-    name := "scrala",
-    version := "1.0",
-    scalaVersion := "2.11.7",
-    libraryDependencies += "org.apache.httpcomponents" % "httpclient" % "4.5.1",
-    libraryDependencies += "org.jsoup" % "jsoup" % "1.8.2",
-    libraryDependencies += "com.typesafe.akka" % "akka-actor_2.11" % "2.4-M1",
-    libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0",
-    libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.1.2",
-    libraryDependencies += "org.json4s" %% "json4s-native" % "3.3.0"
-  )
+name := "scrala"
+
+version := "0.1"
+
+scalaVersion := "2.12.8"
+
+libraryDependencies += "org.apache.httpcomponents" % "httpclient" % "4.5.8"
+
+libraryDependencies += "org.jsoup" % "jsoup" % "1.12.1"
+
+libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.5.23"
+
+libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2"
+
+libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3"
+
+libraryDependencies += "org.json4s" %% "json4s-native" % "3.6.6"
+
+libraryDependencies += "com.google.guava" % "guava" % "27.1-jre"

project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.2.8

project/plugins.sbt

@@ -1 +0,0 @@
-addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")

src/main/scala/com/gaocegege/scrala/core/engine/Engine.scala

@@ -58,7 +58,7 @@ class Engine(val spider: Spider, val scheduler: Scheduler) extends Actor {
     }
     case (Constant.workDownMessage) => {
       if ((scheduler count) == 0) {
-        (context system) shutdown
+        context.system.terminate
       }
     }
     case _ => logger warn ("Unexpected message")

src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/UrlBloomFilter.scala

@@ -0,0 +1,23 @@
+package com.gaocegege.scrala.core.middleware.filter.impl
+
+import com.gaocegege.scrala.core.middleware.filter.Filter
+import com.google.common.hash.{BloomFilter, Funnels}
+import org.slf4j.LoggerFactory
+import com.typesafe.scalalogging.Logger
+
+class UrlBloomFilter(expectedInsertions: Int = 1000000, falsePositiveProbability: Double = 0.01) extends Filter {
+  private val bf = BloomFilter.create[String](
+    Funnels.unencodedCharsFunnel, expectedInsertions,0.01)
+
+  val logger = Logger(LoggerFactory getLogger "urldupfilter")
+
+  def filter(url: String): Boolean = {
+    if (bf.mightContain(url)) {
+      logger debug ("Duplicate url: " + url)
+      false
+    } else {
+      bf.put(url)
+      true
+    }
+  }
+}

src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/UrlDupFilter.scala

@@ -16,7 +16,7 @@ class UrlDupFilter extends Filter {
 
   def filter(url: String): Boolean = {
     if (urlSeens contains url) {
-      logger debug ("Dumplicate Url: " + url)
+      logger debug ("Duplicate url: " + url)
       false
     } else {
       urlSeens = urlSeens + url