avoid expensive queries that do not change certain results (#53)

Author: elcapo

src/Robots.php

@@ -41,15 +41,17 @@ public function mayIndex(string $url, ?string $userAgent = null): bool
 
         $robotsTxt = $this->robotsTxt ?? RobotsTxt::create($this->createRobotsUrl($url));
 
+        if (! $robotsTxt->allows($url, $userAgent)) {
+            return false;
+        }
+
         $content = @file_get_contents($url);
 
         if ($content === false) {
             throw new InvalidArgumentException("Could not read url `{$url}`");
         }
 
-        return
-            $robotsTxt->allows($url, $userAgent)
-            && RobotsMeta::create($content)->mayIndex()
+        return RobotsMeta::create($content)->mayIndex()
             && RobotsHeaders::create($http_response_header ?? [])->mayIndex();
     }