@@ -27,19 +27,16 @@ or via Yarn:
2727``` js
2828var robotsParser = require (' robots-parser' );
2929
30- var robots = robotsParser (
31- ' http://www.example.com/robots.txt' ,
32- [
33- ' User-agent: *' ,
34- ' Disallow: /dir/' ,
35- ' Disallow: /test.html' ,
36- ' Allow: /dir/test.html' ,
37- ' Allow: /test.html' ,
38- ' Crawl-delay: 1' ,
39- ' Sitemap: http://example.com/sitemap.xml' ,
40- ' Host: example.com'
41- ].join (' \n ' )
42- );
30+ var robots = robotsParser (' http://www.example.com/robots.txt' , [
31+ ' User-agent: *' ,
32+ ' Disallow: /dir/' ,
33+ ' Disallow: /test.html' ,
34+ ' Allow: /dir/test.html' ,
35+ ' Allow: /test.html' ,
36+ ' Crawl-delay: 1' ,
37+ ' Sitemap: http://example.com/sitemap.xml' ,
38+ ' Host: example.com'
39+ ].join (' \n ' ));
4340
4441robots .isAllowed (' http://www.example.com/test.html' , ' Sams-Bot/1.0' ); // true
4542robots .isAllowed (' http://www.example.com/dir/test.html' , ' Sams-Bot/1.0' ); // true
@@ -113,16 +110,13 @@ Returns the preferred host name specified by the `host:` directive or null if th
113110 For example:
114111
115112 ``` js
116- var robots = robotsParser (
117- ' /robots.txt' ,
118- [
119- ' User-agent: *' ,
120- ' Disallow: /dir/' ,
121- ' Disallow: /test.html' ,
122- ' Allow: /dir/test.html' ,
123- ' Allow: /test.html'
124- ].join (' \n ' )
125- );
113+ var robots = robotsParser (' /robots.txt' , [
114+ ' User-agent: *' ,
115+ ' Disallow: /dir/' ,
116+ ' Disallow: /test.html' ,
117+ ' Allow: /dir/test.html' ,
118+ ' Allow: /test.html'
119+ ].join (' \n ' ));
126120
127121 robots .isAllowed (' /test.html' , ' Sams-Bot/1.0' ); // false
128122 robots .isAllowed (' /dir/test.html' , ' Sams-Bot/1.0' ); // true
0 commit comments