Documentation

Author: jimsmart

README.md

@@ -124,10 +124,10 @@ in this format, the matching will be done according to the REP specification.
 
 ## License
 
-Package grobotstxt is licensed under the terms of the
-Apache license. See [LICENSE](LICENSE) for more information.
+Like the original library, package grobotstxt is licensed under the terms of the
+Apache License, Version 2.0. See [LICENSE](LICENSE) for more information.
 
 ## Links
 
-*   Original project
+*   Original project:
     [Google robots.txt parser and matcher library](https://github.com/google/robotstxt)

robots_cc.go

@@ -606,6 +606,8 @@ func (m *RobotsMatcher) init(userAgents []string, path string) {
 	m.userAgents = userAgents
 }
 
+// AgentsAllowed parses the given robots.txt content, matching it against
+// the given userAgents and URI, and returns true if access is allowed.
 func (m *RobotsMatcher) AgentsAllowed(robotsBody string, userAgents []string, uri string) bool {
 	// Line :487
 	// The url is not normalized (escaped, percent encoded) here because the user
@@ -616,15 +618,21 @@ func (m *RobotsMatcher) AgentsAllowed(robotsBody string, userAgents []string, ur
 	return !m.disallowed()
 }
 
+// AgentsAllowed parses the given robots.txt content, matching it against
+// the given userAgents and URI, and returns true if access is allowed.
 func AgentsAllowed(robotsBody string, userAgents []string, uri string) bool {
 	return NewRobotsMatcher().AgentsAllowed(robotsBody, userAgents, uri)
 }
 
+// AgentAllowed parses the given robots.txt content, matching it against
+// the given userAgent and URI, and returns true if access is allowed.
 func (m *RobotsMatcher) AgentAllowed(robotsBody string, userAgent string, uri string) bool {
 	// Line :498
 	return m.AgentsAllowed(robotsBody, []string{userAgent}, uri)
 }
 
+// AgentAllowed parses the given robots.txt content, matching it against
+// the given userAgent and URI, and returns true if access is allowed.
 func AgentAllowed(robotsBody string, userAgent string, uri string) bool {
 	return NewRobotsMatcher().AgentAllowed(robotsBody, userAgent, uri)
 }

sitemaps.go

@@ -9,6 +9,7 @@ func (f *sitemapExtractor) Sitemaps(robotsBody string) []string {
 	return f.sitemaps
 }
 
+// Sitemaps extracts all "Sitemap:" values from the given robots.txt content.
 func Sitemaps(robotsBody string) []string {
 	return (&sitemapExtractor{}).Sitemaps(robotsBody)
 }