pax_global_header00006660000000000000000000000064135072174030014514gustar00rootroot0000000000000052 comment=3f668f68d41cf91350a4148f4fcb6720f1472428 robotstxt-1.1.1/000077500000000000000000000000001350721740300135645ustar00rootroot00000000000000robotstxt-1.1.1/.gitignore000066400000000000000000000001161350721740300155520ustar00rootroot00000000000000*.cgo?.* *.o *.so *.sublime-* .DS_Store .idea/ _cgo_* _obj _test coverage.txt robotstxt-1.1.1/.golangci.yml000066400000000000000000000004301350721740300161450ustar00rootroot00000000000000linters: enable: - goconst - gofmt - gosec - maligned - prealloc - staticcheck disable: - deadcode - structcheck - varcheck linters-settings: gofmt: simplify: true govet: check-shadowing: true maligned: suggest-new: true robotstxt-1.1.1/.travis.yml000066400000000000000000000014341350721740300156770ustar00rootroot00000000000000cache: go: true directories: - $HOME/.cache - $HOME/bin - $HOME/gopath/pkg/mod language: go go: - 1.11 - 1.12 - master install: true script: GO111MODULE=on go test -race matrix: include: - go: 1.7 install: go get -t - go: 1.8 install: go get -t - go: 1.9 install: go get -t - go: 1.10.x install: go get -t - go: 1.x env: task=coverage script: GO111MODULE=on go test -race -covermode=atomic -coverprofile=coverage.txt after_success: bash <(curl -s https://codecov.io/bash) - go: 1.x env: task=bench script: GO111MODULE=on ./script/bench - go: 1.x install: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $HOME/bin v1.16.0 env: task=clean script: GO111MODULE=on ./script/clean robotstxt-1.1.1/LICENSE000066400000000000000000000021101350721740300145630ustar00rootroot00000000000000The MIT License Copyright (c) 2010 Sergey Shepelev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. robotstxt-1.1.1/README.rst000066400000000000000000000063041350721740300152560ustar00rootroot00000000000000What ==== This is a robots.txt exclusion protocol implementation for Go language (golang). Build ===== To build and run tests run `script/test` in source directory. Contribute ========== Warm welcome. * If desired, add your name in README.rst, section Who. * Run `script/test && script/clean && echo ok` * You can ignore linter warnings, but everything else must pass. * Send your change as pull request or just a regular patch to current maintainer (see section Who). Thank you. Usage ===== As usual, no special installation is required, just import "github.com/temoto/robotstxt" run `go get` and you're ready. 1. Parse ^^^^^^^^ First of all, you need to parse robots.txt data. You can do it with functions `FromBytes(body []byte) (*RobotsData, error)` or same for `string`:: robots, err := robotstxt.FromBytes([]byte("User-agent: *\nDisallow:")) robots, err := robotstxt.FromString("User-agent: *\nDisallow:") As of 2012-10-03, `FromBytes` is the most efficient method, everything else is a wrapper for this core function. There are few convenient constructors for various purposes: * `FromResponse(*http.Response) (*RobotsData, error)` to init robots data from HTTP response. It *does not* call `response.Body.Close()`:: robots, err := robotstxt.FromResponse(resp) resp.Body.Close() if err != nil { log.Println("Error parsing robots.txt:", err.Error()) } * `FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error)` or `FromStatusAndString` if you prefer to read bytes (string) yourself. Passing status code applies following logic in line with Google's interpretation of robots.txt files: * status 2xx -> parse body with `FromBytes` and apply rules listed there. * status 4xx -> allow all (even 401/403, as recommended by Google). * other (5xx) -> disallow all, consider this a temporary unavailability. 2. Query ^^^^^^^^ Parsing robots.txt content builds a kind of logic database, which you can query with `(r *RobotsData) TestAgent(url, agent string) (bool)`. Explicit passing of agent is useful if you want to query for different agents. For single agent users there is an efficient option: `RobotsData.FindGroup(userAgent string)` returns a structure with `.Test(path string)` method and `.CrawlDelay time.Duration`. Simple query with explicit user agent. Each call will scan all rules. :: allow := robots.TestAgent("/", "FooBot") Or query several paths against same user agent for performance. :: group := robots.FindGroup("BarBot") group.Test("/") group.Test("/download.mp3") group.Test("/news/article-2012-1") Who === Honorable contributors (in undefined order): * Ilya Grigorik (igrigorik) * Martin Angers (PuerkitoBio) * Micha Gorelick (mynameisfiber) Initial commit and other: Sergey Shepelev temotor@gmail.com Flair ===== .. image:: https://travis-ci.org/temoto/robotstxt.svg?branch=master :target: https://travis-ci.org/temoto/robotstxt .. image:: https://codecov.io/gh/temoto/robotstxt/branch/master/graph/badge.svg :target: https://codecov.io/gh/temoto/robotstxt .. image:: https://goreportcard.com/badge/github.com/temoto/robotstxt :target: https://goreportcard.com/report/github.com/temoto/robotstxt robotstxt-1.1.1/codecov.yml000066400000000000000000000000671350721740300157340ustar00rootroot00000000000000codecov: token: 6bf9c7eb-69ff-4b74-8464-e2fb452d0f04 robotstxt-1.1.1/go.mod000066400000000000000000000001301350721740300146640ustar00rootroot00000000000000module github.com/temoto/robotstxt go 1.11 require github.com/stretchr/testify v1.3.0 robotstxt-1.1.1/go.sum000066400000000000000000000011401350721740300147130ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= robotstxt-1.1.1/google_test.go000066400000000000000000000113311350721740300164250ustar00rootroot00000000000000package robotstxt import ( "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( robotsCaseMatching = `user-agent: a Disallow: / user-agent: b Disallow: /* user-agent: c Disallow: /fish user-agent: d Disallow: /fish* user-agent: e Disallow: /fish/ user-agent: f Disallow: fish/ user-agent: g Disallow: /*.php user-agent: h Disallow: /*.php$ user-agent: i Disallow: /fish*.php` robotsCasePrecedence = `user-agent: a Disallow: / Allow: /p user-agent: b Disallow: /folder Allow: /folder/ user-agent: c Disallow: /*.htm Allow: /page user-agent: d Disallow: / Allow: /$ user-agent: e Disallow: / Allow: /$` ) func TestGroupOrder(t *testing.T) { const robotsCaseOrder = `user-agent: googlebot-news Disallow: / user-agent: * Disallow: / user-agent: googlebot Disallow: /` agents := []string{"Googlebot-News (Googlebot)", "Googlebot", "Googlebot-Image (Googlebot)", "Otherbot (web)", "Otherbot (News)"} paths := []string{"/1", "/3", "/3", "/2", "/2"} r, err := FromString(robotsCaseOrder) require.NoError(t, err) for i, a := range agents { expectAccess(t, r, false, paths[i], a) } } func TestSitemaps(t *testing.T) { const robotsCaseSitemaps = `sitemap: http://test.com/a user-agent: a disallow: /c sitemap: http://test.com/b user-agent: b disallow: /d user-agent: e sitemap: http://test.com/c user-agent: f disallow: /g` r, err := FromString(robotsCaseSitemaps) require.NoError(t, err) if len(r.Sitemaps) != 3 { for i, s := range r.Sitemaps { t.Logf("Sitemap %d: %s", i, s) } t.Fatalf("Expected 3 sitemaps, got %d:\n%v", len(r.Sitemaps), r.Sitemaps) } } func TestCrawlDelays(t *testing.T) { const robotsCaseDelays = `useragent: a # some comment : with colon disallow: /c user-agent: b crawldelay: 3.5 disallow: /d user-agent: e sitemap: http://test.com/c user-agent: f disallow: /g crawl-delay: 5` r, err := FromString(robotsCaseDelays) require.NoError(t, err) if len(r.Sitemaps) != 1 { t.Fatalf("Expected 1 sitemaps, got %d", len(r.Sitemaps)) } if g := r.groups["b"]; g.CrawlDelay != time.Duration(3.5*float64(time.Second)) { t.Fatalf("Expected crawl delay of 3.5 for group 2, got %v", g.CrawlDelay) } if g := r.groups["f"]; g.CrawlDelay != (5 * time.Second) { t.Fatalf("Expected crawl delay of 5 for group 3, got %v", g.CrawlDelay) } } func TestWildcards(t *testing.T) { const robotsCaseWildcards = `user-agent: * Disallow: /path*l$` r, err := FromString(robotsCaseWildcards) require.NoError(t, err) assert.Equal(t, "/path.*l$", r.groups["*"].rules[0].pattern.String()) } func TestURLMatching(t *testing.T) { var ok bool cases := map[string][]string{ "a": { "/", "/test", "", "/path/to/whatever", }, "b": { "/", "/test", "", "/path/to/whatever", }, "c": { "/fish", "/fish.html", "/fish/salmon.html", "/fishheads", "/fishheads/yummy.html", "/fish.php?id=anything", "^/Fish.asp", "^/catfish", "^/?id=fish", }, "d": { "/fish", "/fish.html", "/fish/salmon.html", "/fishheads", "/fishheads/yummy.html", "/fish.php?id=anything", "^/Fish.asp", "^/catfish", "^/?id=fish", }, "e": { "/fish/", "/fish/?id=anything", "/fish/salmon.htm", "^/fish", "^/fish.html", "^/Fish/Salmon.asp", }, "f": { "/fish/", "/fish/?id=anything", "/fish/salmon.htm", "^/fish", "^/fish.html", "^/Fish/Salmon.asp", }, "g": { "/filename.php", "/folder/filename.php", "/folder/filename.php?parameters", "/folder/any.php.file.html", "/filename.php/", "^/", "^/windows.PHP", }, "h": { "/filename.php", "/folder/filename.php", "^/filename.php?parameters", "^/filename.php/", "^/filename.php5", "^/windows.PHP", }, "i": { "/fish.php", "/fishheads/catfish.php?parameters", "^/Fish.PHP", }, } r, err := FromString(robotsCaseMatching) require.NoError(t, err) for k, ar := range cases { for _, p := range ar { ok = strings.HasPrefix(p, "^") if ok { p = p[1:] } if allow := r.TestAgent(p, k); allow != ok { t.Errorf("Agent %s, path %s, expected %v, got %v", k, p, ok, allow) } } } } func TestURLPrecedence(t *testing.T) { var ok bool cases := map[string][]string{ "a": { "/page", "^/test", }, "b": { "/folder/page", "^/folder1", "^/folder.htm", }, "c": { "^/page.htm", "/page1.asp", }, "d": { "/", "^/index", }, "e": { "^/page.htm", "/", }, } r, err := FromString(robotsCasePrecedence) require.NoError(t, err) for k, ar := range cases { for _, p := range ar { ok = !strings.HasPrefix(p, "^") if !ok { p = p[1:] } if allow := r.TestAgent(p, k); allow != ok { t.Errorf("Agent %s, path %s, expected %v, got %v", k, p, ok, allow) } } } } robotstxt-1.1.1/parser.go000066400000000000000000000154721350721740300154200ustar00rootroot00000000000000package robotstxt // Comments explaining the logic are taken from either the google's spec: // https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt // // or the Wikipedia's entry on robots.txt: // http://en.wikipedia.org/wiki/Robots.txt import ( "fmt" "io" "regexp" "strconv" "strings" "time" ) type lineType uint const ( lIgnore lineType = iota lUnknown lUserAgent lAllow lDisallow lCrawlDelay lSitemap lHost ) type parser struct { tokens []string pos int } type lineInfo struct { t lineType // Type of line key k string // String representation of the type of key vs string // String value of the key vf float64 // Float value of the key vr *regexp.Regexp // Regexp value of the key } func newParser(tokens []string) *parser { return &parser{tokens: tokens} } func parseGroupMap(groups map[string]*Group, agents []string, fun func(*Group)) { var g *Group for _, a := range agents { if g = groups[a]; g == nil { g = new(Group) groups[a] = g } fun(g) } } func (p *parser) parseAll() (groups map[string]*Group, host string, sitemaps []string, errs []error) { groups = make(map[string]*Group, 16) agents := make([]string, 0, 4) isEmptyGroup := true // Reset internal fields, tokens are assigned at creation time, never change p.pos = 0 for { if li, err := p.parseLine(); err != nil { if err == io.EOF { break } errs = append(errs, err) } else { switch li.t { case lUserAgent: // Two successive user-agent lines are part of the same group. if !isEmptyGroup { // End previous group agents = make([]string, 0, 4) } if len(agents) == 0 { isEmptyGroup = true } agents = append(agents, li.vs) case lDisallow: // Error if no current group if len(agents) == 0 { errs = append(errs, fmt.Errorf("Disallow before User-agent at token #%d.", p.pos)) } else { isEmptyGroup = false var r *rule if li.vr != nil { r = &rule{"", false, li.vr} } else { r = &rule{li.vs, false, nil} } parseGroupMap(groups, agents, func(g *Group) { g.rules = append(g.rules, r) }) } case lAllow: // Error if no current group if len(agents) == 0 { errs = append(errs, fmt.Errorf("Allow before User-agent at token #%d.", p.pos)) } else { isEmptyGroup = false var r *rule if li.vr != nil { r = &rule{"", true, li.vr} } else { r = &rule{li.vs, true, nil} } parseGroupMap(groups, agents, func(g *Group) { g.rules = append(g.rules, r) }) } case lHost: host = li.vs case lSitemap: sitemaps = append(sitemaps, li.vs) case lCrawlDelay: if len(agents) == 0 { errs = append(errs, fmt.Errorf("Crawl-delay before User-agent at token #%d.", p.pos)) } else { isEmptyGroup = false delay := time.Duration(li.vf * float64(time.Second)) parseGroupMap(groups, agents, func(g *Group) { g.CrawlDelay = delay }) } } } } return } func (p *parser) parseLine() (li *lineInfo, err error) { t1, ok1 := p.popToken() if !ok1 { // proper EOF return nil, io.EOF } t2, ok2 := p.peekToken() if !ok2 { // EOF, no value associated with the token, so ignore token and return return nil, io.EOF } // Helper closure for all string-based tokens, common behaviour: // - Consume t2 token // - If empty, return unknown line info // - Otherwise return the specified line info returnStringVal := func(t lineType) (*lineInfo, error) { p.popToken() if t2 != "" { return &lineInfo{t: t, k: t1, vs: t2}, nil } return &lineInfo{t: lIgnore}, nil } // Helper closure for all path tokens (allow/disallow), common behaviour: // - Consume t2 token // - If empty, return unknown line info // - Otherwise, normalize the path (add leading "/" if missing, remove trailing "*") // - Detect if wildcards are present, if so, compile into a regexp // - Return the specified line info returnPathVal := func(t lineType) (*lineInfo, error) { p.popToken() if t2 != "" { if !strings.HasPrefix(t2, "*") && !strings.HasPrefix(t2, "/") { t2 = "/" + t2 } if strings.HasSuffix(t2, "*") { t2 = strings.TrimRight(t2, "*") } // From google's spec: // Google, Bing, Yahoo, and Ask support a limited form of // "wildcards" for path values. These are: // * designates 0 or more instances of any valid character // $ designates the end of the URL if strings.ContainsAny(t2, "*$") { // Must compile a regexp, this is a pattern. // Escape string before compile. t2 = regexp.QuoteMeta(t2) t2 = strings.Replace(t2, `\*`, `.*`, -1) t2 = strings.Replace(t2, `\$`, `$`, -1) if r, e := regexp.Compile(t2); e != nil { return nil, e } else { return &lineInfo{t: t, k: t1, vr: r}, nil } } else { // Simple string path return &lineInfo{t: t, k: t1, vs: t2}, nil } } return &lineInfo{t: lIgnore}, nil } switch strings.ToLower(t1) { case tokEOL: // Don't consume t2 and continue parsing return &lineInfo{t: lIgnore}, nil case "user-agent", "useragent": // From google's spec: // Handling of elements with simple errors / typos (eg "useragent" // instead of "user-agent") is undefined and may be interpreted as correct // directives by some user-agents. // The user-agent is non-case-sensitive. t2 = strings.ToLower(t2) return returnStringVal(lUserAgent) case "disallow": // From google's spec: // When no path is specified, the directive is ignored (so an empty Disallow // CAN be an allow, since allow is the default. The actual result depends // on the other rules in the group). return returnPathVal(lDisallow) case "allow": // From google's spec: // When no path is specified, the directive is ignored. return returnPathVal(lAllow) case "host": // Host directive to specify main site mirror // Read more: https://help.yandex.com/webmaster/controlling-robot/robots-txt.xml#host return returnStringVal(lHost) case "sitemap": // Non-group field, applies to the host as a whole, not to a specific user-agent return returnStringVal(lSitemap) case "crawl-delay", "crawldelay": // From http://en.wikipedia.org/wiki/Robots_exclusion_standard#Nonstandard_extensions // Several major crawlers support a Crawl-delay parameter, set to the // number of seconds to wait between successive requests to the same server. p.popToken() if cd, e := strconv.ParseFloat(t2, 64); e != nil { return nil, e } else { return &lineInfo{t: lCrawlDelay, k: t1, vf: cd}, nil } } // Consume t2 token p.popToken() return &lineInfo{t: lUnknown, k: t1}, nil } func (p *parser) popToken() (tok string, ok bool) { tok, ok = p.peekToken() if !ok { return } p.pos++ return tok, true } func (p *parser) peekToken() (tok string, ok bool) { if p.pos >= len(p.tokens) { return "", false } return p.tokens[p.pos], true } robotstxt-1.1.1/robots.txt-check/000077500000000000000000000000001350721740300167655ustar00rootroot00000000000000robotstxt-1.1.1/robots.txt-check/main.go000066400000000000000000000014251350721740300202420ustar00rootroot00000000000000package main import ( "flag" "log" "net/http" "strings" "github.com/temoto/robotstxt" ) var checkPaths = []string{ "/", } func main() { robotsUrl := flag.String("robots-url", "", "") bot := flag.String("bot", "GoogleBot", "") flag.Parse() if *robotsUrl == "" { log.Fatalln("Robots URL is empty, run with -h to see usage.") } if !strings.HasPrefix(*robotsUrl, "http") { *robotsUrl = "http://" + *robotsUrl } response, err := http.Get(*robotsUrl) if err != nil { log.Fatalln("HTTP error:", err) } robots, err := robotstxt.FromResponse(response) if err != nil { log.Fatalln("Robots.txt error:", err) } log.Println("Running checks as", *bot) group := robots.FindGroup(*bot) for _, path := range checkPaths { log.Println(path, ":", group.Test(path)) } } robotstxt-1.1.1/robotstxt.go000066400000000000000000000131401350721740300161620ustar00rootroot00000000000000// Package robotstxt implements the robots.txt Exclusion Protocol // as specified in http://www.robotstxt.org/wc/robots.html // with various extensions. package robotstxt // Comments explaining the logic are taken from either the Google's spec: // https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt import ( "bytes" "errors" "io/ioutil" "net/http" "regexp" "strconv" "strings" "time" ) type RobotsData struct { // private groups map[string]*Group allowAll bool disallowAll bool Host string Sitemaps []string } type Group struct { rules []*rule Agent string CrawlDelay time.Duration } type rule struct { path string allow bool pattern *regexp.Regexp } type ParseError struct { Errs []error } func newParseError(errs []error) *ParseError { return &ParseError{errs} } func (e ParseError) Error() string { var b bytes.Buffer b.WriteString("Parse error(s): " + "\n") for _, er := range e.Errs { b.WriteString(er.Error() + "\n") } return b.String() } var allowAll = &RobotsData{allowAll: true} var disallowAll = &RobotsData{disallowAll: true} var emptyGroup = &Group{} func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) { switch { case statusCode >= 200 && statusCode < 300: return FromBytes(body) // From https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt // // Google treats all 4xx errors in the same way and assumes that no valid // robots.txt file exists. It is assumed that there are no restrictions. // This is a "full allow" for crawling. Note: this includes 401 // "Unauthorized" and 403 "Forbidden" HTTP result codes. case statusCode >= 400 && statusCode < 500: return allowAll, nil // From Google's spec: // Server errors (5xx) are seen as temporary errors that result in a "full // disallow" of crawling. case statusCode >= 500 && statusCode < 600: return disallowAll, nil } return nil, errors.New("Unexpected status: " + strconv.Itoa(statusCode)) } func FromStatusAndString(statusCode int, body string) (*RobotsData, error) { return FromStatusAndBytes(statusCode, []byte(body)) } func FromResponse(res *http.Response) (*RobotsData, error) { if res == nil { // Edge case, if res is nil, return nil data return nil, nil } buf, e := ioutil.ReadAll(res.Body) if e != nil { return nil, e } return FromStatusAndBytes(res.StatusCode, buf) } func FromBytes(body []byte) (r *RobotsData, err error) { var errs []error // special case (probably not worth optimization?) trimmed := bytes.TrimSpace(body) if len(trimmed) == 0 { return allowAll, nil } sc := newByteScanner("bytes", true) //sc.Quiet = !print_errors sc.feed(body, true) tokens := sc.scanAll() // special case worth optimization if len(tokens) == 0 { return allowAll, nil } r = &RobotsData{} parser := newParser(tokens) r.groups, r.Host, r.Sitemaps, errs = parser.parseAll() if len(errs) > 0 { return nil, newParseError(errs) } return r, nil } func FromString(body string) (r *RobotsData, err error) { return FromBytes([]byte(body)) } func (r *RobotsData) TestAgent(path, agent string) bool { if r.allowAll { return true } if r.disallowAll { return false } // Find a group of rules that applies to this agent // From Google's spec: // The user-agent is non-case-sensitive. g := r.FindGroup(agent) return g.Test(path) } // FindGroup searches block of declarations for specified user-agent. // From Google's spec: // Only one group of group-member records is valid for a particular crawler. // The crawler must determine the correct group of records by finding the group // with the most specific user-agent that still matches. All other groups of // records are ignored by the crawler. The user-agent is non-case-sensitive. // The order of the groups within the robots.txt file is irrelevant. func (r *RobotsData) FindGroup(agent string) (ret *Group) { var prefixLen int agent = strings.ToLower(agent) if ret = r.groups["*"]; ret != nil { // Weakest match possible prefixLen = 1 } for a, g := range r.groups { if a != "*" && strings.HasPrefix(agent, a) { if l := len(a); l > prefixLen { prefixLen = l ret = g } } } if ret == nil { return emptyGroup } return } func (g *Group) Test(path string) bool { if r := g.findRule(path); r != nil { return r.allow } // From Google's spec: // By default, there are no restrictions for crawling for the designated crawlers. return true } // From Google's spec: // The path value is used as a basis to determine whether or not a rule applies // to a specific URL on a site. With the exception of wildcards, the path is // used to match the beginning of a URL (and any valid URLs that start with the // same path). // // At a group-member level, in particular for allow and disallow directives, // the most specific rule based on the length of the [path] entry will trump // the less specific (shorter) rule. The order of precedence for rules with // wildcards is undefined. func (g *Group) findRule(path string) (ret *rule) { var prefixLen int for _, r := range g.rules { if r.pattern != nil { if r.pattern.MatchString(path) { // Consider this a match equal to the length of the pattern. // From Google's spec: // The order of precedence for rules with wildcards is undefined. if l := len(r.pattern.String()); l > prefixLen { prefixLen = len(r.pattern.String()) ret = r } } } else if r.path == "/" && prefixLen == 0 { // Weakest match possible prefixLen = 1 ret = r } else if strings.HasPrefix(path, r.path) { if l := len(r.path); l > prefixLen { prefixLen = l ret = r } } } return } robotstxt-1.1.1/robotstxt_test.go000066400000000000000000000307151350721740300172300ustar00rootroot00000000000000package robotstxt import ( "io/ioutil" "net/http" "strconv" "strings" "testing" "testing/quick" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStatus(t *testing.T) { t.Parallel() type tcase struct { code int allow bool } cases := []tcase{ {200, true}, {400, true}, {401, true}, {403, true}, {404, true}, {500, false}, {502, false}, {503, false}, {504, false}, } for _, c := range cases { t.Run(strconv.Itoa(c.code), func(t *testing.T) { r, err := FromResponse(newHttpResponse(c.code, "")) require.NoError(t, err) expectAll(t, r, c.allow) r, err = FromStatusAndString(c.code, "") require.NoError(t, err) expectAll(t, r, c.allow) }) } } func TestFromStringDisallowAll(t *testing.T) { r, err := FromString("User-Agent: *\r\nDisallow: /\r\n") require.NoError(t, err) expectAll(t, r, false) } func TestFromString002(t *testing.T) { t.Parallel() r, err := FromString("User-Agent: *\r\nDisallow: /account\r\n") require.NoError(t, err) expectAllAgents(t, r, true, "/foobar") expectAllAgents(t, r, false, "/account") expectAllAgents(t, r, false, "/account/sub") } const robotsText001 = "User-agent: * \nDisallow: /administrator/\nDisallow: /cache/\nDisallow: /components/\nDisallow: /editor/\nDisallow: /forum/\nDisallow: /help/\nDisallow: /images/\nDisallow: /includes/\nDisallow: /language/\nDisallow: /mambots/\nDisallow: /media/\nDisallow: /modules/\nDisallow: /templates/\nDisallow: /installation/\nDisallow: /getcid/\nDisallow: /tooltip/\nDisallow: /getuser/\nDisallow: /download/\nDisallow: /index.php?option=com_phorum*,quote=1\nDisallow: /index.php?option=com_phorum*phorum_query=search\nDisallow: /index.php?option=com_phorum*,newer\nDisallow: /index.php?option=com_phorum*,older\n\nUser-agent: Yandex\nAllow: /\nSitemap: http://www.pravorulya.com/sitemap.xml\nSitemap: http://www.pravorulya.com/sitemap1.xml" func TestFromString003(t *testing.T) { t.Parallel() r, err := FromString(robotsText001) require.NoError(t, err) expectAllAgents(t, r, false, "/administrator/") expectAllAgents(t, r, true, "/paruram") } func TestInvalidEncoding(t *testing.T) { // Invalid UTF-8 encoding should not break parser. _, err := FromString("User-agent: H\xef\xbf\xbdm�h�kki\nDisallow: *") require.NoError(t, err) } // http://www.google.com/robots.txt on Wed, 12 Jan 2011 12:22:20 GMT const robotsGoogle = ("User-agent: *\nDisallow: /search\nDisallow: /groups\nDisallow: /images\nDisallow: /catalogs\nDisallow: /catalogues\nDisallow: /news\nAllow: /news/directory\nDisallow: /nwshp\nDisallow: /setnewsprefs?\nDisallow: /index.html?\nDisallow: /?\nDisallow: /addurl/image?\nDisallow: /pagead/\nDisallow: /relpage/\nDisallow: /relcontent\nDisallow: /imgres\nDisallow: /imglanding\nDisallow: /keyword/\nDisallow: /u/\nDisallow: /univ/\nDisallow: /cobrand\nDisallow: /custom\nDisallow: /advanced_group_search\nDisallow: /googlesite\nDisallow: /preferences\nDisallow: /setprefs\nDisallow: /swr\nDisallow: /url\nDisallow: /default\nDisallow: /m?\nDisallow: /m/?\nDisallow: /m/blogs?\nDisallow: /m/directions?\nDisallow: /m/ig\nDisallow: /m/images?\nDisallow: /m/local?\nDisallow: /m/movies?\nDisallow: /m/news?\nDisallow: /m/news/i?\nDisallow: /m/place?\nDisallow: /m/products?\nDisallow: /m/products/\nDisallow: /m/setnewsprefs?\nDisallow: /m/search?\nDisallow: /m/swmloptin?\nDisallow: /m/trends\nDisallow: /m/video?\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /sprint_xhtml\nDisallow: /sprint_wml\nDisallow: /pqa\nDisallow: /palm\nDisallow: /gwt/\nDisallow: /purchases\nDisallow: /hws\nDisallow: /bsd?\nDisallow: /linux?\nDisallow: /mac?\nDisallow: /microsoft?\nDisallow: /unclesam?\nDisallow: /answers/search?q=\nDisallow: /local?\nDisallow: /local_url\nDisallow: /froogle?\nDisallow: /products?\nDisallow: /products/\nDisallow: /froogle_\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books\nDisallow: /bkshp?q=\nAllow: /booksrightsholders\nDisallow: /patents?\nDisallow: /patents/\nAllow: /patents/about\nDisallow: /scholar\nDisallow: /complete\nDisallow: /sponsoredlinks\nDisallow: /videosearch?\nDisallow: /videopreview?\nDisallow: /videoprograminfo?\nDisallow: /maps?\nDisallow: /mapstt?\nDisallow: /mapslt?\nDisallow: /maps/stk/\nDisallow: /maps/br?\nDisallow: /mapabcpoi?\nDisallow: /maphp?\nDisallow: /places/\nAllow: /places/$\nDisallow: /maps/place\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /lochp?\nDisallow: /center\nDisallow: /ie?\nDisallow: /sms/demo?\nDisallow: /katrina?\nDisallow: /blogsearch?\nDisallow: /blogsearch/\nDisallow: /blogsearch_feeds\nDisallow: /advanced_blog_search\nDisallow: /reader/\nAllow: /reader/play\nDisallow: /uds/\nDisallow: /chart?\nDisallow: /transit?\nDisallow: /mbd?\nDisallow: /extern_js/\nDisallow: /calendar/feeds/\nDisallow: /calendar/ical/\nDisallow: /cl2/feeds/\n" + "Disallow: /cl2/ical/\nDisallow: /coop/directory\nDisallow: /coop/manage\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /notebook/search?\nDisallow: /musica\nDisallow: /musicad\nDisallow: /musicas\nDisallow: /musicl\nDisallow: /musics\nDisallow: /musicsearch\nDisallow: /musicsp\nDisallow: /musiclp\nDisallow: /browsersync\nDisallow: /call\nDisallow: /archivesearch?\nDisallow: /archivesearch/url\nDisallow: /archivesearch/advanced_search\nDisallow: /base/reportbadoffer\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /codesearch?\nDisallow: /codesearch/feeds/search?\nDisallow: /wapsearch?\nDisallow: /safebrowsing\nAllow: /safebrowsing/diagnostic\nAllow: /safebrowsing/report_error/\nAllow: /safebrowsing/report_phish/\nDisallow: /reviews/search?\nDisallow: /orkut/albums\nAllow: /jsapi\nDisallow: /views?\nDisallow: /c/\nDisallow: /cbk\nDisallow: /recharge/dashboard/car\nDisallow: /recharge/dashboard/static/\nDisallow: /translate_a/\nDisallow: /translate_c\nDisallow: /translate_f\nDisallow: /translate_static/\nDisallow: /translate_suggestion\nDisallow: /profiles/me\nAllow: /profiles\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/photos\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /gcc/\nDisallow: /aclk\nDisallow: /cse?\nDisallow: /cse/home\nDisallow: /cse/panel\nDisallow: /cse/manage\nDisallow: /tbproxy/\nDisallow: /imesync/\nDisallow: /shenghuo/search?\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /ppob/?\nDisallow: /ppob?\nDisallow: /ig/add?\nDisallow: /adwordsresellers\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /topicsearch?q=\nDisallow: /xfx7/\nDisallow: /squared/api\nDisallow: /squared/search\nDisallow: /squared/table\nDisallow: /toolkit/\nAllow: /toolkit/*.html\nDisallow: /globalmarketfinder/\nAllow: /globalmarketfinder/*.html\nDisallow: /qnasearch?\nDisallow: /errors/\nDisallow: /app/updates\nDisallow: /sidewiki/entry/\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /buzz/post\nDisallow: /compressiontest/\nDisallow: /analytics/reporting/\nDisallow: /analytics/admin/\nDisallow: /analytics/web/\nDisallow: /analytics/feeds/\nDisallow: /analytics/settings/\nDisallow: /alerts/\nDisallow: /phone/compare/?\nAllow: /alerts/manage\nSitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml\nSitemap: http://www.google.com/hostednews/sitemap_index.xml\nSitemap: http://www.google.com/ventures/sitemap_ventures.xml\nSitemap: http://www.google.com/sitemaps_webmasters.xml\nSitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml\nSitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml") func TestFromGoogle(t *testing.T) { t.Parallel() r, err := FromString(robotsGoogle) require.NoError(t, err) expectAllAgents(t, r, true, "/ncr") expectAllAgents(t, r, false, "/search") } func TestAllowAll(t *testing.T) { t.Parallel() cases := []string{ "", "# comment", "User-Agent: * \nAllow: /", "User-Agent: * \nDisallow: ", } for i, input := range cases { t.Run(strconv.Itoa(i), func(t *testing.T) { r, err := FromString(input) require.NoError(t, err) expectAll(t, r, true) }) } } func TestRobotstxtOrg(t *testing.T) { t.Parallel() const robotsText005 = `User-agent: Google Disallow: User-agent: * Disallow: /` r, err := FromString(robotsText005) require.NoError(t, err) expectAccess(t, r, false, "/path/page1.html", "SomeBot") expectAccess(t, r, true, "/path/page1.html", "Googlebot") } func TestHost(t *testing.T) { type tcase struct { input string expect string } cases := []tcase{ {"#Host: site.ru", ""}, {"Host: site.ru", "site.ru"}, {"Host: яндекс.рф", "яндекс.рф"}, } for _, c := range cases { t.Run(c.input, func(t *testing.T) { r, err := FromString(c.input) require.NoError(t, err) assert.Equal(t, c.expect, r.Host) }) } } func TestParseErrors(t *testing.T) { const robotsTextErrs = `Disallow: / User-agent: Google Crawl-delay: bad-time-value` _, err := FromString(robotsTextErrs) assert.Error(t, err) pe, ok := err.(*ParseError) require.True(t, ok, "Expected ParseError") require.Equal(t, 2, len(pe.Errs)) assert.Contains(t, pe.Errs[0].Error(), "Disallow before User-agent") assert.Contains(t, pe.Errs[1].Error(), "bad-time-value") assert.Contains(t, pe.Errs[1].Error(), "invalid syntax") } const robotsTextJustHTML = `

Hello world! This is valid HTML but invalid robots.txt.` func TestHtmlInstead(t *testing.T) { r, err := FromString(robotsTextJustHTML) // According to Google spec, invalid robots.txt file // must be parsed silently. require.NoError(t, err) group := r.FindGroup("SuperBot") require.NotNil(t, group) assert.True(t, group.Test("/")) } // http://perche.vanityfair.it/robots.txt on Sat, 13 Sep 2014 23:00:29 GMT const robotsTextVanityfair = "\xef\xbb\xbfUser-agent: *\nDisallow: */oroscopo-di-oggi/*" func TestWildcardPrefix(t *testing.T) { t.Parallel() r, err := FromString(robotsTextVanityfair) require.NoError(t, err) expectAllAgents(t, r, true, "/foo/bar") expectAllAgents(t, r, false, "/oroscopo-di-oggi/bar") expectAllAgents(t, r, false, "/foo/oroscopo-di-oggi/bar") } func TestGrouping(t *testing.T) { const robotsCaseGrouping = `user-agent: a user-agent: b disallow: /a disallow: /b user-agent: ignore Disallow: /separator user-agent: b user-agent: c disallow: /b disallow: /c` r, err := FromString(robotsCaseGrouping) require.NoError(t, err) expectAccess(t, r, false, "/a", "a") expectAccess(t, r, false, "/b", "a") expectAccess(t, r, true, "/c", "a") expectAccess(t, r, false, "/a", "b") expectAccess(t, r, false, "/b", "b") expectAccess(t, r, false, "/c", "b") expectAccess(t, r, true, "/a", "c") expectAccess(t, r, false, "/b", "c") expectAccess(t, r, false, "/c", "c") } func BenchmarkParseFromString001(b *testing.B) { input := robotsText001 b.ReportAllocs() b.SetBytes(int64(len(input))) b.ResetTimer() for i := 0; i < b.N; i++ { if _, err := FromString(input); err != nil { b.Fatal(err) } } } func BenchmarkParseFromString002(b *testing.B) { input := robotsGoogle b.ReportAllocs() b.SetBytes(int64(len(input))) b.ResetTimer() for i := 0; i < b.N; i++ { if _, err := FromString(input); err != nil { b.Fatal(err) } } } func BenchmarkParseFromStatus401(b *testing.B) { for i := 0; i < b.N; i++ { if _, err := FromStatusAndString(401, ""); err != nil { b.Fatal(err) } } } func expectAll(t *testing.T, r *RobotsData, allow bool) { // TODO fuzz path expectAllAgents(t, r, allow, "/") expectAllAgents(t, r, allow, "/admin/") expectAllAgents(t, r, allow, "/search") expectAllAgents(t, r, allow, "/.htaccess") } func expectAllAgents(t *testing.T, r *RobotsData, allow bool, path string) { f := func(agent string) bool { return expectAccess(t, r, allow, path, agent) } if err := quick.Check(f, nil); err != nil { t.Fatalf("Expected allow path '%s' %v", path, err) } } func expectAccess(t *testing.T, r *RobotsData, allow bool, path, agent string) bool { return assert.Equal(t, allow, r.TestAgent(path, agent), "path='%s' agent='%s'", path, agent) } func newHttpResponse(code int, body string) *http.Response { return &http.Response{ StatusCode: code, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Body: ioutil.NopCloser(strings.NewReader(body)), ContentLength: int64(len(body)), } } robotstxt-1.1.1/scanner.go000066400000000000000000000065331350721740300155530ustar00rootroot00000000000000package robotstxt import ( "bytes" "fmt" "go/token" "os" "sync" "unicode/utf8" ) type byteScanner struct { pos token.Position buf []byte ErrorCount int ch rune Quiet bool keyTokenFound bool lastChunk bool } const tokEOL = "\n" var WhitespaceChars = []rune{' ', '\t', '\v'} var tokBuffers = sync.Pool{New: func() interface{} { return bytes.NewBuffer(make([]byte, 32)) }} func newByteScanner(srcname string, quiet bool) *byteScanner { return &byteScanner{ Quiet: quiet, ch: -1, pos: token.Position{Filename: srcname}, } } func (s *byteScanner) feed(input []byte, end bool) { s.buf = input s.pos.Offset = 0 s.pos.Line = 1 s.pos.Column = 1 s.lastChunk = end // Read first char into look-ahead buffer `s.ch`. if !s.nextChar() { return } // Skip UTF-8 byte order mark if s.ch == 65279 { s.nextChar() s.pos.Column = 1 } } func (s *byteScanner) GetPosition() token.Position { return s.pos } func (s *byteScanner) scan() string { // Note Offset > len, not >=, so we can scan last character. if s.lastChunk && s.pos.Offset > len(s.buf) { return "" } s.skipSpace() if s.ch == -1 { return "" } // EOL if s.isEol() { s.keyTokenFound = false // skip subsequent newline chars for s.ch != -1 && s.isEol() { s.nextChar() } // emit newline as separate token return tokEOL } // skip comments if s.ch == '#' { s.keyTokenFound = false s.skipUntilEol() if s.ch == -1 { return "" } // emit newline as separate token return tokEOL } // else we found something tok := tokBuffers.Get().(*bytes.Buffer) defer tokBuffers.Put(tok) tok.Reset() tok.WriteRune(s.ch) s.nextChar() for s.ch != -1 && !s.isSpace() && !s.isEol() { // Do not consider ":" to be a token separator if a first key token // has already been found on this line (avoid cutting an absolute URL // after the "http:") if s.ch == ':' && !s.keyTokenFound { s.nextChar() s.keyTokenFound = true break } tok.WriteRune(s.ch) s.nextChar() } return tok.String() } func (s *byteScanner) scanAll() []string { results := make([]string, 0, 64) // random guess of average tokens length for { token := s.scan() if token != "" { results = append(results, token) } else { break } } return results } func (s *byteScanner) error(pos token.Position, msg string) { s.ErrorCount++ if !s.Quiet { fmt.Fprintf(os.Stderr, "robotstxt from %s: %s\n", pos.String(), msg) } } func (s *byteScanner) isEol() bool { return s.ch == '\n' || s.ch == '\r' } func (s *byteScanner) isSpace() bool { for _, r := range WhitespaceChars { if s.ch == r { return true } } return false } func (s *byteScanner) skipSpace() { for s.ch != -1 && s.isSpace() { s.nextChar() } } func (s *byteScanner) skipUntilEol() { for s.ch != -1 && !s.isEol() { s.nextChar() } // skip subsequent newline chars for s.ch != -1 && s.isEol() { s.nextChar() } } // Reads next Unicode char. func (s *byteScanner) nextChar() bool { if s.pos.Offset >= len(s.buf) { s.ch = -1 return false } s.pos.Column++ if s.ch == '\n' { s.pos.Line++ s.pos.Column = 1 } r, w := rune(s.buf[s.pos.Offset]), 1 if r >= 0x80 { r, w = utf8.DecodeRune(s.buf[s.pos.Offset:]) if r == utf8.RuneError && w == 1 { s.error(s.pos, "illegal UTF-8 encoding") } } s.pos.Column++ s.pos.Offset += w s.ch = r return true } robotstxt-1.1.1/scanner_test.go000066400000000000000000000016121350721740300166030ustar00rootroot00000000000000package robotstxt import ( "fmt" "testing" "github.com/stretchr/testify/assert" ) func TestScanner(t *testing.T) { t.Parallel() type tcase struct { input string expect []string errCount int } cases := []tcase{ {"foo", []string{"foo"}, 0}, {"\u2010", []string{"‐"}, 0}, {"# comment \r\nSomething: Somewhere\r\n", []string{tokEOL, "Something", "Somewhere", tokEOL}, 0}, {"# comment \r\n# more comments\n\nDisallow:\r", []string{tokEOL, tokEOL, "Disallow", tokEOL}, 0}, {"\xef\xbb\xbfUser-agent: *\n", []string{"User-agent", "*", tokEOL}, 0}, {"\xd9\xd9", []string{"\uFFFD\uFFFD"}, 2}, } for i, c := range cases { tag := fmt.Sprintf("test-%d", i) t.Run(tag, func(t *testing.T) { sc := newByteScanner(tag, true) sc.feed([]byte(c.input), true) tokens := sc.scanAll() assert.Equal(t, c.errCount, sc.ErrorCount) assert.Equal(t, c.expect, tokens) }) } } robotstxt-1.1.1/script/000077500000000000000000000000001350721740300150705ustar00rootroot00000000000000robotstxt-1.1.1/script/bench000077500000000000000000000004661350721740300161030ustar00rootroot00000000000000#!/bin/bash set -eu go get golang.org/x/tools/cmd/benchcmp go get go test -run=NONE -bench=. -count=3 -test.benchmem=true >bench-new git fetch origin master git stash git checkout -f FETCH_HEAD -- . go get go test -run=NONE -bench=. -count=3 -test.benchmem=true >bench-old benchcmp -best -mag bench-{old,new} robotstxt-1.1.1/script/clean000077500000000000000000000007131350721740300161010ustar00rootroot00000000000000#!/bin/bash set -eu err=0 # developer: commit fixes # CI: fail build # smarter CI: send patch (pull request) if ! git diff --exit-code >/dev/null ; then echo "$0: sorry, this script needs clean git tree, try git stash" >&2 exit 1 fi echo "$0: go fmt" go fmt ./... git diff --exit-code || err=1 echo "$0: go fix" go fix ./... git diff --exit-code || err=1 echo "$0: go vet" go vet ./... || err=1 echo "$0: lint" golangci-lint run || err=1 exit $err