pax_global_header00006660000000000000000000000064143772645410014527gustar00rootroot0000000000000052 comment=91b2d484d343fd5d48d51290de72ea3a31bebb78 gospider-1.1.6/000077500000000000000000000000001437726454100133505ustar00rootroot00000000000000gospider-1.1.6/.github/000077500000000000000000000000001437726454100147105ustar00rootroot00000000000000gospider-1.1.6/.github/FUNDING.yml000066400000000000000000000011351437726454100165250ustar00rootroot00000000000000# These are supported funding model platforms github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: j3ssie open_collective: jaeles-project ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username custom: [ 'https://paypal.me/j3ssiejjj' ] gospider-1.1.6/.gitignore000066400000000000000000000000531437726454100153360ustar00rootroot00000000000000.DS_Store gospider out*/ .idea/ *.txt dist/gospider-1.1.6/.goreleaser.yml000066400000000000000000000010321437726454100162750ustar00rootroot00000000000000builds: - binary: gospider goos: - linux - windows - freebsd - openbsd - darwin goarch: - amd64 - 386 - arm - arm64 ignore: - goos: darwin goarch: 386 archives: - name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}" replacements: darwin: macos 386: i386 amd64: x86_64 wrap_in_directory: true format: zip checksum: name_template: "{{ .ProjectName }}_checksums.txt" snapshot: name_template: "{{ .Tag }}" gospider-1.1.6/LICENSE000066400000000000000000000020501437726454100143520ustar00rootroot00000000000000MIT License Copyright (c) 2020 j3ssie Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. gospider-1.1.6/README.md000066400000000000000000000130301437726454100146240ustar00rootroot00000000000000# GoSpider **GoSpider** - Fast web spider written in Go ## Painless integrate Gospider into your recon workflow?

huntersuite

Enjoying this tool? Support it's development and take your game to the next level by using HunterSuite.io

## Installation ``` GO111MODULE=on go get -u github.com/jaeles-project/gospider ``` ## Features * Fast web crawling * Brute force and parse sitemap.xml * Parse robots.txt * Generate and verify link from JavaScript files * Link Finder * Find AWS-S3 from response source * Find subdomains from response source * Get URLs from Wayback Machine, Common Crawl, Virus Total, Alien Vault * Format output easy to Grep * Support Burp input * Crawl multiple sites in parallel * Random mobile/web User-Agent ## Showcases [![asciicast](https://asciinema.org/a/301827.svg)](https://asciinema.org/a/301827) ## Usage ```shell Fast web spider written in Go - v1.1.5 by @thebl4ckturtle & @j3ssiejjj Usage: gospider [flags] Flags: -s, --site string Site to crawl -S, --sites string Site list to crawl -p, --proxy string Proxy (Ex: http://127.0.0.1:8080) -o, --output string Output folder -u, --user-agent string User Agent to use web: random web user-agent mobi: random mobile user-agent or you can set your special user-agent (default "web") --cookie string Cookie to use (testA=a; testB=b) -H, --header stringArray Header to use (Use multiple flag to set multiple header) --burp string Load headers and cookie from burp raw http request --blacklist string Blacklist URL Regex --whitelist string Whitelist URL Regex --whitelist-domain string Whitelist Domain -t, --threads int Number of threads (Run sites in parallel) (default 1) -c, --concurrent int The number of the maximum allowed concurrent requests of the matching domains (default 5) -d, --depth int MaxDepth limits the recursion depth of visited URLs. (Set it to 0 for infinite recursion) (default 1) -k, --delay int Delay is the duration to wait before creating a new request to the matching domains (second) -K, --random-delay int RandomDelay is the extra randomized duration to wait added to Delay before creating a new request (second) -m, --timeout int Request timeout (second) (default 10) -B, --base Disable all and only use HTML content --js Enable linkfinder in javascript file (default true) --subs Include subdomains --sitemap Try to crawl sitemap.xml --robots Try to crawl robots.txt (default true) -a, --other-source Find URLs from 3rd party (Archive.org, CommonCrawl.org, VirusTotal.com, AlienVault.com) -w, --include-subs Include subdomains crawled from 3rd party. Default is main domain -r, --include-other-source Also include other-source's urls (still crawl and request) --debug Turn on debug mode --json Enable JSON output -v, --verbose Turn on verbose -l, --length Turn on length -L, --filter-length Turn on length filter -R, --raw Turn on raw -q, --quiet Suppress all the output and only show URL --no-redirect Disable redirect --version Check version -h, --help help for gospider ``` ## Example commands ### Quite output ``` gospider -q -s "https://google.com/" ``` #### Run with single site ``` gospider -s "https://google.com/" -o output -c 10 -d 1 ``` #### Run with site list ``` gospider -S sites.txt -o output -c 10 -d 1 ``` #### Run with 20 sites at the same time with 10 bot each site ``` gospider -S sites.txt -o output -c 10 -d 1 -t 20 ``` #### Also get URLs from 3rd party (Archive.org, CommonCrawl.org, VirusTotal.com, AlienVault.com) ``` gospider -s "https://google.com/" -o output -c 10 -d 1 --other-source ``` #### Also get URLs from 3rd party (Archive.org, CommonCrawl.org, VirusTotal.com, AlienVault.com) and include subdomains ``` gospider -s "https://google.com/" -o output -c 10 -d 1 --other-source --include-subs ``` #### Use custom header/cookies ``` gospider -s "https://google.com/" -o output -c 10 -d 1 --other-source -H "Accept: */*" -H "Test: test" --cookie "testA=a; testB=b" gospider -s "https://google.com/" -o output -c 10 -d 1 --other-source --burp burp_req.txt ``` #### Blacklist url/file extension. **P/s**: gospider blacklisted `.(jpg|jpeg|gif|css|tif|tiff|png|ttf|woff|woff2|ico)` as default ``` gospider -s "https://google.com/" -o output -c 10 -d 1 --blacklist ".(woff|pdf)" ``` #### Show and Blacklist file length. ``` gospider -s "https://google.com/" -o output -c 10 -d 1 --length --filter-length "6871,24432" ``` ## License `Gospider` is made with ♥ by [@j3ssiejjj](https://twitter.com/j3ssiejjj) & [@thebl4ckturtle](https://twitter.com/thebl4ckturtle) and it is released under the MIT license. ## Donation [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://paypal.me/j3ssiejjj) gospider-1.1.6/core/000077500000000000000000000000001437726454100143005ustar00rootroot00000000000000gospider-1.1.6/core/crawler.go000066400000000000000000000504151437726454100162730ustar00rootroot00000000000000package core import ( "bufio" "crypto/tls" "fmt" "net" "net/http" "net/url" "os" "regexp" "strings" "time" "strconv" jsoniter "github.com/json-iterator/go" "github.com/gocolly/colly/v2" "github.com/gocolly/colly/v2/extensions" "github.com/jaeles-project/gospider/stringset" "github.com/spf13/cobra" ) var DefaultHTTPTransport = &http.Transport{ DialContext: (&net.Dialer{ Timeout: 10 * time.Second, // Default is 15 seconds KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConns: 100, MaxConnsPerHost: 1000, IdleConnTimeout: 30 * time.Second, // ExpectContinueTimeout: 1 * time.Second, // ResponseHeaderTimeout: 3 * time.Second, // DisableCompression: false, TLSClientConfig: &tls.Config{InsecureSkipVerify: true, Renegotiation: tls.RenegotiateOnceAsClient}, } type Crawler struct { cmd *cobra.Command C *colly.Collector LinkFinderCollector *colly.Collector Output *Output subSet *stringset.StringFilter awsSet *stringset.StringFilter jsSet *stringset.StringFilter urlSet *stringset.StringFilter formSet *stringset.StringFilter site *url.URL domain string Input string Quiet bool JsonOutput bool length bool raw bool subs bool filterLength_slice []int } type SpiderOutput struct { Input string `json:"input"` Source string `json:"source"` OutputType string `json:"type"` Output string `json:"output"` StatusCode int `json:"status"` Length int `json:"length"` } func NewCrawler(site *url.URL, cmd *cobra.Command) *Crawler { domain := GetDomain(site) if domain == "" { Logger.Error("Failed to parse domain") os.Exit(1) } Logger.Infof("Start crawling: %s", site) quiet, _ := cmd.Flags().GetBool("quiet") jsonOutput, _ := cmd.Flags().GetBool("json") maxDepth, _ := cmd.Flags().GetInt("depth") concurrent, _ := cmd.Flags().GetInt("concurrent") delay, _ := cmd.Flags().GetInt("delay") randomDelay, _ := cmd.Flags().GetInt("random-delay") length, _ := cmd.Flags().GetBool("length") raw, _ := cmd.Flags().GetBool("raw") subs, _ := cmd.Flags().GetBool("subs") c := colly.NewCollector( colly.Async(true), colly.MaxDepth(maxDepth), colly.IgnoreRobotsTxt(), ) // Setup http client client := &http.Client{} // Set proxy proxy, _ := cmd.Flags().GetString("proxy") if proxy != "" { Logger.Infof("Proxy: %s", proxy) pU, err := url.Parse(proxy) if err != nil { Logger.Error("Failed to set proxy") } else { DefaultHTTPTransport.Proxy = http.ProxyURL(pU) } } // Set request timeout timeout, _ := cmd.Flags().GetInt("timeout") if timeout == 0 { Logger.Info("Your input timeout is 0. Gospider will set it to 10 seconds") client.Timeout = 10 * time.Second } else { client.Timeout = time.Duration(timeout) * time.Second } // Disable redirect noRedirect, _ := cmd.Flags().GetBool("no-redirect") if noRedirect { client.CheckRedirect = func(req *http.Request, via []*http.Request) error { nextLocation := req.Response.Header.Get("Location") Logger.Debugf("Found Redirect: %s", nextLocation) // Allow in redirect from http to https or in same hostname // We just check contain hostname or not because we set URLFilter in main collector so if // the URL is https://otherdomain.com/?url=maindomain.com, it will reject it if strings.Contains(nextLocation, site.Hostname()) { Logger.Infof("Redirecting to: %s", nextLocation) return nil } return http.ErrUseLastResponse } } // Set client transport client.Transport = DefaultHTTPTransport c.SetClient(client) // Get headers here to overwrite if "burp" flag used burpFile, _ := cmd.Flags().GetString("burp") if burpFile != "" { bF, err := os.Open(burpFile) if err != nil { Logger.Errorf("Failed to open Burp File: %s", err) } else { rd := bufio.NewReader(bF) req, err := http.ReadRequest(rd) if err != nil { Logger.Errorf("Failed to Parse Raw Request in %s: %s", burpFile, err) } else { // Set cookie c.OnRequest(func(r *colly.Request) { r.Headers.Set("Cookie", GetRawCookie(req.Cookies())) }) // Set headers c.OnRequest(func(r *colly.Request) { for k, v := range req.Header { r.Headers.Set(strings.TrimSpace(k), strings.TrimSpace(v[0])) } }) } } } // Set cookies cookie, _ := cmd.Flags().GetString("cookie") if cookie != "" && burpFile == "" { c.OnRequest(func(r *colly.Request) { r.Headers.Set("Cookie", cookie) }) } // Set headers headers, _ := cmd.Flags().GetStringArray("header") if burpFile == "" { for _, h := range headers { headerArgs := strings.SplitN(h, ":", 2) headerKey := strings.TrimSpace(headerArgs[0]) headerValue := strings.TrimSpace(headerArgs[1]) c.OnRequest(func(r *colly.Request) { r.Headers.Set(headerKey, headerValue) }) } } // Set User-Agent randomUA, _ := cmd.Flags().GetString("user-agent") switch ua := strings.ToLower(randomUA); { case ua == "mobi": extensions.RandomMobileUserAgent(c) case ua == "web": extensions.RandomUserAgent(c) default: c.UserAgent = ua } // Set referer extensions.Referer(c) // Init Output var output *Output outputFolder, _ := cmd.Flags().GetString("output") if outputFolder != "" { filename := strings.ReplaceAll(site.Hostname(), ".", "_") output = NewOutput(outputFolder, filename) } // Init Length Filter filterLength_slice := []int{} filterLength, _ := cmd.Flags().GetString("filter-length") if filterLength != "" { lengthArgs := strings.Split(filterLength, ",") for i:=0; i < len(lengthArgs);i++ { if i, err := strconv.Atoi(lengthArgs[i]); err == nil { filterLength_slice = append(filterLength_slice,i) } } } // Set url whitelist regex reg :="" if subs { reg = site.Hostname() }else { reg ="(?:https|http)://"+site.Hostname() } sRegex := regexp.MustCompile(reg) c.URLFilters = append(c.URLFilters, sRegex) // Set Limit Rule err := c.Limit(&colly.LimitRule{ DomainGlob: "*", Parallelism: concurrent, Delay: time.Duration(delay) * time.Second, RandomDelay: time.Duration(randomDelay) * time.Second, }) if err != nil { Logger.Errorf("Failed to set Limit Rule: %s", err) os.Exit(1) } // GoSpider default disallowed regex disallowedRegex := `(?i)\.(png|apng|bmp|gif|ico|cur|jpg|jpeg|jfif|pjp|pjpeg|svg|tif|tiff|webp|xbm|3gp|aac|flac|mpg|mpeg|mp3|mp4|m4a|m4v|m4p|oga|ogg|ogv|mov|wav|webm|eot|woff|woff2|ttf|otf|css)(?:\?|#|$)` c.DisallowedURLFilters = append(c.DisallowedURLFilters, regexp.MustCompile(disallowedRegex)) // Set optional blacklist url regex blacklists, _ := cmd.Flags().GetString("blacklist") if blacklists != "" { c.DisallowedURLFilters = append(c.DisallowedURLFilters, regexp.MustCompile(blacklists)) } // Set optional whitelist url regex whiteLists, _ := cmd.Flags().GetString("whitelist") if whiteLists != "" { c.URLFilters = make([]*regexp.Regexp, 0) c.URLFilters = append(c.URLFilters, regexp.MustCompile(whiteLists)) } whiteListDomain, _ := cmd.Flags().GetString("whitelist-domain") if whiteListDomain != "" { c.URLFilters = make([]*regexp.Regexp, 0) c.URLFilters = append(c.URLFilters, regexp.MustCompile("http(s)?://"+whiteListDomain)) } linkFinderCollector := c.Clone() // Try to request as much as Javascript source and don't care about domain. // The result of link finder will be send to Link Finder Collector to check is it working or not. linkFinderCollector.URLFilters = nil if whiteLists != "" { linkFinderCollector.URLFilters = append(linkFinderCollector.URLFilters, regexp.MustCompile(whiteLists)) } if whiteListDomain != "" { linkFinderCollector.URLFilters = append(linkFinderCollector.URLFilters, regexp.MustCompile("http(s)?://"+whiteListDomain)) } return &Crawler{ cmd: cmd, C: c, LinkFinderCollector: linkFinderCollector, site: site, Quiet: quiet, Input: site.String(), JsonOutput: jsonOutput, length: length, raw: raw, domain: domain, Output: output, urlSet: stringset.NewStringFilter(), subSet: stringset.NewStringFilter(), jsSet: stringset.NewStringFilter(), formSet: stringset.NewStringFilter(), awsSet: stringset.NewStringFilter(), filterLength_slice: filterLength_slice, } } func (crawler *Crawler) feedLinkfinder(jsFileUrl string, OutputType string, source string, ) { if !crawler.jsSet.Duplicate(jsFileUrl) { outputFormat := fmt.Sprintf("[%s] - %s", OutputType, jsFileUrl) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: source, OutputType: OutputType, Output: jsFileUrl, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data fmt.Println(outputFormat) } } else if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } // If JS file is minimal format. Try to find original format if strings.Contains(jsFileUrl, ".min.js") { originalJS := strings.ReplaceAll(jsFileUrl, ".min.js", ".js") _ = crawler.LinkFinderCollector.Visit(originalJS) } // Send Javascript to Link Finder Collector _ = crawler.LinkFinderCollector.Visit(jsFileUrl) } } func (crawler *Crawler) Start(linkfinder bool) { // Setup Link Finder if linkfinder { crawler.setupLinkFinder() } // Handle url crawler.C.OnHTML("[href]", func(e *colly.HTMLElement) { urlString := e.Request.AbsoluteURL(e.Attr("href")) urlString = FixUrl(crawler.site, urlString) if urlString == "" { return } if !crawler.urlSet.Duplicate(urlString) { outputFormat := fmt.Sprintf("[href] - %s", urlString) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "form", Output: urlString, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data fmt.Println(outputFormat) } } else if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } _ = e.Request.Visit(urlString) } }) // Handle form crawler.C.OnHTML("form[action]", func(e *colly.HTMLElement) { formUrl := e.Request.URL.String() if !crawler.formSet.Duplicate(formUrl) { outputFormat := fmt.Sprintf("[form] - %s", formUrl) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "form", Output: formUrl, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data fmt.Println(outputFormat) } } else if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } }) // Find Upload Form uploadFormSet := stringset.NewStringFilter() crawler.C.OnHTML(`input[type="file"]`, func(e *colly.HTMLElement) { uploadUrl := e.Request.URL.String() if !uploadFormSet.Duplicate(uploadUrl) { outputFormat := fmt.Sprintf("[upload-form] - %s", uploadUrl) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "upload-form", Output: uploadUrl, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data fmt.Println(outputFormat) } } else if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } }) // Handle js files crawler.C.OnHTML("[src]", func(e *colly.HTMLElement) { jsFileUrl := e.Request.AbsoluteURL(e.Attr("src")) jsFileUrl = FixUrl(crawler.site, jsFileUrl) if jsFileUrl == "" { return } fileExt := GetExtType(jsFileUrl) if fileExt == ".js" || fileExt == ".xml" || fileExt == ".json" { crawler.feedLinkfinder(jsFileUrl,"javascript","body") } }) crawler.C.OnResponse(func(response *colly.Response) { respStr := DecodeChars(string(response.Body)) if len(crawler.filterLength_slice) == 0 || !contains(crawler.filterLength_slice,len(respStr)) { // Verify which link is working u := response.Request.URL.String() outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u) if crawler.length { outputFormat = fmt.Sprintf("[url] - [code-%d] - [len_%d] - %s", response.StatusCode, len(respStr), u) } if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "url", StatusCode: response.StatusCode, Output: u, Length: strings.Count(respStr, "\n"), } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if crawler.Quiet { outputFormat = u } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } if InScope(response.Request.URL, crawler.C.URLFilters) { crawler.findSubdomains(respStr) crawler.findAWSS3(respStr) } if crawler.raw { outputFormat := fmt.Sprintf("[Raw] - \n%s\n", respStr) //PRINTCLEAN RAW for link visited only if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } } }) crawler.C.OnError(func(response *colly.Response, err error) { Logger.Debugf("Error request: %s - Status code: %v - Error: %s", response.Request.URL.String(), response.StatusCode, err) /* 1xx Informational 2xx Success 3xx Redirection 4xx Client Error 5xx Server Error */ if response.StatusCode == 404 || response.StatusCode == 429 || response.StatusCode < 100 || response.StatusCode >= 500 { return } u := response.Request.URL.String() outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "url", StatusCode: response.StatusCode, Output: u, Length: strings.Count(DecodeChars(string(response.Body)), "\n"), } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data fmt.Println(outputFormat) } } else if crawler.Quiet { fmt.Println(u) } else { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } }) err := crawler.C.Visit(crawler.site.String()) if err != nil { Logger.Errorf("Failed to start %s: %s", crawler.site.String(), err) } } // Find subdomains from response func (crawler *Crawler) findSubdomains(resp string) { subs := GetSubdomains(resp, crawler.domain) for _, sub := range subs { if !crawler.subSet.Duplicate(sub) { outputFormat := fmt.Sprintf("[subdomains] - %s", sub) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "subdomain", Output: sub, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } fmt.Println(outputFormat) } else if !crawler.Quiet { outputFormat = fmt.Sprintf("[subdomains] - http://%s", sub) fmt.Println(outputFormat) outputFormat = fmt.Sprintf("[subdomains] - https://%s", sub) fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } } } // Find AWS S3 from response func (crawler *Crawler) findAWSS3(resp string) { aws := GetAWSS3(resp) for _, e := range aws { if !crawler.awsSet.Duplicate(e) { outputFormat := fmt.Sprintf("[aws-s3] - %s", e) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "body", OutputType: "aws", Output: e, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } } } // Setup link finder func (crawler *Crawler) setupLinkFinder() { crawler.LinkFinderCollector.OnResponse(func(response *colly.Response) { if response.StatusCode == 404 || response.StatusCode == 429 || response.StatusCode < 100 { return } respStr := string(response.Body) if len(crawler.filterLength_slice) == 0 || !contains(crawler.filterLength_slice,len(respStr)) { // Verify which link is working u := response.Request.URL.String() outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u) if crawler.length { outputFormat = fmt.Sprintf("[url] - [code-%d] - [len_%d] - %s", response.StatusCode, len(respStr), u) } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } if InScope(response.Request.URL, crawler.C.URLFilters) { crawler.findSubdomains(respStr) crawler.findAWSS3(respStr) paths, err := LinkFinder(respStr) if err != nil { Logger.Error(err) return } currentPathURL, err := url.Parse(u) currentPathURLerr := false if err != nil { currentPathURLerr = true } for _, relPath := range paths { var outputFormat string // JS Regex Result if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: response.Request.URL.String(), OutputType: "linkfinder", Output: relPath, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if !crawler.Quiet { outputFormat = fmt.Sprintf("[linkfinder] - [from: %s] - %s", response.Request.URL.String(), relPath) } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } rebuildURL := "" if !currentPathURLerr { rebuildURL = FixUrl(currentPathURL, relPath) } else { rebuildURL = FixUrl(crawler.site, relPath) } if rebuildURL == "" { continue } // Try to request JS path // Try to generate URLs with main site fileExt := GetExtType(rebuildURL) if fileExt == ".js" || fileExt == ".xml" || fileExt == ".json" || fileExt == ".map" { crawler.feedLinkfinder(rebuildURL,"linkfinder","javascript") }else if !crawler.urlSet.Duplicate(rebuildURL){ if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: response.Request.URL.String(), OutputType: "linkfinder", Output: rebuildURL, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if !crawler.Quiet{ outputFormat = fmt.Sprintf("[linkfinder] - %s", rebuildURL) } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } _ = crawler.C.Visit(rebuildURL) } // Try to generate URLs with the site where Javascript file host in (must be in main or sub domain) urlWithJSHostIn := FixUrl(crawler.site, relPath) if urlWithJSHostIn != "" { fileExt := GetExtType(urlWithJSHostIn) if fileExt == ".js" || fileExt == ".xml" || fileExt == ".json" || fileExt == ".map" { crawler.feedLinkfinder(urlWithJSHostIn,"linkfinder","javascript") }else{ if crawler.urlSet.Duplicate(urlWithJSHostIn){ continue }else{ if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: response.Request.URL.String(), OutputType: "linkfinder", Output: urlWithJSHostIn, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if !crawler.Quiet{ outputFormat = fmt.Sprintf("[linkfinder] - %s", urlWithJSHostIn) } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } _ = crawler.C.Visit(urlWithJSHostIn) //not print care for lost link } } } } if crawler.raw{ outputFormat := fmt.Sprintf("[Raw] - \n%s\n", respStr) //PRINTCLEAN RAW for link visited only if !crawler.Quiet { fmt.Println(outputFormat) } if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } } } }) } gospider-1.1.6/core/grep.go000066400000000000000000000021531437726454100155650ustar00rootroot00000000000000package core import ( "regexp" "strings" ) const SUBRE = `(?i)(([a-zA-Z0-9]{1}|[_a-zA-Z0-9]{1}[_a-zA-Z0-9-]{0,61}[a-zA-Z0-9]{1})[.]{1})+` var AWSS3 = regexp.MustCompile(`(?i)[a-z0-9.-]+\.s3\.amazonaws\.com|[a-z0-9.-]+\.s3-[a-z0-9-]\.amazonaws\.com|[a-z0-9.-]+\.s3-website[.-](eu|ap|us|ca|sa|cn)|//s3\.amazonaws\.com/[a-z0-9._-]+|//s3-[a-z0-9-]+\.amazonaws\.com/[a-z0-9._-]+`) // SubdomainRegex returns a Regexp object initialized to match // subdomain names that end with the domain provided by the parameter. func subdomainRegex(domain string) *regexp.Regexp { // Change all the periods into literal periods for the regex d := strings.Replace(domain, ".", "[.]", -1) return regexp.MustCompile(SUBRE + d) } func GetSubdomains(source, domain string) []string { var subs []string re := subdomainRegex(domain) for _, match := range re.FindAllStringSubmatch(source, -1) { subs = append(subs, CleanSubdomain(match[0])) } return subs } func GetAWSS3(source string) []string { var aws []string for _, match := range AWSS3.FindAllStringSubmatch(source, -1) { aws = append(aws, DecodeChars(match[0])) } return aws } gospider-1.1.6/core/linkfinder.go000066400000000000000000000017441437726454100167620ustar00rootroot00000000000000package core import ( "regexp" "strings" ) var linkFinderRegex = regexp.MustCompile(`(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')`) func LinkFinder(source string) ([]string, error) { var links []string // source = strings.ToLower(source) if len(source) > 1000000 { source = strings.ReplaceAll(source, ";", ";\r\n") source = strings.ReplaceAll(source, ",", ",\r\n") } source = DecodeChars(source) match := linkFinderRegex.FindAllStringSubmatch(source, -1) for _, m := range match { matchGroup1 := FilterNewLines(m[1]) if matchGroup1 == "" { continue } links = append(links, matchGroup1) } links = Unique(links) return links, nil } gospider-1.1.6/core/linkfinder_test.go000066400000000000000000000003301437726454100200070ustar00rootroot00000000000000package core import "testing" func Test_ParseJSSource(t *testing.T) { source := ` "%2faaa.com" "https:\u002F\u002Fs.yimg.com\u002Fnq\u002Fstore-badges\u002F4\u002Fstore-badges\u002F"` t.Log(LinkFinder(source)) } gospider-1.1.6/core/logger.go000066400000000000000000000005661437726454100161150ustar00rootroot00000000000000package core import ( "os" "github.com/sirupsen/logrus" prefixed "github.com/x-cray/logrus-prefixed-formatter" ) var Logger *logrus.Logger func init() { logger := logrus.New() logger = &logrus.Logger{ Out: os.Stderr, Level: logrus.InfoLevel, Formatter: &prefixed.TextFormatter{ ForceColors: true, ForceFormatting: true, }, } Logger = logger } gospider-1.1.6/core/othersource.go000066400000000000000000000077431437726454100172040ustar00rootroot00000000000000package core import ( "bufio" "encoding/json" "fmt" "io/ioutil" "net/http" "os" "sync" ) func OtherSources(domain string, includeSubs bool) []string { noSubs := true if includeSubs { noSubs = false } var urls []string fetchFns := []fetchFn{ getWaybackURLs, getCommonCrawlURLs, getVirusTotalURLs, getOtxUrls, } var wg sync.WaitGroup for _, fn := range fetchFns { wUrlChan := make(chan wurl) wg.Add(1) fetch := fn go func() { defer wg.Done() resp, err := fetch(domain, noSubs) if err != nil { return } for _, r := range resp { wUrlChan <- r } }() go func() { wg.Wait() close(wUrlChan) }() for w := range wUrlChan { urls = append(urls, w.url) } } return Unique(urls) } type wurl struct { date string url string } type fetchFn func(string, bool) ([]wurl, error) func getWaybackURLs(domain string, noSubs bool) ([]wurl, error) { subsWildcard := "*." if noSubs { subsWildcard = "" } res, err := http.Get( fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=%s%s/*&output=json&collapse=urlkey", subsWildcard, domain), ) if err != nil { return []wurl{}, err } raw, err := ioutil.ReadAll(res.Body) res.Body.Close() if err != nil { return []wurl{}, err } var wrapper [][]string err = json.Unmarshal(raw, &wrapper) out := make([]wurl, 0, len(wrapper)) skip := true for _, urls := range wrapper { // The first item is always just the string "original", // so we should skip the first item if skip { skip = false continue } out = append(out, wurl{date: urls[1], url: urls[2]}) } return out, nil } func getCommonCrawlURLs(domain string, noSubs bool) ([]wurl, error) { subsWildcard := "*." if noSubs { subsWildcard = "" } res, err := http.Get( fmt.Sprintf("http://index.commoncrawl.org/CC-MAIN-2019-51-index?url=%s%s/*&output=json", subsWildcard, domain), ) if err != nil { return []wurl{}, err } defer res.Body.Close() sc := bufio.NewScanner(res.Body) out := make([]wurl, 0) for sc.Scan() { wrapper := struct { URL string `json:"url"` Timestamp string `json:"timestamp"` }{} err = json.Unmarshal([]byte(sc.Text()), &wrapper) if err != nil { continue } out = append(out, wurl{date: wrapper.Timestamp, url: wrapper.URL}) } return out, nil } func getVirusTotalURLs(domain string, noSubs bool) ([]wurl, error) { out := make([]wurl, 0) apiKey := os.Getenv("VT_API_KEY") if apiKey == "" { Logger.Warnf("You are not set VirusTotal API Key yet.") return out, nil } fetchURL := fmt.Sprintf( "https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", apiKey, domain, ) resp, err := http.Get(fetchURL) if err != nil { return out, err } defer resp.Body.Close() wrapper := struct { URLs []struct { URL string `json:"url"` } `json:"detected_urls"` }{} dec := json.NewDecoder(resp.Body) err = dec.Decode(&wrapper) for _, u := range wrapper.URLs { out = append(out, wurl{url: u.URL}) } return out, nil } func getOtxUrls(domain string, noSubs bool) ([]wurl, error) { var urls []wurl page := 0 for { r, err := http.Get(fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/hostname/%s/url_list?limit=50&page=%d", domain, page)) if err != nil { return []wurl{}, err } bytes, err := ioutil.ReadAll(r.Body) if err != nil { return []wurl{}, err } r.Body.Close() wrapper := struct { HasNext bool `json:"has_next"` ActualSize int `json:"actual_size"` URLList []struct { Domain string `json:"domain"` URL string `json:"url"` Hostname string `json:"hostname"` Httpcode int `json:"httpcode"` PageNum int `json:"page_num"` FullSize int `json:"full_size"` Paged bool `json:"paged"` } `json:"url_list"` }{} err = json.Unmarshal(bytes, &wrapper) if err != nil { return []wurl{}, err } for _, url := range wrapper.URLList { urls = append(urls, wurl{url: url.URL}) } if !wrapper.HasNext { break } page++ } return urls, nil } gospider-1.1.6/core/othersource_test.go000066400000000000000000000014201437726454100202250ustar00rootroot00000000000000package core import "testing" var domain = "yahoo.com" func TestOtherSources(t *testing.T) { urls := OtherSources(domain, false) t.Log(len(urls)) t.Log(urls) } func TestGetCommonCrawlURLs(t *testing.T) { urls, err := getCommonCrawlURLs(domain, false) if err != nil { t.Fatal(err) } t.Log(len(urls)) t.Log(urls) } func TestGetVirusTotalURLs(t *testing.T) { urls, err := getVirusTotalURLs(domain, false) if err != nil { t.Fatal(err) } t.Log(len(urls)) t.Log(urls) } func TestGetWaybackURLs(t *testing.T) { urls, err := getWaybackURLs(domain, false) if err != nil { t.Fatal(err) } t.Log(len(urls)) t.Log(urls) } func TestGetOtxUrls(t *testing.T) { urls, err := getOtxUrls(domain, false) if err != nil { t.Fatal(err) } t.Log(len(urls)) t.Log(urls) } gospider-1.1.6/core/output.go000066400000000000000000000010731437726454100161700ustar00rootroot00000000000000package core import ( "os" "path/filepath" "sync" ) type Output struct { mu sync.Mutex f *os.File } func NewOutput(folder, filename string) *Output { outFile := filepath.Join(folder, filename) f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) if err != nil { Logger.Errorf("Failed to open file to write Output: %s", err) os.Exit(1) } return &Output{ f: f, } } func (o *Output) WriteToFile(msg string) { o.mu.Lock() defer o.mu.Unlock() _, _ = o.f.WriteString(msg + "\n") } func (o *Output) Close() { o.f.Close() } gospider-1.1.6/core/robots.go000066400000000000000000000024761437726454100161500ustar00rootroot00000000000000package core import ( "fmt" "io/ioutil" "net/http" "net/url" "regexp" "strings" "sync" jsoniter "github.com/json-iterator/go" "github.com/gocolly/colly/v2" ) func ParseRobots(site *url.URL, crawler *Crawler, c *colly.Collector, wg *sync.WaitGroup) { defer wg.Done() robotsURL := site.String() + "/robots.txt" resp, err := http.Get(robotsURL) if err != nil { return } if resp.StatusCode == 200 { Logger.Infof("Found robots.txt: %s", robotsURL) body, err := ioutil.ReadAll(resp.Body) if err != nil { return } lines := strings.Split(string(body), "\n") var re = regexp.MustCompile(".*llow: ") for _, line := range lines { if strings.Contains(line, "llow: ") { url := re.ReplaceAllString(line, "") url = FixUrl(site, url) if url == "" { continue } outputFormat := fmt.Sprintf("[robots] - %s", url) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "robots", OutputType: "url", Output: url, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if crawler.Quiet { outputFormat = url } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } _ = c.Visit(url) } } } } gospider-1.1.6/core/sitemap.go000066400000000000000000000025721437726454100162770ustar00rootroot00000000000000package core import ( "fmt" "net/url" "sync" jsoniter "github.com/json-iterator/go" "github.com/gocolly/colly/v2" sitemap "github.com/oxffaa/gopher-parse-sitemap" ) func ParseSiteMap(site *url.URL, crawler *Crawler, c *colly.Collector, wg *sync.WaitGroup) { defer wg.Done() sitemapUrls := []string{"/sitemap.xml", "/sitemap_news.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemapindex.xml", "/sitemap-news.xml", "/post-sitemap.xml", "/page-sitemap.xml", "/portfolio-sitemap.xml", "/home_slider-sitemap.xml", "/category-sitemap.xml", "/author-sitemap.xml"} for _, path := range sitemapUrls { // Ignore error when that not valid sitemap.xml path Logger.Infof("Trying to find %s", site.String()+path) _ = sitemap.ParseFromSite(site.String()+path, func(entry sitemap.Entry) error { outputFormat := fmt.Sprintf("[sitemap] - %s", entry.GetLocation()) if crawler.JsonOutput { sout := SpiderOutput{ Input: crawler.Input, Source: "sitemap", OutputType: "url", Output: entry.GetLocation(), } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if crawler.Quiet { outputFormat = entry.GetLocation() } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } _ = c.Visit(entry.GetLocation()) return nil }) } } gospider-1.1.6/core/utils.go000066400000000000000000000114331437726454100157710ustar00rootroot00000000000000package core import ( "bufio" "fmt" "net/http" "net/url" "os" "path" "regexp" "strings" "github.com/mitchellh/go-homedir" "golang.org/x/net/publicsuffix" ) var nameStripRE = regexp.MustCompile("(?i)^((20)|(25)|(2b)|(2f)|(3d)|(3a)|(40))+") func GetRawCookie(cookies []*http.Cookie) string { var rawCookies []string for _, c := range cookies { e := fmt.Sprintf("%s=%s", c.Name, c.Value) rawCookies = append(rawCookies, e) } return strings.Join(rawCookies, "; ") } func GetDomain(site *url.URL) string { domain, err := publicsuffix.EffectiveTLDPlusOne(site.Hostname()) if err != nil { return "" } return domain } // func FixUrl(site *url.URL, nextLoc string) string { // var newUrl string // if strings.HasPrefix(nextLoc, "//") { // // //google.com/example.php // newUrl = site.Scheme + ":" + nextLoc // // } else if strings.HasPrefix(nextLoc, "http") { // // http://google.com || https://google.com // newUrl = nextLoc // // } else if !strings.HasPrefix(nextLoc, "//") { // // if strings.HasPrefix(nextLoc, "/") { // // // Ex: /?thread=10 // // newUrl = site.Scheme + "://" + site.Host + nextLoc // // // // } else { // // if strings.HasPrefix(nextLoc, ".") { // // if strings.HasPrefix(nextLoc, "..") { // // newUrl = site.Scheme + "://" + site.Host + nextLoc[2:] // // } else { // // newUrl = site.Scheme + "://" + site.Host + nextLoc[1:] // // } // // } else { // // // "console/test.php" // // newUrl = site.Scheme + "://" + site.Host + "/" + nextLoc // // } // // } // nextLocUrl, err := url.Parse(nextLoc) // if err != nil { // return "" // } // newUrl = site.ResolveReference(nextLocUrl).String() // } // return newUrl // } func FixUrl(mainSite *url.URL, nextLoc string) string { nextLocUrl, err := url.Parse(nextLoc) if err != nil { return "" } return mainSite.ResolveReference(nextLocUrl).String() } func Unique(intSlice []string) []string { keys := make(map[string]bool) var list []string for _, entry := range intSlice { if _, value := keys[entry]; !value { keys[entry] = true list = append(list, entry) } } return list } func LoadCookies(rawCookie string) []*http.Cookie { httpCookies := []*http.Cookie{} cookies := strings.Split(rawCookie, ";") for _, cookie := range cookies { cookieArgs := strings.SplitN(cookie, "=", 2) if len(cookieArgs) > 2 { continue } ck := &http.Cookie{Name: strings.TrimSpace(cookieArgs[0]), Value: strings.TrimSpace(cookieArgs[1])} httpCookies = append(httpCookies, ck) } return httpCookies } func GetExtType(rawUrl string) string { u, err := url.Parse(rawUrl) if err != nil { return "" } return path.Ext(u.Path) } func CleanSubdomain(s string) string { s = strings.TrimSpace(strings.ToLower(s)) s = strings.TrimPrefix(s, "*.") // s = strings.Trim("u00","") s = cleanName(s) return s } // Clean up the names scraped from the web. // Get from Amass func cleanName(name string) string { for { if i := nameStripRE.FindStringIndex(name); i != nil { name = name[i[1]:] } else { break } } name = strings.Trim(name, "-") // Remove dots at the beginning of names if len(name) > 1 && name[0] == '.' { name = name[1:] } return name } func FilterNewLines(s string) string { return regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(s), " ") } func DecodeChars(s string) string { source, err := url.QueryUnescape(s) if err == nil { s = source } // In case json encoded chars replacer := strings.NewReplacer( `\u002f`, "/", `\u0026`, "&", ) s = replacer.Replace(s) return s } func InScope(u *url.URL, regexps []*regexp.Regexp) bool { for _, r := range regexps { if r.MatchString(u.String()) { return true } } return false } // NormalizePath the path func NormalizePath(path string) string { if strings.HasPrefix(path, "~") { path, _ = homedir.Expand(path) } return path } // ReadingLines Reading file and return content as []string func ReadingLines(filename string) []string { var result []string if strings.HasPrefix(filename, "~") { filename, _ = homedir.Expand(filename) } file, err := os.Open(filename) if err != nil { return result } defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { val := strings.TrimSpace(scanner.Text()) if val == "" { continue } result = append(result, val) } if err := scanner.Err(); err != nil { return result } return result } func contains(i []int,j int) bool { for _, value := range i { if value == j { return true } } return false }gospider-1.1.6/core/utils_test.go000066400000000000000000000003221437726454100170230ustar00rootroot00000000000000package core import ( "testing" ) func TestGetExtType(t *testing.T) { url := "https://domain.com/data/avatars/m/123/12312312.jpg?1562846649" t.Log(GetExtType(url)) } func TestFixUrl(t *testing.T) { // } gospider-1.1.6/core/version.go000066400000000000000000000001541437726454100163140ustar00rootroot00000000000000package core const ( CLIName = "gospider" AUTHOR = "@thebl4ckturtle & @j3ssiejjj" VERSION = "v1.1.6" ) gospider-1.1.6/go.mod000066400000000000000000000011051437726454100144530ustar00rootroot00000000000000module github.com/jaeles-project/gospider go 1.16 require ( github.com/gocolly/colly/v2 v2.1.0 github.com/json-iterator/go v1.1.11 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.13.0 // indirect github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/net v0.0.0-20210614182718-04defd469f4e ) gospider-1.1.6/go.sum000066400000000000000000001230551437726454100145110ustar00rootroot00000000000000cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 h1:2vmb32OdDhjZf2ETGDlr9n8RYXx7c+jXPxMiPbwnA+8= github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4/go.mod h1:2JQx4jDHmWrbABvpOayg/+OTU6ehN0IyK2EHzceXpJo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= gospider-1.1.6/main.go000066400000000000000000000174771437726454100146430ustar00rootroot00000000000000package main import ( "bufio" "fmt" jsoniter "github.com/json-iterator/go" "io/ioutil" "net/url" "os" "strings" "sync" "github.com/jaeles-project/gospider/core" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) var commands = &cobra.Command{ Use: core.CLIName, Long: fmt.Sprintf("Fast web spider written in Go - %v by %v", core.VERSION, core.AUTHOR), Run: run, } func main() { commands.Flags().StringP("site", "s", "", "Site to crawl") commands.Flags().StringP("sites", "S", "", "Site list to crawl") commands.Flags().StringP("proxy", "p", "", "Proxy (Ex: http://127.0.0.1:8080)") commands.Flags().StringP("output", "o", "", "Output folder") commands.Flags().StringP("user-agent", "u", "web", "User Agent to use\n\tweb: random web user-agent\n\tmobi: random mobile user-agent\n\tor you can set your special user-agent") commands.Flags().StringP("cookie", "", "", "Cookie to use (testA=a; testB=b)") commands.Flags().StringArrayP("header", "H", []string{}, "Header to use (Use multiple flag to set multiple header)") commands.Flags().StringP("burp", "", "", "Load headers and cookie from burp raw http request") commands.Flags().StringP("blacklist", "", "", "Blacklist URL Regex") commands.Flags().StringP("whitelist", "", "", "Whitelist URL Regex") commands.Flags().StringP("whitelist-domain", "", "", "Whitelist Domain") commands.Flags().StringP("filter-length", "L", "", "Turn on length filter") commands.Flags().IntP("threads", "t", 1, "Number of threads (Run sites in parallel)") commands.Flags().IntP("concurrent", "c", 5, "The number of the maximum allowed concurrent requests of the matching domains") commands.Flags().IntP("depth", "d", 1, "MaxDepth limits the recursion depth of visited URLs. (Set it to 0 for infinite recursion)") commands.Flags().IntP("delay", "k", 0, "Delay is the duration to wait before creating a new request to the matching domains (second)") commands.Flags().IntP("random-delay", "K", 0, "RandomDelay is the extra randomized duration to wait added to Delay before creating a new request (second)") commands.Flags().IntP("timeout", "m", 10, "Request timeout (second)") commands.Flags().BoolP("base", "B", false, "Disable all and only use HTML content") commands.Flags().BoolP("js", "", true, "Enable linkfinder in javascript file") commands.Flags().BoolP("sitemap", "", false, "Try to crawl sitemap.xml") commands.Flags().BoolP("robots", "", true, "Try to crawl robots.txt") commands.Flags().BoolP("other-source", "a", false, "Find URLs from 3rd party (Archive.org, CommonCrawl.org, VirusTotal.com, AlienVault.com)") commands.Flags().BoolP("include-subs", "w", false, "Include subdomains crawled from 3rd party. Default is main domain") commands.Flags().BoolP("include-other-source", "r", false, "Also include other-source's urls (still crawl and request)") commands.Flags().BoolP("subs", "", false, "Include subdomains") commands.Flags().BoolP("debug", "", false, "Turn on debug mode") commands.Flags().BoolP("json", "", false, "Enable JSON output") commands.Flags().BoolP("verbose", "v", false, "Turn on verbose") commands.Flags().BoolP("quiet", "q", false, "Suppress all the output and only show URL") commands.Flags().BoolP("no-redirect", "", false, "Disable redirect") commands.Flags().BoolP("version", "", false, "Check version") commands.Flags().BoolP("length", "l", false, "Turn on length") commands.Flags().BoolP("raw", "R", false, "Enable raw output") commands.Flags().SortFlags = false if err := commands.Execute(); err != nil { core.Logger.Error(err) os.Exit(1) } } func run(cmd *cobra.Command, _ []string) { version, _ := cmd.Flags().GetBool("version") if version { fmt.Printf("Version: %s\n", core.VERSION) Examples() os.Exit(0) } isDebug, _ := cmd.Flags().GetBool("debug") if isDebug { core.Logger.SetLevel(logrus.DebugLevel) } else { core.Logger.SetLevel(logrus.InfoLevel) } verbose, _ := cmd.Flags().GetBool("verbose") if !verbose && !isDebug { core.Logger.SetOutput(ioutil.Discard) } // Create output folder when save file option selected outputFolder, _ := cmd.Flags().GetString("output") if outputFolder != "" { if _, err := os.Stat(outputFolder); os.IsNotExist(err) { _ = os.Mkdir(outputFolder, os.ModePerm) } } // Parse sites input var siteList []string siteInput, _ := cmd.Flags().GetString("site") if siteInput != "" { siteList = append(siteList, siteInput) } sitesListInput, _ := cmd.Flags().GetString("sites") if sitesListInput != "" { // parse from stdin sitesFile := core.ReadingLines(sitesListInput) if len(sitesFile) > 0 { siteList = append(siteList, sitesFile...) } } stat, _ := os.Stdin.Stat() // detect if anything came from std if (stat.Mode() & os.ModeCharDevice) == 0 { sc := bufio.NewScanner(os.Stdin) for sc.Scan() { target := strings.TrimSpace(sc.Text()) if err := sc.Err(); err == nil && target != "" { siteList = append(siteList, target) } } } // Check again to make sure at least one site in slice if len(siteList) == 0 { core.Logger.Info("No site in list. Please check your site input again") os.Exit(1) } threads, _ := cmd.Flags().GetInt("threads") sitemap, _ := cmd.Flags().GetBool("sitemap") linkfinder, _ := cmd.Flags().GetBool("js") robots, _ := cmd.Flags().GetBool("robots") otherSource, _ := cmd.Flags().GetBool("other-source") includeSubs, _ := cmd.Flags().GetBool("include-subs") includeOtherSourceResult, _ := cmd.Flags().GetBool("include-other-source") // disable all options above base, _ := cmd.Flags().GetBool("base") if base { linkfinder = false robots = false otherSource = false includeSubs = false includeOtherSourceResult = false } var wg sync.WaitGroup inputChan := make(chan string, threads) for i := 0; i < threads; i++ { wg.Add(1) go func() { defer wg.Done() for rawSite := range inputChan { site, err := url.Parse(rawSite) if err != nil { logrus.Errorf("Failed to parse %s: %s", rawSite, err) continue } var siteWg sync.WaitGroup crawler := core.NewCrawler(site, cmd) siteWg.Add(1) go func() { defer siteWg.Done() crawler.Start(linkfinder) }() // Brute force Sitemap path if sitemap { siteWg.Add(1) go core.ParseSiteMap(site, crawler, crawler.C, &siteWg) } // Find Robots.txt if robots { siteWg.Add(1) go core.ParseRobots(site, crawler, crawler.C, &siteWg) } if otherSource { siteWg.Add(1) go func() { defer siteWg.Done() urls := core.OtherSources(site.Hostname(), includeSubs) for _, url := range urls { url = strings.TrimSpace(url) if len(url) == 0 { continue } outputFormat := fmt.Sprintf("[other-sources] - %s", url) if includeOtherSourceResult { if crawler.JsonOutput { sout := core.SpiderOutput{ Input: crawler.Input, Source: "other-sources", OutputType: "url", Output: url, } if data, err := jsoniter.MarshalToString(sout); err == nil { outputFormat = data } } else if crawler.Quiet { outputFormat = url } fmt.Println(outputFormat) if crawler.Output != nil { crawler.Output.WriteToFile(outputFormat) } } _ = crawler.C.Visit(url) } }() } siteWg.Wait() crawler.C.Wait() crawler.LinkFinderCollector.Wait() } }() } for _, site := range siteList { inputChan <- site } close(inputChan) wg.Wait() core.Logger.Info("Done.") } func Examples() { h := "\n\nExamples Command:\n" h += `gospider -q -s "https://target.com/"` + "\n" h += `gospider -s "https://target.com/" -o output -c 10 -d 1` + "\n" h += `gospider -s "https://target.com/" -o output -c 10 -d 1 --other-source` + "\n" h += `echo 'http://target.com | gospider -o output -c 10 -d 1 --other-source` + "\n" fmt.Println(h) } gospider-1.1.6/stringset/000077500000000000000000000000001437726454100153725ustar00rootroot00000000000000gospider-1.1.6/stringset/filter.go000066400000000000000000000014011437726454100172020ustar00rootroot00000000000000// Copyright 2017 Jeff Foley. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. package stringset import "sync" // StringFilter implements an object that performs filtering of strings // to ensure that only unique items get through the filter. type StringFilter struct { filter Set lock sync.Mutex } // NewStringFilter returns an initialized StringFilter. func NewStringFilter() *StringFilter { s := New() return &StringFilter{filter: s} } // Duplicate checks if the name provided has been seen before by this filter. func (sf *StringFilter) Duplicate(s string) bool { sf.lock.Lock() defer sf.lock.Unlock() if sf.filter.Has(s) { return true } sf.filter.Insert(s) return false } gospider-1.1.6/stringset/set.go000066400000000000000000000053141437726454100165170ustar00rootroot00000000000000// Copyright 2017 Jeff Foley. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. package stringset import ( "fmt" "strings" ) type ( // Set implements set operations for string values. Set map[string]nothing nothing struct{} ) // Deduplicate utilizes the Set type to generate a unique list of strings from the input slice. func Deduplicate(input []string) []string { return New(input...).Slice() } // New returns a Set containing the values provided in the arguments. func New(initial ...string) Set { s := make(Set) for _, v := range initial { s.Insert(v) } return s } // Has returns true if the receiver Set already contains the element string argument. func (s Set) Has(element string) bool { _, exists := s[element] return exists } // Insert adds the element string argument to the receiver Set. func (s Set) Insert(element string) { s[strings.ToLower(element)] = nothing{} } // InsertMany adds all the elements strings into the receiver Set. func (s Set) InsertMany(elements ...string) { for _, i := range elements { s.Insert(i) } } // Remove will delete the element string from the receiver Set. func (s Set) Remove(element string) { e := strings.ToLower(element) if _, ok := s[e]; ok { delete(s, e) } } // Slice returns a string slice that contains all the elements in the Set. func (s Set) Slice() []string { var i uint64 k := make([]string, len(s)) for key := range s { k[i] = key i++ } return k } // Union adds all the elements from the other Set argument into the receiver Set. func (s Set) Union(other Set) { for k := range other { s.Insert(k) } } // Len returns the number of elements in the receiver Set. func (s Set) Len() int { return len(s) } // Subtract removes all elements in the other Set argument from the receiver Set. func (s Set) Subtract(other Set) { for item := range other { if s.Has(item) { s.Remove(item) } } } // Intersect causes the receiver Set to only contain elements also found in the // other Set argument. func (s Set) Intersect(other Set) { intersect := New() for item := range other { if s.Has(item) { intersect.Insert(item) } } var remove []string for item := range s { if !intersect.Has(item) { remove = append(remove, item) } } for _, r := range remove { s.Remove(r) } } // Set implements the flag.Value interface. func (s *Set) String() string { return strings.Join(s.Slice(), ",") } // Set implements the flag.Value interface. func (s *Set) Set(input string) error { if input == "" { return fmt.Errorf("String parsing failed") } items := strings.Split(input, ",") for _, item := range items { s.Insert(strings.TrimSpace(item)) } return nil }