pax_global_header00006660000000000000000000000064142504167330014517gustar00rootroot0000000000000052 comment=7e8abdb4096ab224f21616478b699b4dfda9d409 httprobe-0.2/000077500000000000000000000000001425041673300132075ustar00rootroot00000000000000httprobe-0.2/.gitignore000066400000000000000000000000511425041673300151730ustar00rootroot00000000000000domains hosts httprobe *.tgz *.zip *.exe httprobe-0.2/Dockerfile000066400000000000000000000011611425041673300152000ustar00rootroot00000000000000FROM golang:1.11.4-alpine3.7 AS build-env RUN apk add --no-cache --upgrade git openssh-client ca-certificates RUN go get -u github.com/golang/dep/cmd/dep WORKDIR /go/src/app COPY . /go/src/app RUN go build -o httprobe main.go FROM alpine:3.9 RUN apk add shadow bash && \ useradd --create-home --shell /sbin/nologin httprobe && \ mkdir /httprobe && \ chown httprobe:httprobe /httprobe COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=build-env /go/src/app/httprobe /httprobe/httprobe USER httprobe WORKDIR /httprobe ENTRYPOINT ["/httprobe/httprobe"] httprobe-0.2/LICENSE000066400000000000000000000020541425041673300142150ustar00rootroot00000000000000MIT License Copyright (c) 2019 Tom Hudson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. httprobe-0.2/README.md000066400000000000000000000033111425041673300144640ustar00rootroot00000000000000# httprobe Take a list of domains and probe for working http and https servers. ## Install ``` ▶ go get -u github.com/tomnomnom/httprobe ``` ## Basic Usage httprobe accepts line-delimited domains on `stdin`: ``` ▶ cat recon/example/domains.txt example.com example.edu example.net ▶ cat recon/example/domains.txt | httprobe http://example.com http://example.net http://example.edu https://example.com https://example.edu https://example.net ``` ## Extra Probes By default httprobe checks for HTTP on port 80 and HTTPS on port 443. You can add additional probes with the `-p` flag by specifying a protocol and port pair: ``` ▶ cat domains.txt | httprobe -p http:81 -p https:8443 ``` ## Concurrency You can set the concurrency level with the `-c` flag: ``` ▶ cat domains.txt | httprobe -c 50 ``` ## Timeout You can change the timeout by using the `-t` flag and specifying a timeout in milliseconds: ``` ▶ cat domains.txt | httprobe -t 20000 ``` ## Skipping Default Probes If you don't want to probe for HTTP on port 80 or HTTPS on port 443, you can use the `-s` flag. You'll need to specify the probes you do want using the `-p` flag: ``` ▶ cat domains.txt | httprobe -s -p https:8443 ``` ## Prefer HTTPS Sometimes you don't care about checking HTTP if HTTPS is working. You can do that with the `--prefer-https` flag: ``` ▶ cat domains.txt | httprobe --prefer-https ``` ## Docker Build the docker container: ``` ▶ docker build -t httprobe . ``` Run the container, passing the contents of a file into stdin of the process inside the container. `-i` is required to correctly map `stdin` into the container and to the `httprobe` binary. ``` ▶ cat domains.txt | docker run -i httprobe ``` httprobe-0.2/go.mod000066400000000000000000000000561425041673300143160ustar00rootroot00000000000000module github.com/tomnomnom/httprobe go 1.18 httprobe-0.2/main.go000066400000000000000000000127461425041673300144740ustar00rootroot00000000000000package main import ( "bufio" "crypto/tls" "flag" "fmt" "io" "io/ioutil" "net" "net/http" "os" "strings" "sync" "time" ) type probeArgs []string func (p *probeArgs) Set(val string) error { *p = append(*p, val) return nil } func (p probeArgs) String() string { return strings.Join(p, ",") } func main() { // concurrency flag var concurrency int flag.IntVar(&concurrency, "c", 20, "set the concurrency level (split equally between HTTPS and HTTP requests)") // probe flags var probes probeArgs flag.Var(&probes, "p", "add additional probe (proto:port)") // skip default probes flag var skipDefault bool flag.BoolVar(&skipDefault, "s", false, "skip the default probes (http:80 and https:443)") // timeout flag var to int flag.IntVar(&to, "t", 10000, "timeout (milliseconds)") // prefer https var preferHTTPS bool flag.BoolVar(&preferHTTPS, "prefer-https", false, "only try plain HTTP if HTTPS fails") // HTTP method to use var method string flag.StringVar(&method, "method", "GET", "HTTP method to use") flag.Parse() // make an actual time.Duration out of the timeout timeout := time.Duration(to * 1000000) var tr = &http.Transport{ MaxIdleConns: 30, IdleConnTimeout: time.Second, DisableKeepAlives: true, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DialContext: (&net.Dialer{ Timeout: timeout, KeepAlive: time.Second, }).DialContext, } re := func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } client := &http.Client{ Transport: tr, CheckRedirect: re, Timeout: timeout, } // domain/port pairs are initially sent on the httpsURLs channel. // If they are listening and the --prefer-https flag is set then // no HTTP check is performed; otherwise they're put onto the httpURLs // channel for an HTTP check. httpsURLs := make(chan string) httpURLs := make(chan string) output := make(chan string) // HTTPS workers var httpsWG sync.WaitGroup for i := 0; i < concurrency/2; i++ { httpsWG.Add(1) go func() { for url := range httpsURLs { // always try HTTPS first withProto := "https://" + url if isListening(client, withProto, method) { output <- withProto // skip trying HTTP if --prefer-https is set if preferHTTPS { continue } } httpURLs <- url } httpsWG.Done() }() } // HTTP workers var httpWG sync.WaitGroup for i := 0; i < concurrency/2; i++ { httpWG.Add(1) go func() { for url := range httpURLs { withProto := "http://" + url if isListening(client, withProto, method) { output <- withProto continue } } httpWG.Done() }() } // Close the httpURLs channel when the HTTPS workers are done go func() { httpsWG.Wait() close(httpURLs) }() // Output worker var outputWG sync.WaitGroup outputWG.Add(1) go func() { for o := range output { fmt.Println(o) } outputWG.Done() }() // Close the output channel when the HTTP workers are done go func() { httpWG.Wait() close(output) }() // accept domains on stdin sc := bufio.NewScanner(os.Stdin) for sc.Scan() { domain := strings.ToLower(sc.Text()) // submit standard port checks if !skipDefault { httpsURLs <- domain } // Adding port templates xlarge := []string{"81", "300", "591", "593", "832", "981", "1010", "1311", "2082", "2087", "2095", "2096", "2480", "3000", "3128", "3333", "4243", "4567", "4711", "4712", "4993", "5000", "5104", "5108", "5800", "6543", "7000", "7396", "7474", "8000", "8001", "8008", "8014", "8042", "8069", "8080", "8081", "8088", "8090", "8091", "8118", "8123", "8172", "8222", "8243", "8280", "8281", "8333", "8443", "8500", "8834", "8880", "8888", "8983", "9000", "9043", "9060", "9080", "9090", "9091", "9200", "9443", "9800", "9981", "12443", "16080", "18091", "18092", "20720", "28017"} large := []string{"81", "591", "2082", "2087", "2095", "2096", "3000", "8000", "8001", "8008", "8080", "8083", "8443", "8834", "8888"} // submit any additional proto:port probes for _, p := range probes { switch p { case "xlarge": for _, port := range xlarge { httpsURLs <- fmt.Sprintf("%s:%s", domain, port) } case "large": for _, port := range large { httpsURLs <- fmt.Sprintf("%s:%s", domain, port) } default: pair := strings.SplitN(p, ":", 2) if len(pair) != 2 { continue } // This is a little bit funny as "https" will imply an // http check as well unless the --prefer-https flag is // set. On balance I don't think that's *such* a bad thing // but it is maybe a little unexpected. if strings.ToLower(pair[0]) == "https" { httpsURLs <- fmt.Sprintf("%s:%s", domain, pair[1]) } else { httpURLs <- fmt.Sprintf("%s:%s", domain, pair[1]) } } } } // once we've sent all the URLs off we can close the // input/httpsURLs channel. The workers will finish what they're // doing and then call 'Done' on the WaitGroup close(httpsURLs) // check there were no errors reading stdin (unlikely) if err := sc.Err(); err != nil { fmt.Fprintf(os.Stderr, "failed to read input: %s\n", err) } // Wait until the output waitgroup is done outputWG.Wait() } func isListening(client *http.Client, url, method string) bool { req, err := http.NewRequest(method, url, nil) if err != nil { return false } req.Header.Add("Connection", "close") req.Close = true resp, err := client.Do(req) if resp != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } if err != nil { return false } return true } httprobe-0.2/script/000077500000000000000000000000001425041673300145135ustar00rootroot00000000000000httprobe-0.2/script/release000077500000000000000000000023731425041673300160660ustar00rootroot00000000000000#!/bin/bash PROJDIR=$(cd `dirname $0`/.. && pwd) VERSION="${1}" TAG="v${VERSION}" USER="tomnomnom" REPO="httprobe" BINARY="${REPO}" if [[ -z "${VERSION}" ]]; then echo "Usage: ${0} " exit 1 fi if [[ -z "${GITHUB_TOKEN}" ]]; then echo "You forgot to set your GITHUB_TOKEN" exit 2 fi cd ${PROJDIR} # Run the tests go test if [ $? -ne 0 ]; then echo "Tests failed. Aborting." exit 3 fi FILELIST="" for ARCH in "amd64" "386"; do for OS in "darwin" "linux" "windows" "freebsd"; do if [[ "${OS}" == "darwin" && "${ARCH}" == "386" ]]; then continue fi BINFILE="${BINARY}" if [[ "${OS}" == "windows" ]]; then BINFILE="${BINFILE}.exe" fi rm -f ${BINFILE} GOOS=${OS} GOARCH=${ARCH} go build github.com/${USER}/${REPO} if [[ "${OS}" == "windows" ]]; then ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.zip" zip ${ARCHIVE} ${BINFILE} rm ${BINFILE} else ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.tgz" tar --create --gzip --file=${ARCHIVE} ${BINFILE} fi FILELIST="${FILELIST} ${PROJDIR}/${ARCHIVE}" done done gh release create ${TAG} ${FILELIST} rm ${FILELIST}