feat(runner): probe a target through the ip of a JSON input line

README.md

@@ -365,6 +365,12 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas
 
 - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
 - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
+- A target can be probed through a given IP with a JSON input line, like `curl --resolve`. The hostname is still used for the Host header and SNI, and the connection goes to the IP, which `host_ip` reports. The same host can be listed with several IPs:
+  ```
+  {"target": "https://app.example.com", "ip": "192.0.2.10"}
+  {"target": "https://app.example.com", "ip": "192.0.2.11"}
+  ```
+  Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with a warning with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP; a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well.
 - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
 - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)

common/httpx/tls.go

@@ -63,7 +63,8 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response {
 	}
 	// canonical net concatenation
 	host = net.JoinHostPort(hostname, fmt.Sprint(port))
-	ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout)
+	// keeps the ip the request was pinned to, without its cancellation
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Request.Context()), h.Options.Timeout)
 	defer cancel()
 	tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host)
 	if err != nil {

go.mod

@@ -169,3 +169,5 @@ require (
 	golang.org/x/time v0.15.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
+
+replace github.com/projectdiscovery/fastdialer => github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c

go.sum

@@ -256,8 +256,8 @@ github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9Mch
 github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
 github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
 github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
-github.com/projectdiscovery/fastdialer v0.5.24 h1:wA0E6UTKtfR+Uqrw5vSR5XUmYHK3IkVoGLrt8yojjbc=
-github.com/projectdiscovery/fastdialer v0.5.24/go.mod h1:GybngvpSevg/48Kk/NgPw2HQOB95FngbOS93f1pTJ84=
+github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c h1:nWPzF0xe4LzMfSHllHbVzBxdAhwtusRisN091e049KI=
+github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw=
 github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
 github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
 github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=

runner/runner.go

@@ -546,6 +546,10 @@ func (r *Runner) prepareInput() {
 	// check if input target host(s) have been provided
 	if len(r.options.InputTargetHost) > 0 {
 		for _, target := range r.options.InputTargetHost {
+			target, err := r.normalizeInput(target)
+			if err != nil {
+				gologger.Fatal().Msgf("Could not read input target: %s\n", err)
+			}
 			expandedTarget, err := r.countTargetFromRawTarget(target)
 			if err == nil && expandedTarget > 0 {
 				numHosts += expandedTarget
@@ -722,6 +726,20 @@ func (r *Runner) streamInput() (chan string, error) {
 				return true
 			}
 		}
+		// sendLine normalizes, deduplicates and sends an input line, returning
+		// false if interrupted. The scan has already started, so an invalid
+		// line is skipped with a warning.
+		sendLine := func(item string) bool {
+			item, err := r.normalizeInput(item)
+			if err != nil {
+				gologger.Error().Msgf("Skipping input: %s\n", err)
+				return true
+			}
+			if r.options.SkipDedupe || r.testAndSet(item) {
+				return trySend(item)
+			}
+			return true
+		}
 
 		if fileutil.FileExists(r.options.InputFile) {
 			// check if input mode is specified for special format handling
@@ -744,14 +762,9 @@ func (r *Runner) streamInput() (chan string, error) {
 				}
 			} else {
 				for item, err := range fileutil.Lines(r.options.InputFile) {
-					if err != nil {
+					if err != nil || !sendLine(item) {
 						return
 					}
-					if r.options.SkipDedupe || r.testAndSet(item) {
-						if !trySend(item) {
-							return
-						}
-					}
 				}
 			}
 		} else if r.options.InputFile != "" {
@@ -761,27 +774,17 @@ func (r *Runner) streamInput() (chan string, error) {
 			}
 			for _, file := range files {
 				for item, err := range fileutil.Lines(file) {
-					if err != nil {
+					if err != nil || !sendLine(item) {
 						return
 					}
-					if r.options.SkipDedupe || r.testAndSet(item) {
-						if !trySend(item) {
-							return
-						}
-					}
 				}
 			}
 		}
 		if fileutil.HasStdin() {
 			for item, err := range fileutil.LinesReader(os.Stdin) {
-				if err != nil {
+				if err != nil || !sendLine(item) {
 					return
 				}
-				if r.options.SkipDedupe || r.testAndSet(item) {
-					if !trySend(item) {
-						return
-					}
-				}
 			}
 		}
 	}()
@@ -791,7 +794,11 @@ func (r *Runner) streamInput() (chan string, error) {
 func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	scanner := bufio.NewScanner(finput)
 	for scanner.Scan() {
-		target := strings.TrimSpace(scanner.Text())
+		target, err := r.normalizeInput(scanner.Text())
+		if err != nil {
+			_ = finput.Close()
+			return 0, err
+		}
 		// Used just to get the exact number of targets
 		expandedTarget, err := r.countTargetFromRawTarget(target)
 		if err == nil && expandedTarget > 0 {
@@ -1524,7 +1531,7 @@ func (r *Runner) RunEnumeration() {
 
 		protocol := r.options.protocol
 		// attempt to parse url as is
-		if u, err := r.parseURL(k); err == nil {
+		if u, err := r.parseURL(targetOf(k)); err == nil {
 			if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS {
 				protocol = u.Scheme
 			}
@@ -1776,6 +1783,13 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
 		target = strings.TrimSpace(target)
 
 		switch {
+		case isTargetLine(target):
+			line, err := parseTargetLine(target)
+			if err != nil {
+				gologger.Warning().Msgf("%s\n", err)
+				return
+			}
+			results <- httpx.Target{Host: line.Target, CustomIP: line.IP}
 		case stringsutil.HasPrefixAny(target, "*", "."):
 			// A valid target does not contain:
 			// trim * and/or . (prefix) from the target to return the domain instead of wildcard
@@ -1852,6 +1866,10 @@ retry:
 
 	// check if we have to skip the host:port as a result of a previous failure
 	hostPort := net.JoinHostPort(URL.Host, URL.Port())
+	if target.CustomIP != "" {
+		// the ips of a host fail independently
+		hostPort += "@" + target.CustomIP
+	}
 	if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
 		numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
 		if err == nil && numberOfErrors >= r.options.HostMaxErrors {
@@ -1880,16 +1898,11 @@ retry:
 		req *retryablehttp.Request
 		ctx context.Context
 	)
+	ctx = context.Background()
 	if target.CustomIP != "" {
-		var requestIP string
-		if iputil.IsIPv6(target.CustomIP) {
-			requestIP = fmt.Sprintf("[%s]", target.CustomIP)
-		} else {
-			requestIP = target.CustomIP
-		}
-		ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP)
-	} else {
-		ctx = context.Background()
+		// only dials to the target's host go to the ip, not those to a
+		// redirect target on another host
+		ctx = fastdialer.WithResolveOverride(ctx, URL.Hostname(), target.CustomIP)
 	}
 	req, err = hp.NewRequestWithContext(ctx, method, URL.String())
 	if err != nil {
@@ -2761,6 +2774,11 @@ func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (
 		return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target host is not allowed by network policy")}
 	}
 
+	if target.CustomIP != "" && !r.hp.NetworkPolicy.Validate(target.CustomIP) {
+		gologger.Debug().Msgf("Skipping target due to network policy: %s (%s)\n", URL.Hostname(), target.CustomIP)
+		return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target ip is not allowed by network policy")}
+	}
+
 	return false, Result{}
 }
 
@@ -2811,8 +2829,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 		}
 	}
 
-	// Clone original request (reuse headers/cookies)
-	clone := req.Clone(context.Background())
+	// Clone original request (reuse headers/cookies, and the pinned ip)
+	clone := req.Clone(req.Context())
 
 	var (
 		faviconMMH3 string

runner/targetline.go

@@ -0,0 +1,153 @@
+package runner
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/netip"
+	"strings"
+
+	"github.com/projectdiscovery/mapcidr/asn"
+	iputil "github.com/projectdiscovery/utils/ip"
+	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+// targetLine is an input line that pins a target to an ip, like
+// curl --resolve:
+//
+//	{"target": "https://app.example", "ip": "192.0.2.10"}
+//
+// The target keeps its hostname for the Host header and SNI, while the
+// connection goes to the ip.
+type targetLine struct {
+	Target string `json:"target"`
+	IP     string `json:"ip,omitempty"`
+}
+
+// isTargetLine reports whether an input line is a target line.
+func isTargetLine(line string) bool {
+	return strings.HasPrefix(strings.TrimSpace(line), "{")
+}
+
+// parseTargetLine decodes a target line. Unknown keys are rejected so that a
+// typo cannot silently probe the host through DNS.
+func parseTargetLine(line string) (targetLine, error) {
+	decoder := json.NewDecoder(strings.NewReader(line))
+	decoder.DisallowUnknownFields()
+	var parsed targetLine
+	if err := decoder.Decode(&parsed); err != nil {
+		return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err)
+	}
+	if _, err := decoder.Token(); !errors.Is(err, io.EOF) {
+		return targetLine{}, fmt.Errorf("invalid target line %q: unexpected data after the object", line)
+	}
+
+	parsed.Target = strings.TrimSpace(parsed.Target)
+	if parsed.Target == "" {
+		return targetLine{}, fmt.Errorf("target line %q has no target", line)
+	}
+	if parsed.IP == "" {
+		return parsed, nil
+	}
+	ip, err := pinnedIP(parsed.Target, parsed.IP)
+	if err != nil {
+		return targetLine{}, fmt.Errorf("target %q: %w", parsed.Target, err)
+	}
+	parsed.IP = ip
+	return parsed, nil
+}
+
+// pinnedIP validates the ip a target is pinned to and returns it in canonical
+// form. Only a target with a hostname can be pinned.
+func pinnedIP(target, value string) (string, error) {
+	addr, err := netip.ParseAddr(strings.TrimSpace(value))
+	if err != nil || addr.Zone() != "" || addr.IsUnspecified() {
+		return "", fmt.Errorf("invalid ip %q", value)
+	}
+	if iputil.IsCIDR(target) || asn.IsASN(target) {
+		return "", errors.New("a CIDR or ASN target cannot have an ip")
+	}
+	if stringsutil.HasPrefixAny(target, "*", ".") || (!stringsutil.HasPrefixAny(target, "http://", "https://") && strings.Contains(target, ",")) {
+		return "", errors.New("a wildcard or vhost,ip target cannot have an ip")
+	}
+	parsed, err := urlutil.Parse(target)
+	if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) {
+		return "", errors.New("a target with an ip must have a hostname")
+	}
+	return addr.Unmap().String(), nil
+}
+
+// key returns the line as it is stored and deduplicated: the target alone
+// when it is not pinned, and a canonical target line otherwise.
+func (t targetLine) key() string {
+	if t.IP == "" {
+		return t.Target
+	}
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetEscapeHTML(false)
+	_ = encoder.Encode(t)
+	return strings.TrimSpace(buf.String())
+}
+
+// targetOf returns the target of an input line: the line itself, or the
+// target of a target line.
+func targetOf(item string) string {
+	if !isTargetLine(item) {
+		return item
+	}
+	if line, err := parseTargetLine(item); err == nil {
+		return line.Target
+	}
+	return item
+}
+
+// normalizeInput validates an input line and returns the form it is stored
+// and deduplicated in.
+func (r *Runner) normalizeInput(item string) (string, error) {
+	item = strings.TrimSpace(item)
+	if !isTargetLine(item) {
+		return item, nil
+	}
+	line, err := parseTargetLine(item)
+	if err != nil {
+		return "", err
+	}
+	if line.IP != "" {
+		if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 {
+			return "", fmt.Errorf("target %q: a target with an ip cannot be probed with %s", line.Target, strings.Join(flags, ", "))
+		}
+	}
+	return line.key(), nil
+}
+
+// pinnedTargetConflicts returns the flags whose probes do not connect through
+// the ip a target is pinned to.
+func (options *Options) pinnedTargetConflicts() []string {
+	var flags []string
+	if options.Unsafe {
+		flags = append(flags, "-unsafe")
+	}
+	if options.Pipeline {
+		flags = append(flags, "-pipeline")
+	}
+	if options.HTTP2Probe {
+		flags = append(flags, "-http2")
+	}
+	if options.Jarm {
+		flags = append(flags, "-jarm")
+	}
+	if options.Screenshot {
+		flags = append(flags, "-screenshot")
+	}
+	if options.VHostInput {
+		flags = append(flags, "-vhost-input")
+	}
+	if options.Proxy != "" || options.HTTPProxy != "" || options.SocksProxy != "" {
+		flags = append(flags, "-proxy")
+	}
+	return flags
+}