diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..174478de --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Proposed changes + + + +### Proof + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/httpx/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 77e54f4c..78037c08 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -73,21 +73,26 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { - for range c { - gologger.Info().Msgf("CTRL+C pressed: Exiting\n") - httpxRunner.Close() - if options.ShouldSaveResume() { - gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFile) - err := httpxRunner.SaveResumeConfig() - if err != nil { - gologger.Error().Msgf("Couldn't create resume file: %s\n", err) - } - } - os.Exit(1) - } + // First Ctrl+C: stop dispatching, let in-flight requests finish + <-c + gologger.Info().Msgf("CTRL+C pressed: Exiting\n") + httpxRunner.Interrupt() + // Second Ctrl+C: force exit + <-c + gologger.Info().Msgf("Forcing exit\n") + os.Exit(1) }() httpxRunner.RunEnumeration() + + if httpxRunner.IsInterrupted() && options.ShouldSaveResume() { + gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFile) + err := httpxRunner.SaveResumeConfig() + if err != nil { + gologger.Error().Msgf("Couldn't create resume file: %s\n", err) + } + } + httpxRunner.Close() } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 039f4c4c..6175303c 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -77,6 +77,10 @@ func New(options *Options) (*HTTPX, error) { retryablehttpOptions.Timeout = httpx.Options.Timeout retryablehttpOptions.RetryMax = httpx.Options.RetryMax retryablehttpOptions.Trace = options.Trace + // Disable HTTP/2 fallback when HTTP/1.1 is explicitly requested + if httpx.Options.Protocol == "http11" { + retryablehttpOptions.DisableHTTP2Fallback = true + } handleHSTS := func(req *http.Request) { if req.Response.Header.Get("Strict-Transport-Security") == "" { return diff --git a/common/httpx/httpx_test.go b/common/httpx/httpx_test.go index 7da6ad12..c1294961 100644 --- a/common/httpx/httpx_test.go +++ b/common/httpx/httpx_test.go @@ -28,3 +28,46 @@ func TestDo(t *testing.T) { require.Greater(t, len(resp.Raw), 800) }) } + +// TestHTTP11ProtocolEnforcement verifies that when Protocol is set to "http11", +// the HTTP/2 fallback is disabled in retryablehttp-go client. +// This test addresses issue #2240 where the -pr http11 flag was being ignored. +func TestHTTP11ProtocolEnforcement(t *testing.T) { + t.Run("http11 protocol disables http2 fallback", func(t *testing.T) { + opts := DefaultOptions + opts.Protocol = HTTP11 + + ht, err := New(&opts) + require.Nil(t, err) + require.NotNil(t, ht) + + // The client should be configured with DisableHTTP2Fallback=true + // when Protocol is set to HTTP11 + // Note: We cannot directly access client options from here, but we can + // verify the setup doesn't error and the protocol is correctly set + require.Equal(t, HTTP11, ht.Options.Protocol) + }) + + t.Run("http2 protocol allows http2 fallback", func(t *testing.T) { + opts := DefaultOptions + opts.Protocol = HTTP2 + + ht, err := New(&opts) + require.Nil(t, err) + require.NotNil(t, ht) + + // When Protocol is HTTP2 or not HTTP11, the fallback should remain enabled + require.Equal(t, HTTP2, ht.Options.Protocol) + }) + + t.Run("default protocol allows http2 fallback", func(t *testing.T) { + opts := DefaultOptions + // Don't set Protocol, use default + + ht, err := New(&opts) + require.Nil(t, err) + require.NotNil(t, ht) + + // Default should not disable HTTP/2 fallback + }) +} diff --git a/go.mod b/go.mod index da7280df..c0349976 100644 --- a/go.mod +++ b/go.mod @@ -19,24 +19,24 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.19 + github.com/projectdiscovery/cdncheck v1.2.21 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/dsl v0.8.12 - github.com/projectdiscovery/fastdialer v0.5.3 + github.com/projectdiscovery/dsl v0.8.13 + github.com/projectdiscovery/fastdialer v0.5.4 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/gologger v1.1.67 - github.com/projectdiscovery/hmap v0.0.99 + github.com/projectdiscovery/gologger v1.1.68 + github.com/projectdiscovery/hmap v0.0.100 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.33 + github.com/projectdiscovery/networkpolicy v0.1.34 github.com/projectdiscovery/ratelimit v0.0.83 github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.3.4 + github.com/projectdiscovery/retryablehttp-go v1.3.6 github.com/projectdiscovery/tlsx v1.2.2 - github.com/projectdiscovery/useragent v0.0.106 + github.com/projectdiscovery/useragent v0.0.107 github.com/projectdiscovery/utils v0.9.0 - github.com/projectdiscovery/wappalyzergo v0.2.64 + github.com/projectdiscovery/wappalyzergo v0.2.66 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 @@ -135,7 +135,7 @@ require ( github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect - github.com/projectdiscovery/retryabledns v1.0.112 // indirect + github.com/projectdiscovery/retryabledns v1.0.113 // indirect github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -179,3 +179,5 @@ require ( golang.org/x/tools v0.40.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/projectdiscovery/retryablehttp-go => github.com/MrLawrenceKwan/retryablehttp-go v1.3.7-0.20260220033207-14afad3596fa diff --git a/go.sum b/go.sum index f5833c7d..db99af91 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/MrLawrenceKwan/retryablehttp-go v1.3.7-0.20260220033207-14afad3596fa h1:hZifTulU4ehXIo4HLC+1DQ3UyhAE1aIvAJMm+Hi7Lxw= +github.com/MrLawrenceKwan/retryablehttp-go v1.3.7-0.20260220033207-14afad3596fa/go.mod h1:tKVxmL4ixWy1MjYk5GJvFL0Cp10fnQgSp2F6bSBEypI= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= @@ -324,14 +326,14 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.19 h1:UU0ii1z8WZFsFODD89MYJ4i9h1EXhBJSZt/rzIH94JY= -github.com/projectdiscovery/cdncheck v1.2.19/go.mod h1:RRA4KOiUTBhkk2tImdoxqPpD0fB5C9rBP7W0r+ji9Cg= +github.com/projectdiscovery/cdncheck v1.2.21 h1:+y77BGCZoduX5bja2SGn4AdBXFwfOycaLnWWUIiZCBM= +github.com/projectdiscovery/cdncheck v1.2.21/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.8.12 h1:gQL8k5zPok+5JGc7poiXzHCElNY/WnaTKoRB2wI3CYA= -github.com/projectdiscovery/dsl v0.8.12/go.mod h1:pdMfUTNHMxlt6M94CSrCpZ1QObTP44rLqWifMMWW+IA= -github.com/projectdiscovery/fastdialer v0.5.3 h1:Io57Q37ouFzrPK53ZdzK6jsELgqjIMCWcoDs+lRDGMA= -github.com/projectdiscovery/fastdialer v0.5.3/go.mod h1:euoxS1E93LDnl0OnNN0UALedAFF+EehBxyU3z+79l0g= +github.com/projectdiscovery/dsl v0.8.13 h1:HjjHta7c02saH2tUGs8CN5vDeE2MyWvCV32koT8ZCWs= +github.com/projectdiscovery/dsl v0.8.13/go.mod h1:hgFaXhz/JuO+HqIXqBqYIR3ntPnqTo38MJJAzb5tIbg= +github.com/projectdiscovery/fastdialer v0.5.4 h1:+0oesDDqZcIPE5bNDmm/Xm9Xm3yjnhl4xwP+h5D1TE4= +github.com/projectdiscovery/fastdialer v0.5.4/go.mod h1:KCzt6WnSAj9umiUBRCaC0EJSEyeshxDoowfwjxodmQw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -340,36 +342,34 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.67 h1:GZU3AjYiJvcwJT5TlfIv+152/TVmaz62Zyn3/wWXlig= -github.com/projectdiscovery/gologger v1.1.67/go.mod h1:35oeQP6wvj58S+o+Km6boED/t786FXQkI0exhFHJbNE= +github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU= +github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.99 h1:XPfLnD3CUrMqVCIdpK9ozD7Xmp3simx3T+2j4WWhHnU= -github.com/projectdiscovery/hmap v0.0.99/go.mod h1:koyUJi83K5G3w35ZLFXOYZIyYJsO+6hQrgDDN1RBrVE= +github.com/projectdiscovery/hmap v0.0.100 h1:DBZ3Req9lWf4P1YC9PRa4eiMvLY0Uxud43NRBcocPfs= +github.com/projectdiscovery/hmap v0.0.100/go.mod h1:2O06pR8pHOP9wSmxAoxuM45U7E+UqOqOdlSIeddM0bA= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.33 h1:bVgp+XpLEsQ7ZEJt3UaUqIwhI01MMdt7F2dfIKFQg/w= -github.com/projectdiscovery/networkpolicy v0.1.33/go.mod h1:YAPddAXUc/lhoU85AFdvgOQKx8Qh8r0vzSjexRWk6Yk= +github.com/projectdiscovery/networkpolicy v0.1.34 h1:TRwNbgMwdx3NC190TKSLwtTvr0JAIZAlnWkOhW0yBME= +github.com/projectdiscovery/networkpolicy v0.1.34/go.mod h1:GJ20E7fJoA2vk8ZBSa1Cvc5WyP8RxglF5bZmYgK8jag= github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54= github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= -github.com/projectdiscovery/retryabledns v1.0.112 h1:4iCiuo6jMnw/pdOZRzBQrbUOUu5tOeuvGupxVV8RDLw= -github.com/projectdiscovery/retryabledns v1.0.112/go.mod h1:xsJTKbo+KGqd7+88z1naEUFJybLH2yjB/zUyOweA7k0= -github.com/projectdiscovery/retryablehttp-go v1.3.4 h1:QgGah0Py9MvvjrzGxGthgzhh5jzG18uRfqkJNUXKDIo= -github.com/projectdiscovery/retryablehttp-go v1.3.4/go.mod h1:4disixzHEhNd2pEO2kpg0kqyy9Tx1WMZtgd7hI/XiuM= +github.com/projectdiscovery/retryabledns v1.0.113 h1:s+DAzdJ8XhLxRgt5636H0HG9OqHsGRjX9wTrLSTMqlQ= +github.com/projectdiscovery/retryabledns v1.0.113/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= -github.com/projectdiscovery/useragent v0.0.106 h1:9fS08MRUUJvfBskTxcXY9TA4X1TwpH6iJ3P3YNaXNlo= -github.com/projectdiscovery/useragent v0.0.106/go.mod h1:9oVMjgd7CchIsyeweyigIPtW83gpiGf2NtR6UM5XK+o= +github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZEQRGAoYGP6F5M= +github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= -github.com/projectdiscovery/wappalyzergo v0.2.64 h1:Y55sb5qUdFvMtR81m1hr54PdGh/hZ4XtuGPdCFAirEk= -github.com/projectdiscovery/wappalyzergo v0.2.64/go.mod h1:8FtSVcmPRZU0g1euBpdSYEBHIvB7Zz9MOb754ZqZmfU= +github.com/projectdiscovery/wappalyzergo v0.2.66 h1:DEF7wthjvBo6oYKxfKL6vPNaqsKYUmiWODt7Mybcins= +github.com/projectdiscovery/wappalyzergo v0.2.66/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= diff --git a/runner/runner.go b/runner/runner.go index 14b91625..8647c23f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -97,12 +97,32 @@ type Runner struct { simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server authProvider authprovider.AuthProvider + interruptCh chan struct{} } func (r *Runner) HTTPX() *httpx.HTTPX { return r.hp } +// Interrupt signals the runner to stop dispatching new items. +func (r *Runner) Interrupt() { + select { + case <-r.interruptCh: + default: + close(r.interruptCh) + } +} + +// IsInterrupted returns true if the runner was interrupted. +func (r *Runner) IsInterrupted() bool { + select { + case <-r.interruptCh: + return true + default: + return false + } +} + // picked based on try-fail but it seems to close to one it's used https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html#c1992 var hammingDistanceThreshold int = 22 @@ -121,7 +141,8 @@ type pHashUrl struct { // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - options: options, + options: options, + interruptCh: make(chan struct{}), } var err error if options.Wappalyzer != nil { @@ -664,6 +685,16 @@ func (r *Runner) streamInput() (chan string, error) { go func() { defer close(out) + // trySend sends item to out, returning false if interrupted + trySend := func(item string) bool { + select { + case <-r.interruptCh: + return false + case out <- item: + return true + } + } + if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling if format := r.getInputFormat(); format != nil { @@ -676,9 +707,9 @@ func (r *Runner) streamInput() (chan string, error) { if err := format.Parse(finput, func(item string) bool { item = strings.TrimSpace(item) if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + return trySend(item) } - return true + return !r.IsInterrupted() }); err != nil { gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) return @@ -690,7 +721,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -706,7 +739,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -718,7 +753,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -1402,6 +1439,12 @@ func (r *Runner) RunEnumeration() { wg, _ := syncutil.New(syncutil.WithSize(r.options.Threads)) processItem := func(k string) error { + select { + case <-r.interruptCh: + return nil + default: + } + if r.options.resumeCfg != nil { r.options.resumeCfg.current = k r.options.resumeCfg.currentIndex++ @@ -1447,6 +1490,9 @@ func (r *Runner) RunEnumeration() { if r.options.Stream { for item := range streamChan { + if r.IsInterrupted() { + break + } _ = processItem(item) } } else { diff --git a/runner/runner_test.go b/runner/runner_test.go index 10b8320b..832c4e36 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -15,6 +15,74 @@ import ( "github.com/stretchr/testify/require" ) +func TestRunner_resumeAfterInterrupt(t *testing.T) { + domains := []string{"a.com", "b.com", "c.com", "d.com", "e.com", "f.com", "g.com", "h.com", "i.com", "j.com"} + interruptAfter := 4 + + // --- Full scan (reference): process all domains without interrupt --- + rFull, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rFull.options.resumeCfg = &ResumeCfg{} + var fullOutput []string + for _, d := range domains { + rFull.options.resumeCfg.current = d + rFull.options.resumeCfg.currentIndex++ + fullOutput = append(fullOutput, d) + } + + // --- Interrupted scan: process items, interrupt after interruptAfter --- + rInt, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rInt.options.resumeCfg = &ResumeCfg{} + var interruptedOutput []string + for _, d := range domains { + // same check as processItem: bail out if interrupted + select { + case <-rInt.interruptCh: + continue + default: + } + + rInt.options.resumeCfg.current = d + rInt.options.resumeCfg.currentIndex++ + interruptedOutput = append(interruptedOutput, d) + + if len(interruptedOutput) == interruptAfter { + rInt.Interrupt() + } + } + + // simulate SaveResumeConfig: save the index after interrupt + savedIndex := rInt.options.resumeCfg.currentIndex + + // the saved index must equal exactly the number of items that were processed + require.Equal(t, interruptAfter, savedIndex, "resume index should equal number of completed items") + // every domain before the index must be in the interrupted output + require.Equal(t, domains[:interruptAfter], interruptedOutput, "interrupted output should contain exactly the first N domains") + + // --- Resumed scan: load saved index, skip already-processed items --- + rRes, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rRes.options.resumeCfg = &ResumeCfg{Index: savedIndex} + var resumedOutput []string + for _, d := range domains { + // same resume-skip logic as processItem + rRes.options.resumeCfg.current = d + rRes.options.resumeCfg.currentIndex++ + if rRes.options.resumeCfg.currentIndex <= rRes.options.resumeCfg.Index { + continue + } + resumedOutput = append(resumedOutput, d) + } + + // every domain after the index must be in the resumed output + require.Equal(t, domains[interruptAfter:], resumedOutput, "resumed output should contain exactly the remaining domains") + + // union of interrupted + resumed must equal the full scan + combined := append(interruptedOutput, resumedOutput...) + require.Equal(t, fullOutput, combined, "interrupted + resumed should equal full scan") +} + func TestRunner_domain_targets(t *testing.T) { options := &Options{} r, err := New(options)