feat(detection): stealth-browser detection for botasaurus-class scrapers (0.4.0)

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webdecoy/client",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Web Decoy browser widget - signal collection, proof-of-work, and captcha UI",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

packages/client/src/collectors/environment.ts

@@ -29,6 +29,9 @@ export class EnvironmentalCollector {
       cssMediaQueries: this._getCSSMediaQueries(),
       permissionsInfo: this._getPermissionsInfo(),
       fontsInfo: this._getFontsInfo(),
+
+      // Native-function integrity (stealth self-hiding leaves patched toStrings)
+      lieDetection: this._getLieDetection(),
     };
   }
 
@@ -124,20 +127,14 @@ export class EnvironmentalCollector {
       );
       if (pwKeys.length > 0) signals.push('playwright_globals');
 
-      // Check if navigator.webdriver was deleted or reconfigured
+      // navigator.webdriver *deleted* from the prototype is a genuine automation
+      // tell. NOTE: a merely *configurable* descriptor is normal in real Chrome,
+      // and chrome.runtime is absent on ordinary pages in real Chrome — both
+      // fired on a real browser in live testing, so neither is a signal here.
       const proto = Object.getPrototypeOf(navigator);
       const desc = Object.getOwnPropertyDescriptor(proto, 'webdriver');
       if (!desc) {
-        // Property was deleted from prototype — browsers always have it
         signals.push('webdriver_deleted');
-      } else if (desc.configurable !== false) {
-        signals.push('webdriver_configurable');
-      }
-
-      // Check for missing chrome.runtime in Chrome UA
-      const isChrome = /Chrome\//.test(navigator.userAgent) && !/Edg\//.test(navigator.userAgent);
-      if (isChrome && w.chrome && !w.chrome.runtime) {
-        signals.push('chrome_runtime_missing');
       }
 
       return { detected: signals.length > 0, signals };
@@ -277,6 +274,57 @@ export class EnvironmentalCollector {
     }
   }
 
+  /**
+   * Native-function integrity check. Stealth automation hides itself by
+   * overriding native functions; a patched native's `toString()` no longer
+   * reports `[native code]`. A genuine browser never patches its own natives,
+   * so any hit here is deliberate evasion (scored in the `stealth` category).
+   */
+  _getLieDetection(): Record<string, unknown> {
+    try {
+      const w = window as any;
+      const patched: string[] = [];
+
+      const isPatched = (fn: unknown): boolean => {
+        try {
+          return typeof fn === 'function' && (fn as { toString(): string }).toString().indexOf('[native code]') === -1;
+        } catch {
+          return false;
+        }
+      };
+      const check = (fn: unknown, name: string): void => {
+        if (isPatched(fn)) patched.push(name);
+      };
+
+      // If toString itself is patched, every other check is unreliable — flag it.
+      check(Function.prototype.toString, 'Function.prototype.toString');
+      check(navigator.permissions && navigator.permissions.query, 'navigator.permissions.query');
+      check(w.Notification && w.Notification.requestPermission, 'Notification.requestPermission');
+      check(w.HTMLCanvasElement && w.HTMLCanvasElement.prototype.toDataURL, 'HTMLCanvasElement.toDataURL');
+      check(
+        w.WebGLRenderingContext && w.WebGLRenderingContext.prototype.getParameter,
+        'WebGLRenderingContext.getParameter',
+      );
+      check(navigator.mediaDevices && navigator.mediaDevices.enumerateDevices, 'mediaDevices.enumerateDevices');
+
+      // The navigator.webdriver getter is the most-patched native.
+      try {
+        const desc =
+          Object.getOwnPropertyDescriptor(Object.getPrototypeOf(navigator), 'webdriver') ||
+          Object.getOwnPropertyDescriptor(navigator, 'webdriver');
+        if (desc && typeof desc.get === 'function' && desc.get.toString().indexOf('[native code]') === -1) {
+          patched.push('navigator.webdriver getter');
+        }
+      } catch {
+        // ignore
+      }
+
+      return { supported: true, patched, patchedCount: patched.length };
+    } catch (e) {
+      return { supported: false };
+    }
+  }
+
   _getHeadlessIndicators(): Record<string, unknown> {
     const nav = navigator as any;
     return {

packages/express/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webdecoy/express",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Web Decoy middleware for Express.js",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -40,7 +40,7 @@
     "url": "https://github.com/WebDecoy/node-sdk/issues"
   },
   "dependencies": {
-    "@webdecoy/node": "^0.3.0"
+    "@webdecoy/node": "^0.4.0"
   },
   "peerDependencies": {
     "express": "^4.18.0 || ^5.0.0"

packages/fastify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webdecoy/fastify",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Web Decoy plugin for Fastify",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -40,7 +40,7 @@
     "url": "https://github.com/WebDecoy/node-sdk/issues"
   },
   "dependencies": {
-    "@webdecoy/node": "^0.3.0",
+    "@webdecoy/node": "^0.4.0",
     "fastify-plugin": "^4.5.1"
   },
   "peerDependencies": {

packages/nextjs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webdecoy/nextjs",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Web Decoy middleware for Next.js",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -41,7 +41,7 @@
     "url": "https://github.com/WebDecoy/node-sdk/issues"
   },
   "dependencies": {
-    "@webdecoy/node": "^0.3.0"
+    "@webdecoy/node": "^0.4.0"
   },
   "peerDependencies": {
     "next": ">=13.0.0"

packages/webdecoy/harness/botasaurus_crawl_test.py

@@ -0,0 +1,48 @@
+"""
+F4 tripwire validation — the test botasaurus (or any fingerprint-stealth tool)
+CANNOT pass.
+
+A scraper that *crawls* — fetches a page, extracts links, follows them — will
+request the hidden honeytoken decoy link and trip the tripwire. Unlike F1
+fingerprinting (which botasaurus defeats), this catches *intent* (going where a
+human can't), which stealth cannot spoof away.
+
+    <venv>/bin/python harness/botasaurus_crawl_test.py
+
+Run request mode (the exact mode that evaded F1). A real human/browser loads the
+page but never follows the invisible link, so it is never flagged.
+"""
+
+import re
+
+HARNESS = "http://localhost:8787"
+
+
+def crawl() -> None:
+    from botasaurus.request import request, Request
+
+    @request
+    def run(req: "Request", data):  # noqa: ANN001
+        page = req.get(f"{HARNESS}/").text
+        hrefs = re.findall(r'href="([^"]+)"', page)
+        print(f"scraper extracted {len(hrefs)} link(s): {hrefs}")
+
+        tripped = []
+        for h in hrefs:
+            url = h if h.startswith("http") else HARNESS + h
+            r = req.get(url)
+            flag = ""
+            if r.status_code == 403:
+                flag = "   <-- TRIPWIRE: BLOCKED"
+                tripped.append(h)
+            print(f"  GET {h}  ->  {r.status_code}{flag}")
+
+        print(f"\nRESULT: {'CAUGHT (tripwire blocked the scraper)' if tripped else 'evaded'}"
+              f" — tripwires hit: {tripped}")
+        return tripped
+
+    run()
+
+
+if __name__ == "__main__":
+    crawl()

packages/webdecoy/harness/botasaurus_test.py

@@ -0,0 +1,65 @@
+"""
+Live botasaurus test against the WebDecoy stealth harness.
+
+Prereqs:
+    pip install botasaurus
+    npx tsx harness/server.ts        # in another terminal (serves :8787)
+
+Then:
+    python3 harness/botasaurus_test.py
+
+It drives botasaurus in BOTH modes against the harness and prints the real
+DetectionEngine verdict the server returns:
+  - browser mode  -> loads the page, the page collects real signals & scores
+  - request mode  -> GET /probe (no JS; scored on UA + headers only)
+
+API note: botasaurus's surface shifts across versions; if `driver.select`/`.get`
+differ in your version, adjust the two marked lines. The verdict also prints on
+the SERVER stdout regardless, so you can read results there too.
+"""
+
+HARNESS = "http://localhost:8787"
+
+
+def run_browser_mode() -> None:
+    try:
+        from botasaurus.browser import browser, Driver
+    except Exception as e:  # noqa: BLE001
+        print(f"[browser] botasaurus import failed: {e}")
+        return
+
+    @browser(headless=True, block_images=True)
+    def scrape(driver: "Driver", data):  # noqa: ANN001
+        driver.get(f"{HARNESS}/")             # <- adjust if your botasaurus differs
+        driver.sleep(3)                        # let the page collect + POST /score
+        try:
+            text = driver.select("#result").text   # <- adjust selector API if needed
+        except Exception:                      # noqa: BLE001
+            text = driver.run_js("return document.getElementById('result').textContent")
+        print("\n===== BOTASAURUS BROWSER MODE — server verdict =====")
+        print(text)
+        return text
+
+    scrape()
+
+
+def run_request_mode() -> None:
+    try:
+        from botasaurus.request import request, Request
+    except Exception as e:  # noqa: BLE001
+        print(f"[request] botasaurus import failed: {e}")
+        return
+
+    @request
+    def fetch(req: "Request", data):  # noqa: ANN001
+        r = req.get(f"{HARNESS}/probe")
+        print("\n===== BOTASAURUS REQUEST MODE (@request) — server verdict =====")
+        print(r.text)
+        return r.text
+
+    fetch()
+
+
+if __name__ == "__main__":
+    run_browser_mode()
+    run_request_mode()