Security

// 1. Fast fail for IP literals
if (net.isIP(urlObj.hostname)) {
    if (IPGuard.isInternal(urlObj.hostname)) {
        return this.errorResult('blocked_internal_ip', currentUrl, redirectChain)
    }
}

// 2. DNS lookup validation (prevents TOCTOU attacks)
const secureDispatcher = IPGuard.getSecureDispatcher()

static secureLookup(
    hostname: string,
    options: dns.LookupOneOptions,
    callback: (err, address, family) => void
): void {
    dns.lookup(hostname, options, (err, address, family) => {
        if (IPGuard.isInternal(address)) {
            const blockedError = new Error(`Blocked internal IP: ${address}`)
            blockedError.code = 'EBLOCKED'
            return callback(blockedError, address, family)
        }
        callback(null, address, family)
    })
}

# Direct IP literals
crawlith crawl http://127.0.0.1:8080
crawlith crawl http://192.168.1.1
crawlith crawl http://[::1]:3000

# DNS rebinding attacks
crawlith crawl http://evil.com  # Resolves to 10.0.0.1

status: 'blocked_internal_ip'

crawlith crawl https://example.com  # Public IP

crawlith crawl https://example.com --allow example.com,cdn.example.com

isAllowed(hostname: string): boolean {
    const normalized = this.normalize(hostname)

    // 1. Deny list match -> Reject
    if (this.denied.has(normalized)) {
        return false
    }

    // 2. Allow list not empty AND no match -> Reject
    if (this.allowed.size > 0 && !this.allowed.has(normalized)) {
        return false
    }

    return true
}

crawlith crawl https://example.com --deny api.example.com,admin.example.com

private normalize(hostname: string): string {
    let h = hostname.toLowerCase().trim()
    if (h.endsWith('.')) h = h.slice(0, -1)
    return new URL(`http://${h}`).hostname
}

crawlith crawl https://example.com --include-subdomains

isAllowed(hostname: string): boolean {
    // Exact match always allowed
    if (target === this.rootHost) return true

    if (!this.includeSubdomains) return false

    // Must end with .rootHost
    if (!target.endsWith(`.${this.rootHost}`)) return false
    return true
}

crawlith crawl https://example.com

crawlith crawl https://example.com \
  --include-subdomains \
  --allow example.com,cdn.example.net

export type EligibilityResult = 
    | 'allowed' 
    | 'blocked_by_domain_filter' 
    | 'blocked_subdomain'

isUrlEligible(url: string): EligibilityResult {
    // 1. Domain filter check
    if (!this.domainFilter.isAllowed(hostname)) {
        return 'blocked_by_domain_filter'
    }

    // 2. Explicit whitelist bypass
    if (this.explicitAllowed.has(hostname)) {
        return 'allowed'
    }

    // 3. Subdomain policy check
    if (!this.subdomainPolicy.isAllowed(hostname)) {
        return 'blocked_subdomain'
    }

    return 'allowed'
}

if (status >= 300 && status < 400 && status !== 304) {
    const location = getHeader('location')
    if (location) {
        let targetUrl = new URL(location, currentUrl).toString()

        // Validate redirect target through scope manager
        const redirectError = redirectController.nextHop(targetUrl)
        if (redirectError) {
            return this.errorResult(redirectError, currentUrl, redirectChain)
        }

        // Continue with redirect...
    }
}

crawlith crawl https://example.com --max-redirects 5

nextHop(url: string): 'redirect_limit_exceeded' | 'redirect_loop' | null {
    if (this.history.has(normalized)) return 'redirect_loop'
    if (this.currentHops >= this.maxHops) return 'redirect_limit_exceeded'
    this.history.add(normalized)
    this.currentHops++
    return null
}

crawlith crawl https://example.com --max-bytes 2000000

static async streamToString(
    stream: Readable,
    maxBytes: number,
    onOversized?: (bytes: number) => void
): Promise<string> {
    let accumulated = 0
    if (accumulated > maxBytes) {
        stream.destroy()
        throw new Error('Oversized response')
    }
}

status: 'oversized'
bytesReceived: 2048576

crawlith crawl https://example.com --proxy http://proxy.example.com:8080

# With authentication
crawlith crawl https://example.com --proxy http://user:[email protected]:8080

constructor(proxyUrl?: string) {
    if (proxyUrl) {
        new URL(proxyUrl) // Validate URL
        this.agent = new ProxyAgent(proxyUrl)
    }
}

crawlith crawl https://example.com \
  --allow example.com \
  --log-level debug