Headless mode

import nodriver as uc

# Start in headless mode
browser = await uc.start(headless=True)
tab = await browser.get('https://example.com')

print(f'Page loaded: {tab.url}')
browser.stop()

from nodriver import Config

config = Config(
    headless=True,
    sandbox=True  # Important for server environments
)

browser = await uc.start(config=config)

class Config:
    def __init__(
        self,
        user_data_dir: Optional[PathLike] = AUTO,
        headless: Optional[bool] = False,
        browser_executable_path: Optional[PathLike] = AUTO,
        browser_args: Optional[List[str]] = AUTO,
        sandbox: Optional[bool] = True,
        lang: Optional[str] = "en-US",
        host: str = AUTO,
        port: int = AUTO,
        expert: bool = AUTO,
        **kwargs: dict,
    ):
        """
        Creates a config object.
        Can be called without any arguments to generate a best-practice config.
        
        :param headless: set to True for headless mode
        :param sandbox: disables sandbox
        """

browser = await uc.start(headless=True)
tab = browser.main_tab

# User agent is automatically cleaned
ua = await tab.evaluate('navigator.userAgent', return_by_value=True)
print(ua)  # Won't contain "Headless"

async def _prepare_headless(self):
    if getattr(self, "_prep_headless_done", None):
        return
    resp = await self._send_oneshot(
        cdp.runtime.evaluate(
            expression="navigator.userAgent",
        )
    )
    if not resp:
        return
    response, error = resp
    if response and response.value:
        ua = response.value
        await self._send_oneshot(
            cdp.network.set_user_agent_override(
                user_agent=ua.replace("Headless", ""),
            )
        )
    setattr(self, "_prep_headless_done", True)

browser = await uc.start(headless=True)
tab = await browser.get('https://example.com')

# Screenshots work
await tab.save_screenshot('headless.jpg')

# Element interaction works
button = await tab.select('button')
await button.click()

# JavaScript execution works
result = await tab.evaluate('document.title', return_by_value=True)
print(result)

# Network monitoring works
import nodriver.cdp as cdp
tab.add_handler(cdp.network.ResponseReceived, lambda e: print(e.response.url))

if self.headless:
    args.append("--headless=new")

# With sandbox (recommended)
browser = await uc.start(
    headless=True,
    sandbox=True
)

# Without sandbox (only if needed)
browser = await uc.start(
    headless=True,
    sandbox=False  # Disables Chrome sandbox
)

self.sandbox = sandbox
# When using posix-ish operating system and running as root
# you must use no_sandbox = True, which in case is corrected here
if is_posix and is_root() and sandbox:
    logger.info("detected root usage, auto disabling sandbox mode")
    self.sandbox = False

FROM python:3.11-slim

# Install Chrome dependencies
RUN apt-get update && apt-get install -y \
    wget \
    gnupg \
    ca-certificates \
    fonts-liberation \
    libasound2 \
    libatk-bridge2.0-0 \
    libatk1.0-0 \
    libcups2 \
    libdbus-1-3 \
    libdrm2 \
    libgbm1 \
    libgtk-3-0 \
    libnspr4 \
    libnss3 \
    libxcomposite1 \
    libxdamage1 \
    libxrandr2 \
    xdg-utils \
    --no-install-recommends

# Install Chrome
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list \
    && apt-get update \
    && apt-get install -y google-chrome-stable \
    && rm -rf /var/lib/apt/lists/*

# Install Python packages
RUN pip install nodriver

# Copy your script
COPY script.py /app/script.py
WORKDIR /app

# Run script
CMD ["python", "script.py"]

# script.py
import nodriver as uc

async def main():
    # Headless mode for Docker
    browser = await uc.start(
        headless=True,
        sandbox=False  # Usually needed in Docker
    )
    
    tab = await browser.get('https://example.com')
    await tab.wait(2)
    
    title = await tab.evaluate('document.title', return_by_value=True)
    print(f'Page title: {title}')
    
    browser.stop()

if __name__ == '__main__':
    uc.loop().run_until_complete(main())

# Ubuntu/Debian
sudo apt-get update
sudo apt-get install -y \
    chromium-browser \
    chromium-chromedriver \
    xvfb

# Install nodriver
pip install nodriver

import nodriver as uc

async def main():
    browser = await uc.start(
        headless=True,
        browser_executable_path='/usr/bin/chromium-browser'
    )
    
    # Your automation here
    tab = await browser.get('https://example.com')
    # ...
    
    browser.stop()

browser = await uc.start(
    headless=True,
    host='0.0.0.0',  # Allow remote connections
    port=9222
)

tab = browser.main_tab
print(f'Debug URL: {tab.inspector_url}')

# Now connect to this URL from another browser
await tab.get('https://example.com')
await tab.wait(60)  # Keep alive for debugging

async def open_external_inspector(self):
    """
    Opens the system's browser containing the devtools inspector page
    for this tab. Could be handy, especially to debug in headless mode.
    """
    import webbrowser
    
    webbrowser.open(self.inspector_url)

browser = await uc.start(headless=True)
tab = browser.main_tab

# Open inspector in your default browser
await tab.open_external_inspector()

# Continue automation
await tab.get('https://example.com')
await tab.wait(60)

browser = await uc.start(headless=True)
tab = await browser.get('https://example.com')

# Take screenshot at each step
await tab.save_screenshot('step1.jpg')

button = await tab.select('button')
await button.click()

await tab.save_screenshot('step2.jpg')

# Compare screenshots to debug issues

import time

start = time.time()
browser = await uc.start(headless=True)
tab = await browser.get('https://example.com')
await tab.wait(2)
browser.stop()
print(f'Time taken: {time.time() - start:.2f}s')

# Optimize for low memory
config = Config(
    headless=True,
    browser_args=[
        '--disable-gpu',
        '--disable-dev-shm-usage',
        '--disable-software-rasterizer',
        '--no-sandbox'
    ]
)

browser = await uc.start(config=config)

config = Config(
    headless=True,
    browser_args=[
        '--disable-gpu',
        '--disable-dev-shm-usage',
        '--single-process',  # Use single process (saves memory)
        '--no-zygote'  # Don't use zygote process
    ]
)

import nodriver as uc
from pathlib import Path
import asyncio
import json

class HeadlessScraperService:
    def __init__(self):
        self.browser = None
        self.results_dir = Path('results')
        self.results_dir.mkdir(exist_ok=True)
    
    async def start(self):
        """Initialize headless browser"""
        self.browser = await uc.start(
            headless=True,
            sandbox=True
        )
        print('Scraper service started')
    
    async def scrape_page(self, url: str) -> dict:
        """Scrape a single page"""
        tab = await self.browser.get(url)
        await tab.wait(2)
        
        # Extract data
        title = await tab.evaluate('document.title', return_by_value=True)
        html = await tab.get_content()
        
        # Take screenshot
        screenshot_path = self.results_dir / f'{url.split("/")[-1]}.jpg'
        await tab.save_screenshot(str(screenshot_path))
        
        # Close tab to free memory
        await tab.close()
        
        return {
            'url': url,
            'title': title,
            'screenshot': str(screenshot_path),
            'html_length': len(html)
        }
    
    async def scrape_multiple(self, urls: list) -> list:
        """Scrape multiple pages"""
        results = []
        
        for url in urls:
            try:
                result = await self.scrape_page(url)
                results.append(result)
                print(f'Scraped: {url}')
            except Exception as e:
                print(f'Error scraping {url}: {e}')
                results.append({'url': url, 'error': str(e)})
            
            # Small delay between requests
            await asyncio.sleep(1)
        
        return results
    
    async def stop(self):
        """Cleanup"""
        if self.browser:
            self.browser.stop()
        print('Scraper service stopped')

async def main():
    service = HeadlessScraperService()
    await service.start()
    
    urls = [
        'https://example.com',
        'https://example.org',
        'https://example.net'
    ]
    
    results = await service.scrape_multiple(urls)
    
    # Save results
    with open('results/scrape_results.json', 'w') as f:
        json.dump(results, f, indent=2)
    
    print(f'Scraped {len(results)} pages')
    
    await service.stop()

if __name__ == '__main__':
    uc.loop().run_until_complete(main())

name: Scraper Tests

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest
    
    steps:
    - uses: actions/checkout@v2
    
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.11'
    
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y chromium-browser
        pip install nodriver pytest
    
    - name: Run tests
      run: |
        pytest tests/ -v

# tests/test_scraper.py
import pytest
import nodriver as uc

@pytest.mark.asyncio
async def test_headless_scraping():
    browser = await uc.start(headless=True, sandbox=False)
    tab = await browser.get('https://example.com')
    
    title = await tab.evaluate('document.title', return_by_value=True)
    assert 'Example' in title
    
    browser.stop()