count_tokens / compute_tokens

def count_tokens(
    self,
    *,
    model: str,
    contents: Union[ContentListUnion, ContentListUnionDict],
    config: Optional[CountTokensConfig] = None,
) -> CountTokensResponse

async def count_tokens(
    self,
    *,
    model: str,
    contents: Union[ContentListUnion, ContentListUnionDict],
    config: Optional[CountTokensConfig] = None,
) -> CountTokensResponse

from google import genai

client = genai.Client(api_key='your-api-key')

response = client.models.count_tokens(
    model='gemini-2.0-flash',
    contents='What is your name?',
)

print(f"Total tokens: {response.total_tokens}")
# Output: Total tokens: 5

from google.genai import types

conversation = [
    types.Content(role='user', parts=[types.Part.from_text('Hello!')]),
    types.Content(role='model', parts=[types.Part.from_text('Hi there! How can I help you?')]),
    types.Content(role='user', parts=[types.Part.from_text('Tell me about Python')]),
]

response = client.models.count_tokens(
    model='gemini-2.0-flash',
    contents=conversation,
)

print(f"Conversation tokens: {response.total_tokens}")

response = client.models.count_tokens(
    model='gemini-2.0-flash',
    contents=[
        types.Part.from_text('Describe this image'),
        types.Part.from_uri(
            file_uri='gs://generativeai-downloads/images/scones.jpg',
            mime_type='image/jpeg'
        ),
    ],
)

print(f"Multimodal tokens: {response.total_tokens}")
# Includes both text and image tokens

client = genai.Client(vertexai=True, project='my-project', location='us-central1')

response = client.models.count_tokens(
    model='gemini-2.0-flash',
    contents='What is the weather?',
    config=types.CountTokensConfig(
        system_instruction='You are a helpful weather assistant.',
        tools=[weather_tool],
    ),
)

print(f"Total tokens (including system instruction and tools): {response.total_tokens}")

def safe_generate(client, model, contents, max_tokens=30000):
    """Generate content only if within token limit."""
    token_response = client.models.count_tokens(
        model=model,
        contents=contents,
    )
    
    if token_response.total_tokens > max_tokens:
        raise ValueError(f"Content exceeds {max_tokens} tokens: {token_response.total_tokens}")
    
    return client.models.generate_content(
        model=model,
        contents=contents,
    )

# Use it
try:
    response = safe_generate(client, 'gemini-2.0-flash', 'What is AI?')
    print(response.text)
except ValueError as e:
    print(f"Error: {e}")

def compute_tokens(
    self,
    *,
    model: str,
    contents: Union[ContentListUnion, ContentListUnionDict],
    config: Optional[ComputeTokensConfig] = None,
) -> ComputeTokensResponse

async def compute_tokens(
    self,
    *,
    model: str,
    contents: Union[ContentListUnion, ContentListUnionDict],
    config: Optional[ComputeTokensConfig] = None,
) -> ComputeTokensResponse

from google import genai

client = genai.Client(vertexai=True, project='my-project', location='us-central1')

response = client.models.compute_tokens(
    model='gemini-2.0-flash',
    contents='What is your name?',
)

print(f"Tokens: {response.tokens_info[0].tokens}")
print(f"Token IDs: {response.tokens_info[0].token_ids}")
# Output:
# Tokens: [b'What', b' is', b' your', b' name', b'?']
# Token IDs: ['1841', '374', '574', '836', '30']

text = "The quick brown fox jumps over the lazy dog."

response = client.models.compute_tokens(
    model='gemini-2.0-flash',
    contents=text,
)

tokens_info = response.tokens_info[0]
print(f"Original text: {text}")
print(f"Number of tokens: {len(tokens_info.tokens)}")
print("\nTokenization:")
for token, token_id in zip(tokens_info.tokens, tokens_info.token_ids):
    print(f"  '{token.decode('utf-8')}' -> ID: {token_id}")

from google.genai import types

contents = [
    types.Content(role='user', parts=[types.Part.from_text('Hello!')]),
    types.Content(role='model', parts=[types.Part.from_text('Hi there!')]),
]

response = client.models.compute_tokens(
    model='gemini-2.0-flash',
    contents=contents,
)

for i, tokens_info in enumerate(response.tokens_info):
    print(f"\nContent {i+1} ({tokens_info.role}):")
    print(f"  Tokens: {len(tokens_info.tokens)}")
    print(f"  Token breakdown: {[t.decode('utf-8') for t in tokens_info.tokens]}")

special_text = "Hello! 你好 🌍"

response = client.models.compute_tokens(
    model='gemini-2.0-flash',
    contents=special_text,
)

tokens_info = response.tokens_info[0]
print("Token analysis for special characters:")
for token, token_id in zip(tokens_info.tokens, tokens_info.token_ids):
    try:
        decoded = token.decode('utf-8')
    except:
        decoded = f"<binary: {token.hex()}>"
    print(f"  '{decoded}' -> ID: {token_id}")

import asyncio
from google import genai

client = genai.Client(api_key='your-api-key')

async def count():
    # count_tokens
    response = await client.aio.models.count_tokens(
        model='gemini-2.0-flash',
        contents='What is AI?',
    )
    print(f"Tokens: {response.total_tokens}")

asyncio.run(count())

import asyncio
from google import genai

client = genai.Client(vertexai=True, project='my-project', location='us-central1')

async def compute():
    # compute_tokens (Vertex AI only)
    response = await client.aio.models.compute_tokens(
        model='gemini-2.0-flash',
        contents='What is AI?',
    )
    print(f"Token IDs: {response.tokens_info[0].token_ids}")

asyncio.run(compute())