def accuracy(output, target, topk=(1,)):
    """Compute top-k accuracy"""
    pred = output.topk(max(topk), 1, True, True)[1].t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))
    return [float(correct[:k].reshape(-1).float().sum(0)) for k in topk]

Top-1 Accuracy = (Number of correct predictions) / (Total number of samples)

Top-5 Accuracy = (Samples with correct label in top 5) / (Total number of samples)

from open_clip import IMAGENET_CLASSNAMES, OPENAI_IMAGENET_TEMPLATES

# Example class: "golden retriever"
classname = "golden retriever"
templates = [
    "a photo of a {}.",
    "a picture of a {}.",
    "an image of a {}.",
    # ... 80 templates total
]

# Encode all templates for this class
text_features = []
for template in templates:
    text = template.format(classname)
    text_features.append(model.encode_text(tokenize(text)))

# Average the features
class_embedding = torch.mean(torch.stack(text_features), dim=0)
class_embedding /= class_embedding.norm()  # L2 normalize

image_features = model.encode_image(preprocess(image))
image_features /= image_features.norm()  # L2 normalize

logits = 100.0 * image_features @ class_embeddings.T
probs = logits.softmax(dim=-1)
predicted_class = probs.argmax()

top1_count = 0
top5_count = 0
total = 0

for image, label in dataloader:
    logits = compute_logits(model, image, class_embeddings)
    
    # Top-1
    pred = logits.argmax(dim=-1)
    top1_count += (pred == label).sum().item()
    
    # Top-5
    top5_preds = logits.topk(5, dim=-1)[1]
    top5_count += (top5_preds == label.unsqueeze(-1)).any(dim=-1).sum().item()
    
    total += len(label)

top1_accuracy = top1_count / total
top5_accuracy = top5_count / total

Recall@K = (Queries with correct item in top K) / (Total queries)

image_features = model.encode_image(images)  # [B, D]
text_features = model.encode_text(texts)      # [N, D]

# Compute similarities
similarity = image_features @ text_features.T  # [B, N]

# For each image, find top K texts
top_k_indices = similarity.topk(k=5, dim=1)[1]  # [B, K]

# Check if correct caption is in top K
recall_at_5 = (top_k_indices == ground_truth_idx.unsqueeze(1)).any(dim=1).float().mean()

# Transpose similarity matrix
similarity = text_features @ image_features.T  # [N, B]

# For each text, find top K images
top_k_indices = similarity.topk(k=5, dim=1)[1]  # [N, K]

# Check if correct image is in top K
recall_at_5 = (top_k_indices == ground_truth_idx.unsqueeze(1)).any(dim=1).float().mean()

ranks = []
for query_idx in range(len(queries)):
    similarity_scores = compute_similarity(query_idx, all_items)
    rank = (similarity_scores.argsort(descending=True) == ground_truth[query_idx]).nonzero()[0]
    ranks.append(rank.item() + 1)  # 1-indexed

mean_rank = sum(ranks) / len(ranks)

average_performance = sum(accuracy_per_dataset) / num_datasets

weighted_avg = sum(accuracy[i] * size[i] for i in datasets) / sum(size)

python -m open_clip_train.main \
    --report-to tensorboard \
    --logs ./logs/tensorboard \
    # ... other args

tensorboard --logdir=logs/tensorboard/ --port=7777

python -m open_clip_train.main \
    --report-to wandb \
    --wandb-project-name my-clip-project \
    # ... other args

from open_clip_train.main import train_one_epoch

def custom_metrics(model, data, epoch):
    # Your custom evaluation
    custom_score = evaluate_custom_task(model, data)
    return {'custom_metric': custom_score}

# Metrics will be logged to your configured backend

from clip_benchmark.datasets.builder import build_dataset
from clip_benchmark.metrics import zeroshot_classification

# Load dataset
dataset = build_dataset('imagenet1k', root='/path/to/imagenet', split='val')

# Evaluate
metrics = zeroshot_classification.evaluate(
    model,
    dataset,
    tokenizer,
    batch_size=64,
    num_workers=4
)

print(f"Top-1: {metrics['acc1']:.2%}")
print(f"Top-5: {metrics['acc5']:.2%}")
print(f"Mean per-class: {metrics['mean_per_class_recall']:.2%}")

import torch
from tqdm import tqdm

def evaluate_zero_shot(model, dataloader, classifier):
    model.eval()
    top1_correct = 0
    top5_correct = 0
    total = 0
    
    with torch.no_grad():
        for images, targets in tqdm(dataloader):
            images = images.cuda()
            targets = targets.cuda()
            
            # Get image features
            image_features = model.encode_image(images)
            image_features /= image_features.norm(dim=-1, keepdim=True)
            
            # Compute logits
            logits = 100.0 * image_features @ classifier
            
            # Top-1
            pred = logits.argmax(dim=-1)
            top1_correct += (pred == targets).sum().item()
            
            # Top-5
            _, top5_pred = logits.topk(5, dim=-1)
            top5_correct += (top5_pred == targets.unsqueeze(-1)).any(dim=-1).sum().item()
            
            total += len(targets)
    
    top1_acc = 100.0 * top1_correct / total
    top5_acc = 100.0 * top5_correct / total
    
    return {'top1': top1_acc, 'top5': top5_acc}