ViT SO400M 16 SigLIP2 Naflex

Model Details

• Model Type: Contrastive Image-Text, Zero-Shot Image Classification.
• Original: https://github.com/google-research/big_vision
• Dataset: WebLI
• Papers:
  • SigLIP 2: Multilingual Vision-Language Encoders with Improved Semantic Understanding, Localization, and Dense Features: https://arxiv.org/abs/2502.14786
  • Sigmoid loss for language image pre-training: https://arxiv.org/abs/2303.15343

Model Usage

Fixed-size image inference

import torch
from urllib.request import urlopen
from PIL import Image
from open_clip import create_model_from_pretrained, get_tokenizer

repo_id = "hf-hub:timm/ViT-SO400M-16-SigLIP2-naflex"
device = "cuda" if torch.cuda.is_available() else "cpu"

model, preprocess = create_model_from_pretrained(repo_id, device=device)
model = model.eval()
tokenizer = get_tokenizer(repo_id)

image = Image.open(urlopen(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png"
)).convert("RGB")
image = preprocess(image).unsqueeze(0).to(device)

labels_list = ["a dog", "a cat", "a donut", "a beignet"]
text = tokenizer(labels_list, context_length=model.context_length).to(device)

with torch.no_grad():
    image_features = model.encode_image(image, normalize=True)
    text_features = model.encode_text(text, normalize=True)
    logits = image_features @ text_features.T * model.logit_scale.exp()
    if model.logit_bias is not None:
        logits = logits + model.logit_bias
    text_probs = torch.sigmoid(logits)

print("Label probabilities:", list(zip(labels_list, [round(p.item(), 3) for p in text_probs[0]])))

Native NaFlex variable-aspect inference

import torch
from urllib.request import urlopen
from PIL import Image
from open_clip import create_model_and_transforms, get_tokenizer

repo_id = "hf-hub:timm/ViT-SO400M-16-SigLIP2-naflex"
device = "cuda" if torch.cuda.is_available() else "cpu"

model, _, preprocess_factory = create_model_and_transforms(
    repo_id,
    device=device,
    aug_cfg={"naflex": True, "use_timm": True},
)
model = model.eval()
tokenizer = get_tokenizer(repo_id)

# 576 tokens = 24 x 24 patch grid at patch size 16.
preprocess = preprocess_factory(max_seq_len=576, patch_size=16)

image = Image.open(urlopen(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png"
)).convert("RGB")
image = preprocess(image)

# Add batch dimension for a single image.
image = {
    k: v.unsqueeze(0).to(device) if torch.is_tensor(v) else v
    for k, v in image.items()
}
image["seq_len"] = image["patches"].shape[1]

labels_list = ["a dog", "a cat", "a donut", "a beignet"]
text = tokenizer(labels_list, context_length=model.context_length).to(device)

with torch.no_grad():
    image_features = model.encode_image(image, normalize=True)
    text_features = model.encode_text(text, normalize=True)
    logits = image_features @ text_features.T * model.logit_scale.exp()
    if model.logit_bias is not None:
        logits = logits + model.logit_bias
    text_probs = torch.sigmoid(logits)

print("Label probabilities:", list(zip(labels_list, [round(p.item(), 3) for p in text_probs[0]])))

Citation

@article{tschannen2025siglip,
  title={SigLIP 2: Multilingual Vision-Language Encoders with Improved Semantic Understanding, Localization, and Dense Features},
  author={Tschannen, Michael and Gritsenko, Alexey and Wang, Xiao and Naeem, Muhammad Ferjad and Alabdulmohsin, Ibrahim and Parthasarathy, Nikhil and Evans, Talfan
and Beyer, Lucas and Xia, Ye and Mustafa, Basil and H'enaff, Olivier and Harmsen, Jeremiah and Steiner, Andreas and Zhai, Xiaohua},
  year={2025},
  journal={arXiv preprint arXiv:2502.14786}
}

@article{zhai2023sigmoid,
  title={Sigmoid loss for language image pre-training},
  author={Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},
  journal={arXiv preprint arXiv:2303.15343},
  year={2023}
}

@misc{big_vision,
  author = {Beyer, Lucas and Zhai, Xiaohua and Kolesnikov, Alexander},
  title = {Big Vision},
  year = {2022},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/google-research/big_vision}}
}