import torch from transformers import PreTrainedModel, PretrainedConfig from PIL import Image import numpy as np from typing import Optional, Union, List import json class Pix2TextConfig(PretrainedConfig): model_type = "pix2text" def __init__( self, vocab_size=30000, hidden_size=768, num_attention_heads=12, num_hidden_layers=12, intermediate_size=3072, max_position_embeddings=512, dropout_prob=0.1, **kwargs ): super().__init__(**kwargs) self.vocab_size = vocab_size self.hidden_size = hidden_size self.num_attention_heads = num_attention_heads self.num_hidden_layers = num_hidden_layers self.intermediate_size = intermediate_size self.max_position_embeddings = max_position_embeddings self.dropout_prob = dropout_prob class Pix2TextModel(PreTrainedModel): config_class = Pix2TextConfig def __init__(self, config): super().__init__(config) self.config = config # Basit bir CNN encoder self.image_encoder = torch.nn.Sequential( torch.nn.Conv2d(3, 64, 3, padding=1), torch.nn.ReLU(), torch.nn.MaxPool2d(2), torch.nn.Conv2d(64, 128, 3, padding=1), torch.nn.ReLU(), torch.nn.MaxPool2d(2), torch.nn.Conv2d(128, 256, 3, padding=1), torch.nn.ReLU(), torch.nn.AdaptiveAvgPool2d((8, 8)), torch.nn.Flatten(), torch.nn.Linear(256 * 8 * 8, config.hidden_size) ) # Text decoder self.text_decoder = torch.nn.Sequential( torch.nn.Linear(config.hidden_size, config.intermediate_size), torch.nn.ReLU(), torch.nn.Dropout(config.dropout_prob), torch.nn.Linear(config.intermediate_size, config.vocab_size) ) # Basit tokenizer için vocab self.vocab = {str(i): i for i in range(10)} # 0-9 rakamları self.vocab.update({chr(i): i+10 for i in range(ord('a'), ord('z')+1)}) # a-z self.vocab.update({chr(i): i+36 for i in range(ord('A'), ord('Z')+1)}) # A-Z self.vocab.update({' ': 62, '.': 63, ',': 64, '!': 65, '?': 66}) self.inv_vocab = {v: k for k, v in self.vocab.items()} def forward(self, pixel_values, labels=None): # Görüntüyü encode et image_features = self.image_encoder(pixel_values) # Text'e decode et logits = self.text_decoder(image_features) loss = None if labels is not None: loss_fct = torch.nn.CrossEntropyLoss() loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1)) return {"loss": loss, "logits": logits} def predict(self, image: Union[Image.Image, np.ndarray]) -> str: """Görüntüden metin çıkarma fonksiyonu""" if isinstance(image, Image.Image): image = np.array(image) # Görüntüyü ön işle if len(image.shape) == 3: image = torch.tensor(image).permute(2, 0, 1).float() / 255.0 else: image = torch.tensor(image).unsqueeze(0).float() / 255.0 image = image.unsqueeze(0) # Batch dimension with torch.no_grad(): outputs = self.forward(image) logits = outputs["logits"] # En yüksek olasılıklı tokenları seç predicted_ids = torch.argmax(logits, dim=-1) # Token'ları metne çevir text = "" for token_id in predicted_ids[0][:10]: # İlk 10 token if token_id.item() in self.inv_vocab: text += self.inv_vocab[token_id.item()] return text