U
    |`8                     @   sN   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ G dd dejZ	dS )    N)autocast)CrossEntropyLoss)AutoModelWithLMHeadc                       s.   e Zd Zded fddZd	ddZ  ZS )
GPT2OdinModelF)use_ampc                    sX   t t|   tdj| _| jj}tj	|j
ddd| _tj	|j
|jdd| _|| _d S )N
distilgpt2   T)in_featuresout_featuresbias)superr   __init__r   from_pretrained
base_modeltransformerconfignnLinearn_embdlinear_g_component
vocab_sizelinear_h_componentr   )selfr   r   	__class__ 1/mnt/c/Rosko/projects/ood/src/models/gpt2_odin.pyr   	   s        zGPT2OdinModel.__init__Nc              
   C   s   t | j | j||dj}t| |}| |}|| }d }|d k	r|dd dd d f  }	|ddd f  }
t	 }||	
d|	d|

d}|r||f}n
|||f}|W  5 Q R  S Q R X d S )N)	input_idsattention_mask.r   )r   r   r   last_hidden_statetorchsigmoidr   r   
contiguousr   viewsize)r   r   r   labelshidden_statesg_logitsh_prod	lm_logitslossshift_logitsshift_labelsloss_fctoutputr   r   r   forward   s.     
 
zGPT2OdinModel.forward)F)N)__name__
__module____qualname__boolr   r0   __classcell__r   r   r   r   r      s    r   )
r!   torch.nnr   Ztorch.cuda.ampr   r   transformersr   Moduler   r   r   r   r   <module>   s
   