U
    `t                     @   sL   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ G dd deZ	dS )    N)autocast)	Bernoulli)Module)AutoModelWithLMHeadc                       s.   e Zd Zd	ed fddZd
ddZ  ZS )GPT2LMHeadModelBackground333333?F)use_ampc                    s6   t t|   td| _tt|g| _	|| _
d S )N
distilgpt2)superr   __init__r   from_pretrainedtransformerr   torchtensorbernoulli_distr   )selfmur   	__class__ 7/mnt/c/Rosko/projects/ood/src/models/gpt2_background.pyr   	   s    z"GPT2LMHeadModelBackground.__init__Nc           	   
   C   s   t | j | jr| }| jj|d}tjd| jj	j
|d}t|d D ]<}t|d D ]*}|||f dkrpqZ|||f |||f< qZqJ| j|||dW  5 Q R  S Q R X d S )N)sample_shaper   )lowhighsize   )	input_idsattention_masklabels)r   r   trainingr   r   sampler   randintr   config
vocab_sizerange)	r   r   r   r   
input_sizeZpertrubation_maskZrandom_ints	batch_dimZ
sample_dimr   r   r   forward   s*        z!GPT2LMHeadModelBackground.forward)r   F)N)__name__
__module____qualname__boolr   r'   __classcell__r   r   r   r   r      s   
 r   )
r   Ztorch.cuda.ampr   Ztorch.distributions.bernoullir   torch.nnr   transformersr   r   r   r   r   r   <module>   s
   