39 lines
1.6 KiB
Python
39 lines
1.6 KiB
Python
import torch
|
|
from AiBase import AiBase
|
|
|
|
|
|
class QwenAi(AiBase):
|
|
default_device = 'cuda' # the device to load the model onto
|
|
default_model_id = 'Qwen/Qwen1.5-1.8B-Chat'
|
|
|
|
default_instruction = {'role': 'system',
|
|
'name': 'system',
|
|
'content': 'Your name is "Laura". You are an AI created by Alice.'}
|
|
|
|
def __init__(self, model_id_or_path=default_model_id):
|
|
super().__init__(model_id_or_path)
|
|
|
|
def generate(self, messages):
|
|
try:
|
|
# prepare
|
|
messages = [m for m in messages if m['role'] != 'system']
|
|
input_messages = [self.default_instruction] + messages
|
|
|
|
# generate
|
|
text = self.tokenizer.apply_chat_template(input_messages, tokenize=False, add_generation_prompt=True)
|
|
model_inputs = self.tokenizer([text], return_tensors='pt').to(self.default_device)
|
|
generated_ids = self.model.generate(model_inputs.input_ids, max_new_tokens=300)
|
|
generated_ids = [
|
|
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
|
]
|
|
response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
|
|
|
# add response and save conversation
|
|
response_entry = {'role': 'assistant', 'name': 'assistant', 'content': response}
|
|
messages.append(response_entry)
|
|
self.record_conversation(input_messages, response_entry)
|
|
|
|
return messages
|
|
finally:
|
|
torch.cuda.empty_cache() # clear cache or the gpu mem will be used a lot
|