56 lines
2.3 KiB
Python
56 lines
2.3 KiB
Python
import atexit
|
|
import os
|
|
import sys
|
|
|
|
import torch
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
from utils.conversation import save_conversation_json
|
|
from utils.prompt import prompt
|
|
|
|
|
|
class ChatQwen:
|
|
default_device = 'cuda' # the device to load the model onto
|
|
# default_model_id = 'Qwen/Qwen1.5-0.5B-Chat'
|
|
default_model_id = 'Qwen/Qwen1.5-1.8B-Chat'
|
|
# default_model_id = 'Qwen/Qwen1.5-4B-Chat'
|
|
|
|
default_instruction = {'role': 'system', 'content': 'Your name is "Laura". You are an AI created by Alice.'}
|
|
|
|
def __init__(self, model_id_or_path=default_model_id):
|
|
# model_id = model_id_or_path if not load_from_disk else os.path.abspath(sys.argv[1])
|
|
|
|
print('Loading ' + model_id_or_path)
|
|
self.model_id_or_path = model_id_or_path
|
|
self.model = AutoModelForCausalLM.from_pretrained(model_id_or_path, torch_dtype='auto', device_map='auto')
|
|
self.tokenizer = AutoTokenizer.from_pretrained(model_id_or_path)
|
|
# print(tokenizer.default_chat_template)
|
|
# print(type(model))
|
|
# print(type(tokenizer))
|
|
print('Loaded')
|
|
|
|
def generate(self, messages):
|
|
# prepare
|
|
messages = [m for m in messages if m['role'] != 'system']
|
|
input_messages = [self.default_instruction] + messages
|
|
|
|
# generate
|
|
text = self.tokenizer.apply_chat_template(input_messages, tokenize=False, add_generation_prompt=True)
|
|
model_inputs = self.tokenizer([text], return_tensors='pt').to(self.default_device)
|
|
generated_ids = self.model.generate(model_inputs.input_ids, max_new_tokens=100)
|
|
generated_ids = [
|
|
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
|
]
|
|
response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
|
|
|
# add response and save conversation
|
|
messages.append({'role': 'assistant', 'content': response})
|
|
self.record_conversation(input_messages, {'role': 'assistant', 'content': response})
|
|
|
|
torch.cuda.empty_cache() # clear cache or the gpu mem will be used a lot
|
|
return messages
|
|
|
|
def record_conversation(self, messages, response):
|
|
messages = messages + [response]
|
|
save_conversation_json(self.model_id_or_path, messages)
|