initial commit

chat_qwen.py

@@ -0,0 +1,55 @@
+import atexit
+import os
+import sys
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from utils.conversation import save_conversation_json
+from utils.prompt import prompt
+
+
+class ChatQwen:
+    default_device = 'cuda'  # the device to load the model onto
+    # default_model_id = 'Qwen/Qwen1.5-0.5B-Chat'
+    default_model_id = 'Qwen/Qwen1.5-1.8B-Chat'
+    # default_model_id = 'Qwen/Qwen1.5-4B-Chat'
+
+    default_instruction = {'role': 'system', 'content': 'Your name is "Laura". You are an AI created by Alice.'}
+
+    def __init__(self, model_id_or_path=default_model_id):
+        # model_id = model_id_or_path if not load_from_disk else os.path.abspath(sys.argv[1])
+
+        print('Loading ' + model_id_or_path)
+        self.model_id_or_path = model_id_or_path
+        self.model = AutoModelForCausalLM.from_pretrained(model_id_or_path, torch_dtype='auto', device_map='auto')
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id_or_path)
+        # print(tokenizer.default_chat_template)
+        # print(type(model))
+        # print(type(tokenizer))
+        print('Loaded')
+
+    def generate(self, messages):
+        # prepare
+        messages = [m for m in messages if m['role'] != 'system']
+        input_messages = [self.default_instruction] + messages
+
+        # generate
+        text = self.tokenizer.apply_chat_template(input_messages, tokenize=False, add_generation_prompt=True)
+        model_inputs = self.tokenizer([text], return_tensors='pt').to(self.default_device)
+        generated_ids = self.model.generate(model_inputs.input_ids, max_new_tokens=100)
+        generated_ids = [
+            output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+        ]
+        response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+        # add response and save conversation
+        messages.append({'role': 'assistant', 'content': response})
+        self.record_conversation(input_messages, {'role': 'assistant', 'content': response})
+
+        torch.cuda.empty_cache()  # clear cache or the gpu mem will be used a lot
+        return messages
+
+    def record_conversation(self, messages, response):
+        messages = messages + [response]
+        save_conversation_json(self.model_id_or_path, messages)