maifeeulasad/askubuntu-runner.py

askubuntu-runner.py

from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
from peft import PeftModel

base_model_id = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
peft_model_id = "maifeeulasad/askubuntu-model"

model = AutoModelForCausalLM.from_pretrained(base_model_id, device_map="auto", trust_remote_code=True)
model = PeftModel.from_pretrained(model, peft_model_id)
tokenizer = AutoTokenizer.from_pretrained(base_model_id)

generator = pipeline("text-generation", model=model, tokenizer=tokenizer)

question = "Tell me how to install rootless docker on ubuntu 18 LTS?"
output = generator(question, max_new_tokens=16384, return_full_text=False)[0]["generated_text"]
print(output)