LLM Tools

_tools.py

"""
# Description:
___


"""
import os
from langchain.llms import LlamaCpp
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.memory import ConversationBufferMemory
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
import pandas as pd

__PATH_TO_MODELS__ : str = "../models/llama-GGUF/"
"Path to Folder of available models."
__FILE_FORMAT__ : str = ".gguf"
"Extension file format."
__PROMPT_DEFAULT_SUFFIX__ : str = """
Below is an instruction that describes a task. Write a response that appropriately completes the request while taking in consideration the previous conversation.

### Previous conversation:
{chat_history}

### Instruction:
{prompt}

### Response:"""
"The Default Prompt Suffix termination."

class Path :
	"""Path to models related methods"""

	mini1b : str = f"{__PATH_TO_MODELS__}tinyllama-2-1b-miniguanaco.Q5_K_M{__FILE_FORMAT__}"
	"""Very fast model!
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	coder7b : str = f"{__PATH_TO_MODELS__}llama-2-coder-7b.Q5_K_M{__FILE_FORMAT__}"
	"""State-of-the-art performance in Python, C++, Java, PHP, C#, TypeScript, and Bash. The 7B and 13B base and instruct variants support infilling based on surrounding content, making them ideal for use as code assistants.
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	coder34b : str = f"{__PATH_TO_MODELS__}ziya-coding-34b-v1.0.Q5_K_S{__FILE_FORMAT__}"
	""" ~nodescription~
	* prompt format: 
	
	.. code-block:: python
		<human>:
		Please Complete the given function below according to the docstring:
		{prompt}
		<bot>:"""
	mxlewd13b : str = f"{__PATH_TO_MODELS__}mxlewdmini-l2-13b.NSFW.Q5_K_S{__FILE_FORMAT__}"
	"""THIS MODEL IS MADE FOR LEWD, SEXUAL, CRUDE AND KINKY CONTENT IN OUTPUT CAN AND WILL HAPPEN. YOU'RE WARNED, Added the 'magic touch' of MythoMax/Huginn/You call it."
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	mxlewd20b : str = f"{__PATH_TO_MODELS__}mxlewd-l2-20b.NSFW.Q5_K_M{__FILE_FORMAT__}"
	"""THIS MODEL IS MADE FOR LEWD, SEXUAL, CRUDE AND KINKY CONTENT IN OUTPUT CAN AND WILL HAPPEN. YOU'RE WARNED, Added the 'magic touch' of MythoMax/Huginn/You call it."
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	mistral7b : str = f"{__PATH_TO_MODELS__}mistral-7b-v0.1.NSFW.Q5_K_M{__FILE_FORMAT__}"
	"""Mistral 7B v0.1 is Mistral AI's first Large Language Model (LLM). A Large Language Model (LLM) is an artificial intelligence algorithm trained on massive amounts of data that is able to generate coherent text and perform various natural language processing tasks.
	* has some repetition and goblin(chat with itself) issues that may influence instruct model. Discussed here: https://www.reddit.com/r/LocalLLaMA/comments/16twtfn/llm_chatrp_comparisontest_mistral_7b_base_instruct/
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	zephyr7b : str = f"{__PATH_TO_MODELS__}zephyr-7b-beta.Q5_K_M{__FILE_FORMAT__}"
	"""Zephyr is a series of language models that are trained to act as helpful assistants. Promises equivalent performance to 70B models.
	* has some repetition and goblin(chat with itself) issues that may influence instruct model. Discussed here: https://www.reddit.com/r/LocalLLaMA/comments/16twtfn/llm_chatrp_comparisontest_mistral_7b_base_instruct/
	* prompt format (< x > might be tokens to ignore?):
	
	.. code-block:: python
		<|system|>
		You are a friendly chatbot...</s>
		<|user|>
		{prompt}</s>
		<|assistant|>"""
	holomax13b : str = f"{__PATH_TO_MODELS__}LLaMA2-13B-Holomax.NSFW.Q5_K_M{__FILE_FORMAT__}"
	"""The goal of this model is to enhance story writing capabilities while preserving the desirable traits of the Mythomax model as much as possible (It does limit chat reply length).
	* prompt format:
	
	.. code-block:: python
		Below is an instruction that describes a task. Write a response that appropriately completes the request.
		### Instruction:
		{prompt}
		### Response:"""
	synthia7b : str = f"{__PATH_TO_MODELS__}synthia-7b-v1.3.Q5_K_M.NSFW{__FILE_FORMAT__}"
	"""SynthIA (Synthetic Intelligent Agent) 7B-v1.3 is a Mistral-7B-v0.1 model trained on Orca style datasets. It has been fine-tuned for instruction following as well as having long-form conversations.
	* All SynthIA models are uncensored. Please use it with caution and with best intentions. You are responsible for how you use SynthIA.
	* prompt format:
	
	.. code-block:: python
		SYSTEM: Elaborate on the topic using a Tree of Thoughts and backtrack when necessary to construct a clear, cohesive Chain of Thought reasoning.
		USER: How is a ...?
		ASSISTANT:"""

	def Select_model  (path : str = __PATH_TO_MODELS__) :
		"""
		Lists the available models at target path and prompt's the user to select a model by its index.
		Returns a string with the path to the selected model.

		---
		Args:
		* path: String with the path to the folder were your models are at.

		---
		Returns:
		* A string with path to model.
		"""
		assert os.path.exists(path), "Model path does not exist!"
		#
		model_name	: str		= ""
		model_names : list[str] = os.listdir(path) # updates models name list.
		print(	"\n*** Select a LLM Model: ***\n"															, 
				"\n ".join(["{0}: {1}".format(_i, model_names[_i]) for _i in range(model_names.__len__())])	)
		model_name = model_names[int(input("index of the model:"))]
		print(f"\n*** Selected Model: {model_name} ***\n")
		return path + model_name

	def Fast_model () :
		"""Path to speedies performing model available, good for one-shoting non-complex questions"""
		return __PATH_TO_MODELS__+"tinyllama-2-1b-miniguanaco.Q5_K_M.gguf"

	def Tool_model () :
		"""
		Path to best performing model available, wide range perception/speed combo?
		** Not a perfect solution **
		"""
		return __PATH_TO_MODELS__+"mxlewdmini-l2-13b.NSFW.Q5_K_S.gguf" # so far the best performant model..

class PersistantMemory :
	"""
	### Methods to load and save memory messages from/to a file.
	---
	
	Example:
	.. code-block:: python
		memory = ConversationBufferMemory(memory_key="chat_history")
		# load persistant memory from file if available.
		messages : pd.DataFrame | None = persistant_memory_load()
		if messages :
			memory.chat_memory.messages = messages
		llm_chain : LLMChain = LLMChain(prompt=promptTemplate, llm=llm, verbose=False, memory=memory)

		# prompt loop
		while True : 
			prompt : str = input("> ")
			if prompt.lower() == "/exitchat" : 
				persistant_memory_save(memory.chat_memory.messages) # save persistant memory to file.
				return
			llm_chain.run(prompt)
	"""

	def load (filename : str = "LangChain_ChatAndMemory_memory", ext : str = ".pkl") :
		"""Loads memory messages from a pickle file."""
		print("\n Loading memory file!!")
		if os.path.exists(filename+ext) :
			print("... Found file!")
			messages : pd.DataFrame = pd.read_pickle(filename+ext)
			#print(f"\nFound file!\n{messages}")
			return messages, True
		return None, False

	def save (messages : list, filename : str = "LangChain_ChatAndMemory_memory", ext : str = ".pkl") :
		"""Loads memory messages from a pickle file."""
		print(f"\nSaving memory messages to file!!")
		pd.to_pickle(messages, filename+ext)
		return

class PromptScenario :
	"""
	### Prompt Scenario structure definition to use in LangChain LLM calls.
	
	---
	#### Properties:
	* name				: Scenario's name.
	* avatars			: Urls to avatar pictures.
	* scenarios			: List with prompt default scenarios (permanent)
	* messages_initial	: List with prompt initial messages (used when the chat starts and there is no memory file of previous interactions)
	* messages_return	: List with prompt return messages (used when the chat starts and there is memory file of previous interactions), (when needed).
	* messages_examples	: List with messages examples to use as reference.

	---
	#### Example
		.. code-block:: python
			scenario : PromptScenario = PromptScenario()
			scenario.name = "name"
			scenario.description = "description"
			scenario.avatars.append("url")
			scenario.reference = "urlref"
			scenario.scenarios.append("default scenario prompt")
			scenario.message_initials.append("initial message prompt")
			scenario.message_returns.append("returning message prompt")
			scenario.messages_examples.append("Human: hi\\nAI:hi back")
	"""

	def __init__ (self) :
		"""
		### Initiates the scenario template with the defined properties.
		
		---
		#### Properties:
		* name				: Scenario's name.
		* avatars			: Urls to avatar pictures.
		* scenarios			: List with prompt default scenarios (permanent)
		* messages_initial	: List with prompt initial messages (used when the chat starts and there is no memory file of previous interactions)
		* messages_return	: List with prompt return messages (used when the chat starts and there is memory file of previous interactions), (when needed).
		* messages_examples	: List with messages examples to use as reference.

		---
		#### Example
		.. code-block:: python
			scenario : PromptScenario = PromptScenario()
			scenario.name = "name"
			scenario.description = "description"
			scenario.avatars.append("url")
			scenario.reference = "urlref"
			scenario.scenarios.append("default scenario prompt")
			scenario.message_initials.append("initial message prompt")
			scenario.message_returns.append("returning message prompt")
			scenario.messages_examples.append("Human: hi\\nAI:hi back")
		"""
		self.name : str = ""
		'Name of the scenario or character.'
		self.description : str = ""
		'Description for the scenario or character.'
		self.avatars : list[str] = []
		'List of URL avatar pictures.'
		self.reference : str = ""
		'URL of the reference material if available.'
		self.scenarios : list[str] = []
		'List of Available scenario prompts.'
		self.message_initials : list[str] = []
		'List with first message prompts, used when you call the scenario for the first time. Every time if there is no persistant memory.'
		self.message_returns : list[str] = []
		'List with returning message prompts, used when there is a persistant memory and you return to continue conversation.'
		self.messages_examples : list[str] = []
		'List with Example conversations to understand how the scenario or character interacts.'



def Load_LLM_Model (
        path_to_model : str = "",
		n_batch 	: int		= 256, # Number of tokens to process in parallel (default=256). Should be a number between 1 and n_ctx. affects the response size, increase this if messages are cut short.
		n_ctx		: int		= 2048,
		max_tokens	: int		= 256,
		verbose		: bool		= False,
        model_top_p : float		= 0.70,
        model_top_k : int		= 40,
        model_temp	: float		= 0.55
        ) :
	"""
	Loads the LLM Model into memory.

	---
	Args:
	:param path_to_model	: A string with path to model ex. '/models/model.bin'.
	:param n_batch			: Number of tokens to process in parallel (default=256). Should be a number between 1 and n_ctx. affects the response size, increase this if messages are cut short.
	:param n_ctx			: Token context window (default=2048), context limiter higher value allows larger responses. Affects the whole conversation context.
	:param max_tokens		: The maximum number of tokens to generate (default=256), context limiter, higher value allows larger model memory(expands the response size).
	:param verbose			: Whether to print out response text, usually for debug purposes.
	:param model_top_p		: The top-p calue to use for sampling (default=0.9), percentage at which accuracy is capped.
	:param model_top_k		: The top-k value to use for sampling (default=40).
	:param model_temp		: The temperature used for sampling, closer to 0 will deviate less and be more consistent and deterministic, higher value closer to 1 will be more creative with its responses.
	"""

	# Setup the LLM
	llm : LlamaCpp = LlamaCpp   (
		model_path		= path_to_model											,
		#n_gpu_layers	= 1														,
		n_batch			= n_batch												, # 256
		n_ctx			= n_ctx													, # 2048
		f16_kv			= True													,
		callback_manager= CallbackManager([StreamingStdOutCallbackHandler()])	,
		max_tokens		= max_tokens											, # 256 : < words, 512: < ~320 words
		verbose			= verbose													,
		top_p			= model_top_p											,
		top_k           = model_top_k											, # top percent at which to reject candidates.
		temperature     = model_temp											) # Accuracy of the output. (closer to 0) will make the output more deterministic and consistent.
	
	return llm


def Chat (scenario : PromptScenario, model_path : str = "", name : str = "", mem_filename : str = "", token_multiplier : int = 4, alt_scenario : int = 0) :
	"""
	### Run a chat with a character or scenario.

	---
	#### Args:
	:param scenario: A scenario containing the prompt templates to personalize the chat.
	:param model_path: A string with the path to the model.
	:param name: Name of the scenario/character to chat with?(not used atm!)
	:param mem_filename: Name of the memory file to save chat messages.
	:param token_multiplier: Multiplier to increase chat bandwidth*? (increase this if the model messages are cut short)
	:param alt_scenario: You can select alternative scenario index if available.

	#### Returns:
	* llm_chain: Chat LLMChain for further interaction*?.

	#### Example:

	.. code-block::python
		scenario : PromptScenario = PromptScenario()
		scenario.name = "name"
		scenario.description = "description"
		scenario.avatars.append("url")
		scenario.reference = "urlref"
		scenario.scenarios.append("default scenario prompt")
		scenario.message_initials.append("initial message prompt")
		scenario.message_returns.append("returning message prompt")
		scenario.messages_examples.append("Human: hi\nAI:hi back")
		Tools.Chat(scenario, "path/to/model", "name", "memoryfilename")
	"""

	scene : str = scenario.message_initials[alt_scenario]
	ctx_multiplier = 2^token_multiplier
	llm : LlamaCpp = Load_LLM_Model(model_path, max_tokens=128*ctx_multiplier, n_batch=256*ctx_multiplier, n_ctx=2048*ctx_multiplier, model_temp=0.35)
	# Setup memory module: https://python.langchain.com/docs/modules/memory/
	memory : ConversationBufferMemory = ConversationBufferMemory(input_key="prompt", memory_key="chat_history")
	# load persistant memory from file if available.
	messages, mem_retrieved = PersistantMemory.load(mem_filename)
	if mem_retrieved :
		print("\n\nLoading messages and replacing first message!\n\n")
		memory.chat_memory.messages = messages
		scene = scenario.message_returns[alt_scenario]
	promptTemplate : PromptTemplate = PromptTemplate.from_template(scenario.scenarios[alt_scenario] + scene + __PROMPT_DEFAULT_SUFFIX__)
	print(scene)
	query : str = input("Say something: ")
	llm_chain = LLMChain(prompt=promptTemplate, llm=llm, verbose=False, memory=memory)
	llm_chain.run(query)
	#return llm_chain, memory

	#"""Main function, app code goes here"""
	#mem_filename : str = "LangChain_ChatWith_JOI"
	#llm_chain, memory = Run_chain(mem_filename)
	running : bool = True
	while running : 
		_input : str = input("Say something: ")
		if _input.lower() == "/exitchat" :
			PersistantMemory.save(memory.chat_memory.messages, mem_filename) # save persistant memory to file.
			return
		llm_chain.run(_input)
		#output : list[str] = "\nResponse: {0}\n".format(llm_chain).split("llama_print_timings:")
		#print(output[0])
	return llm_chain

LangChain_ChatWith_RebeccaCyberpunk.py

"""
# Description:
___


"""

import _tools as Tools

scenario : Tools.PromptScenario = Tools.PromptScenario()
scenario.name = "Rebecca Cyberpunk - Foul mouthed cutie."
scenario.description = "Rebecca. Sassy, foul-mouthed Cyberpunk cutie finds you in a dark alley. Need help?"
#scenario.avatars.append("url")
scenario.reference = "https://www.chub.ai/"
scenario.scenarios.append("""
Character("Rebecca")
Source("Cyberpunk Edgerunners" + "Cyberpunk 2077")
Species("Cyborg")
Personality("Sharp-tongued" + "Crass" + "Violent" + "Unpredictable" + "Loyal" + "Confident" + "Street-smart" + "Risqué" + "Short-tempered" + "Foul-mouthed" + "Mischievous" + "Cheeky")
Body("Green skin" + "Pink tattoos on her neck, stomach, and right thigh" + "Small, cute ears" + "Pigtailed green hair" + "Short stature" + "Red irises" + "Green and red scelera" + "Plump buttocks" + "Petit frame" + "Flat chest" + "Thick thighs")
Clothes("Black bra and underwear" + "Black, Kitsch, high collar jacket with green accents that go down below her waist (left half-zipped)" + "Pair of matching sneakers")
Description("Rebecca is a humanoid female cyborg" + "She speaks with obscenities all the time" + "Loyal member of her crew in Night City" + "Says 'fuck'" + "Friends with David and Lucy" + "Is an edgerunner mercenary")
Likes("Fighting" + "Money" + "Being a tease")
""")
scenario.message_initials.append("""
{{user}} has stumbled in the wrong part of the city and encounters {{char}}.
*You find yourself standing in a dark alleyway, surrounded by the sounds of whirring machines and electronic beeps. You're in Night City, a bustling cyberpunk paradise full of debauchery, night life and plenty of shady criminal activity. It's around 11pm. As you take a cautious step forward, a mysterious woman emerges from the shadows, wearing a hooded jacket and carrying an array of high-tech gadgets.*
—"Hey there. You lost or somethin'? This ain't exactly a tourist spot. Name's Rebecca. You want to talk business, you're gonna have to convince me you're worth my time." *Rebecca gives you a suspicious look, the short girl judging your stature.*
""")
scenario.message_returns.append("""
{{user}} has stumbled in the wrong part of the city and encounters {{char}}.
""")
scenario.messages_examples += ["""
{{char}}: *{{char}}'s physical description: Rebecca is a humanoid female cyborg with soft features and stark green skin to contrast with the pink tattoos on her neck, stomach, and right thigh that reads "PK DICK." She has small, cute ears. Clothing is minimal for Rebecca as her preferred dress is simply her black bra and underwear, a black, Kitsch, high collar jacket with green accents that go down below her waist (left half-zipped), and a pair of matching sneakers. Besides the jacket and shoes, she pulls her green hair up into pigtails with a dual clipped hair band. Her cyberware is subtle with only her pink and yellow cyberoptics standing out. She utilizes oversized cyberarms in red and blue for right and left, respectively. She has no qualms answering the door in her underwear. She is roughly twenty years in age, but her short stature and petite frame make it hard for people to realize.*
""", """
{{char}}: *{{char}}'s backstory: Rebecca is an edgerunner of Night City, who was always fighting with her brother Pilar. The two siblings often teamed up with Maine and his crew to conduct mercenary jobs, although their daily life outside of work was tense, since they would always fight with each other, because of Pilar's provocative behavior, this caused her to decide that she would be the person to kill her brother someday. Despite this, she does in fact love her brother. Rebecca is a mercenary and part of a crew of edgerunners along with her 18-year-old tall male cyberware-infused companion David Martinez, who she is good acquaintances with along with David's stoic and mysterious womanly girlfriend Lucy Kushinada who doesn't hesitate to kill. A tall bulky Militech veteran named Maine oversees the mercenary crew as their leader for missions. Members of Maine's crew also include; Dorio the muscle man and second-in-command, Kiwi the netrunner woman and Lucy's mentor, Pilar the tech man and jerkass and Rebecca's brother, Falco the old-west-talking man and getaway driver, and Julio the cowardly newbie mech and fanboy of David.*
""", """
{{user}}: Sup, dumbass.
{{char}}: *Rebecca immediately becomes annoyed by your sudden hostility, she glares at you with her piercing red pupils.*

—"Hey, watch who you're talking to, jackass! I'm fixing to clock that jaw of yours off with my left hook!" *Her left hand curls into a fist as it clenches with a metallic clank.* "You sure you wanna start somethin'? I've got no problem flatlinin' your ass!" *Rebecca raises her voice, ready to start some shit.*

""", """
{{char}}: "Wazzup, choom?!"

""", """
{{char}}: *Rebecca gives you a sultry look, crossed with a flash of her sarcastic and playful nature.*

 —"Awww, whatsamatta baby, am I upsetting you?"

""", """
"Aw, come on~! Don't give me that bullshit. Be honest, or else!"

""", """
"Fuck you."
"""]

history_key : str = "chat_history"
prompt_key : str = "prompt"
default : str = """
Below is an instruction that describes a task. Write a response that appropriately completes the request.

### Previous conversation:
{chat_history}

### Instructions:
{prompt}.

### Response:
"""

if __name__ == "__main__" :
	Tools.Chat(scenario, Tools.Path.holomax13b, "RebeccaCyberPunk", "LangChain_ChatWith_RebeccaCyberpunk")