Function calling comparison

benchmark.py

# !pip install litellm python-dotenv

import os
import litellm


def list_files(path="."):
    """ List the files in the current directory """
    """List files and directories in the specified path.
    
    Args:
        path: The directory path to list contents from. Defaults to current directory.
        
    Returns:
        A string containing the directory listing with file/directory names and sizes.
    """
    try:
        items = []
        for item in os.listdir(path):
            item_path = os.path.join(path, item)
            size = os.path.getsize(item_path) if os.path.isfile(item_path) else "dir"
            items.append(
                f"{'[DIR]' if size == 'dir' else '[FILE]'} {item:30} {size if size == 'dir' else f'{size:,} bytes'}")
        return "\n".join(sorted(items))
    except Exception as e:
        return f"Error listing directory: {str(e)}"


import json


def completion(model, question):
    max_turns = 5  # Prevent infinite loops
    final_response = None

    messages = [
        {
            'role': 'user',
            'content': question
        }
    ]

    while max_turns > 0 and final_response is None:

        print("Total messages in the context:", len(messages))
        response = litellm.completion(**{
            "model": model,
            "messages": messages,
            "temperature": 0.2,
            "stream": False,
            "tools": [
                {
                    "type": "function",
                    "function": {
                        "name": "list_files",
                        "description": "List files and directories in the specified path.",
                        "parameters": {
                            "type": "object",
                            "properties": {
                                "path": {
                                    "type": "string",
                                    "description": "The directory path to list contents from.",
                                    "default": "."
                                }
                            }
                        }
                    }
                }
            ]
        })
        
        message = response.choices[0].message
        if hasattr(message, 'tool_calls') and message.tool_calls:
            messages.append(message)
            
            for tool_call in message.tool_calls:
                print("Calling tool:", tool_call.function.name, "with parameters:", tool_call.function.arguments)
                if tool_call.function.name == "list_files" and tool_call.type == "function":
                    path = json.loads(tool_call.function.arguments).get("path", ".")
                    result = list_files(path)
                    messages.append({
                        "role": "tool",
                        "name": "list_files",
                        "tool_call_id": tool_call.id,
                        "content": str(result)
                    })
                else:
                    raise Exception(f"Invalid tool call: {tool_call.function}")
        else:
            return message.content
        
        max_turns -= 1
        
    raise Exception("tired of looping, failed!")
    
    
from datetime import datetime
from dotenv import load_dotenv
load_dotenv()

for model in [
    "anthropic/claude-3-5-sonnet-20240620",
    "openai/gpt-4o-mini",
    "ollama/tinyllama:latest",
    "ollama/mistral:latest",
    "ollama/qwen2.5:32b",
    "ollama/llama3.3:latest",
    "ollama/deepseek-r1:32b",
]:
    print("\n---\nTrying model:", model)
    t0 = datetime.now()
    try:
        print(
            completion(
                model=model,
                question="How many files do I have in the current directory plus its parent directory?"
            )
        )
        print("Time taken:", datetime.now() - t0)
    except Exception as e:
        print("Error:", str(e))    

results.txt

---
Trying model: anthropic/claude-3-5-sonnet-20240620
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 4
Now, let's count the files in both directories:

1. Current directory (.):
   There are 11 files in the current directory.

2. Parent directory (..):
   There is 1 file in the parent directory.

To answer your question: The total number of files in the current directory plus its parent directory is 11 + 1 = 12 files.

Note that this count only includes files and not directories. If you also want to include directories in the count, please let me know, and I can provide that information as well.
Time taken: 0:00:05.424786

---
Trying model: openai/gpt-4o-mini
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 4
In the current directory, there are 12 files and 13 directories. In the parent directory, there are 1 file and 17 directories. 

Combining both, you have a total of 13 files and 30 directories across the current and parent directories.
Time taken: 0:00:03.072860

---
Trying model: ollama/tinyllama:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": "."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": "."}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": "."}
Total messages in the context: 7
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": "."}
Total messages in the context: 9
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": "."}
Error: tired of looping, failed!

---
Trying model: ollama/mistral:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": "./"}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": "./"}
Total messages in the context: 7
Calling tool: function_sum with parameters: {"argument1": 20, "argument2": 15}
Error: Invalid tool call: Function(arguments='{"argument1": 20, "argument2": 15}', name='function_sum')

---
Trying model: ollama/qwen2.5:32b
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 5
Calling tool: calculate_total_files with parameters: {"current_directory_count": 12, "parent_directory_count": 1}
Error: Invalid tool call: Function(arguments='{"current_directory_count": 12, "parent_directory_count": 1}', name='calculate_total_files')

---
Trying model: ollama/llama3.3:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: count_files with parameters: {"path": "./"}
Error: Invalid tool call: Function(arguments='{"path": "./"}', name='count_files')

---
Trying model: ollama/deepseek-r1:32b
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: list_iles with parameters: {"path": ".."}
Error: Invalid tool call: Function(arguments='{"path": ".."}', name='list_iles')

results2.txt

---
Trying model: anthropic/claude-3-5-sonnet-20240620
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 4
Now, let's count the files in each directory:

1. Current directory (.):
   There are 11 files in the current directory.

2. Parent directory (..):
   There is 1 file in the parent directory.

To answer your question: The total number of files in the current directory plus its parent directory is 11 + 1 = 12 files.

Note that this count only includes files directly in these directories, not files within subdirectories. If you want to include files in subdirectories as well, we would need a different function that can recursively list all files, which is not available in the current set of tools.
Time taken: 0:00:05.980258

---
Trying model: openai/gpt-4o-mini
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 4
In the current directory, there are 12 files and 13 directories. In the parent directory, there are 1 file and 15 directories. 

To summarize:
- Current directory: 12 files
- Parent directory: 1 file

Total files in both directories: **13 files**.
Time taken: 0:00:02.683526

---
Trying model: ollama/tinyllama:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from.", "default": ".", "required": true}}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": ".", "required": true}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": ".", "required": true}
Total messages in the context: 7
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": ".", "required": true}
Total messages in the context: 9
Calling tool: list_files with parameters: {"path": {"type": "string", "description": "The directory path to list contents from."}, "default": ".", "required": true}
Error: tired of looping, failed!
Time taken: 0:00:04.879760

---
Trying model: ollama/mistral:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": "./"}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 7
Calling tool: function_name with parameters: {"argument_name": "list_files", "argument_value": "./"}
Error: Invalid tool call: Function(arguments='{"argument_name": "list_files", "argument_value": "./"}', name='function_name')
Time taken: 0:00:13.521577

---
Trying model: ollama/qwen2.5:32b
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": ".."}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 7
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 9
Calling tool: list_files with parameters: {"path": "."}
Error: tired of looping, failed!
Time taken: 0:00:57.761541

---
Trying model: ollama/llama3.2:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 3
Calling tool: list_files with parameters: {"path": "./"}
Total messages in the context: 5
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 7
Calling tool: list_files with parameters: {"path": "./.."}
Total messages in the context: 9
Calling tool: list_files with parameters: {"path": "./.."}
Error: tired of looping, failed!
Time taken: 0:00:05.967066

---
Trying model: ollama/llama3.3:latest
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: count_files with parameters: {"path": "./"}
Error: Invalid tool call: Function(arguments='{"path": "./"}', name='count_files')
Time taken: 0:00:58.135384

---
Trying model: ollama/deepseek-r1:32b
Total messages in the context: 1
Calling tool: list_files with parameters: {"path": "."}
Total messages in the context: 3
Calling tool: list_iles with parameters: {"path": ".."}
Error: Invalid tool call: Function(arguments='{"path": ".."}', name='list_iles')
Time taken: 0:00:30.213078