stdin.patch

#!/bin/bash

host=m1-ultra

function lms() {
  ssh $host lms $*
}

function testcase() {
  echo '{
      "model": "'"$1"'",
      "messages": [{"role": "user", "content": "What dell products do you have under $50 in electronics?"}],
      "tools": [
        {
          "type": "function",
          "function": {
            "name": "search_products",
            "description": "Search the product catalog by various criteria. Use this whenever a customer asks about product availability, pricing, or specifications./nothink",
            "parameters": {
              "type": "object",
              "properties": {
                "query": {
                  "type": "string",
                  "description": "Search terms or product name"
                },
                "category": {
                  "type": "string",
                  "description": "Product category to filter by",
                  "enum": ["electronics", "clothing", "home", "outdoor"]
                },
                "max_price": {
                  "type": "number",
                  "description": "Maximum price in dollars"
                }
              },
              "required": ["query"],
              "additionalProperties": false
            }
          }
        }
      ]
    }'
}

version=$(ssh $host mdls "/Applications/LM\ Studio.app" -name kMDItemVersion  -raw)

echo "Testing models on LM Studio $version using testcase:"
testcase "<model-goes-here>" | jq

lms ls --json | jq -r '.[] | select(.trainedForToolUse == true) | .modelKey' | while read model; do
    echo "🤖 Testing $model..."
    output=$(curl -s "http://$host:1234/v1/chat/completions" \
    -H "Content-Type: application/json" \
    -d "$(testcase $model)")
    echo $output | jq .choices

    echo "$output" | jq -r '
      .choices[] |
      select(
        .finish_reason != "tool_calls" or
        (.message.tool_calls == null or (.message.tool_calls | length == 0))
      ) |
      "⚠️ Unexpected tool response: finish_reason is \"\(.finish_reason)\", tool_calls is \(.message.tool_calls // "missing or empty")"
    '


done