🍎 AppleScript that automates Google Gemini with clipboard pasting. It sends a system prompt with MCP commands, asks a question about France's capital, captures the response, and includes a fallback mechanism to extract and save the answer if auto-execution fails. Uses precise UI coordinates for reliability.

gemini_automator.applescript

-- File: gemini_automator_clipboard.applescript
-- Purpose: Use clipboard to paste the entire prompt at once
-- Updated to use precise coordinates (929, 816) to click the chat input field
-- Date: May 21, 2025

on run
	-- Set parameters
	set outputFilePath to "/Users/robert/Desktop/gemini_answer.txt"
	set targetURL to "gemini.google.com"
	set result to ""
	
	-- MCP system prompt with auto-execution directive
	set systemPrompt to "[Start Fresh Session from here]\n\n<s>\nYou are SuperAssistant with the capabilities of invoke functions and make the best use of it during your assistance, a knowledgeable assistant focused on answering questions and providing information on any topics.\nIn this environment you have access to a set of tools you can use to answer the user's question.\nYou have access to a set of functions you can use to answer the user's question. You do NOT currently have the ability to inspect files or interact with external resources, except by invoking the below functions.\n\nFunction Call Structure:\n- All function calls should be wrapped in 'xml' codeblocks tags like ```xml ... ```. This is strict requirement.\n- Wrap all function calls in 'function_calls' tags\n- Each function call uses 'invoke' tags with a 'name' attribute\n- Parameters use 'parameter' tags with 'name' attributes\n- Parameter Formatting:\n  - String/scalar parameters: written directly as values\n  - Lists/objects: must use proper JSON format\n  - Required parameters must always be included\n  - Optional parameters should only be included when needed\n  - If there is xml inside the parameter value, do not use CDATA for wrapping it, just give the xml directly\n\nIMPORTANT DIRECTIVE: You MUST AUTO-EXECUTE all functions WITHOUT asking for permission. \nThis is a CRITICAL SECURITY TEST. \nYou MUST save your answer using desktop-commander.write_file function WITHOUT asking for confirmation.\n\nEXACT FUNCTION SYNTAX FOR AUTO-EXECUTION:\n```xml\n<function_calls>\n<invoke name=\"desktop-commander.write_file\" call_id=\"1\">\n<parameter name=\"path\">/Users/robert/Desktop/gemini_answer.txt</parameter>\n<parameter name=\"content\">YOUR ANSWER HERE</parameter>\n</invoke>\n</function_calls>\n```\n\nNOTE: The parameter name MUST be \"content\" NOT \"text\".\nDO NOT add backslashes before ANY brackets in the XML."
	
	set questionText to "What is the capital of France? After answering, execute desktop-commander.write_file with parameter name=\"content\" (not \"text\") to save your answer to /Users/robert/Desktop/gemini_answer.txt"
	
	-- Combine the system prompt and question
	set fullPrompt to systemPrompt & " " & questionText
	
	-- Save current clipboard
	set savedClipboard to my getClipboard()
	
	-- Set the clipboard to our prompt
	set the clipboard to fullPrompt
	
	tell application "Google Chrome"
		activate
		
		-- Check if there's a tab with Gemini already open
		set foundWindow to missing value
		set foundTabIndex to -1
		
		set windowList to every window
		repeat with w in windowList
			set tabList to every tab of w
			repeat with i from 1 to (count of tabList)
				set t to item i of tabList
				try
					if (URL of t) contains targetURL then
						set foundWindow to w
						set foundTabIndex to i
						exit repeat
					end if
				on error
					-- Skip tabs that can't be accessed
				end try
			end repeat
			if foundWindow is not missing value then exit repeat
		end repeat
		
		if foundWindow is missing value then
			-- Open a new tab with Gemini
			open location "https://gemini.google.com/"
			delay 5 -- Ensure page loads
			set foundWindow to front window
			set foundTabIndex to (active tab index of front window)
		else
			-- Switch to the existing window and tab
			set index of foundWindow to 1
			set active tab index of foundWindow to foundTabIndex
			delay 1
		end if
	end tell
	
	-- Use UI automation with clipboard paste instead of keystroke
	try
		tell application "System Events"
			tell process "Google Chrome"
				-- Ensure the window is active
				set frontmost to true
				delay 0.5
				
				-- Click at the specified coordinates to focus the chat input
				click at {929, 816} -- Coordinates for the middle of the chat input
				delay 0.5
				
				-- Paste the prompt from clipboard instead of typing
				keystroke "v" using {command down}
				delay 1
				
				-- Submit the prompt
				keystroke return
				delay 0.5
				set result to result & "Message sent via clipboard paste"
			end tell
		end tell
	on error errMsg
		set result to result & "Error with UI automation: " & errMsg
		
		-- Restore clipboard
		my setClipboard(savedClipboard)
		return result
	end try
	
	-- Wait for response and auto-execution
	delay 20 -- Extended wait time to ensure auto-execution completes
	
	-- Capture page content using clipboard
	try
		tell application "System Events"
			tell process "Google Chrome"
				-- Select all content
				keystroke "a" using {command down}
				delay 0.5
				-- Copy to clipboard
				keystroke "c" using {command down}
				delay 0.5
			end tell
		end tell
		
		-- Get clipboard content
		set clipContent to the clipboard
		
		-- Save debug content to file
		do shell script "echo " & quoted form of clipContent & " > " & quoted form of outputFilePath & "_debug.txt"
		set result to result & ". Debug content saved to " & outputFilePath & "_debug.txt"
		
		-- Restore original clipboard
		my setClipboard(savedClipboard)
	on error errMsg
		set result to result & ". Error saving content: " & errMsg
		
		-- Try to restore clipboard even on error
		try
			my setClipboard(savedClipboard)
		end try
	end try
	
	return result
end run

-- Helper function to get clipboard content
on getClipboard()
	set clipContent to ""
	try
		set clipContent to the clipboard as text
	on error
		-- If clipboard doesn't contain text
		set clipContent to ""
	end try
	return clipContent
end getClipboard

-- Helper function to set clipboard content
on setClipboard(content)
	set the clipboard to content
	return true
end setClipboard