script for a Myo device to control a presentation using own guestures

hci_learning.lua

scriptId = 'HCI Exercise - PowerPoint Connector - Extended Example'
scriptDetailsUrl = ''
scriptTitle = 'HCI - PowerPoint Connector - Trained'

-- -------------------------- HCI 2017 EXERCISE ----------------------
-- Lowis Engel, Christian Knabenhans, Florian Moser
-- ========================== COMMANDS ==========================
-- Fist -> Spread 		Enter training phase, center the arm
--		Spread				Begin new data acquisition
--		Fist				Go on to train next gesture
--		Fist 				Leave Training phase, after last gesture
-- Spread				Begin recording of gesture



myo.debug("debug")


myo.setLockingPolicy("none")

centreYaw = 0  
centreRoll = 0
centrePitch = 0

PI = 3.1416  
TWOPI = PI * 2
E = 2.71828

previous, next, first, last = 1, 2, 3, 4
no_train = 0
train_state = no_train
actions = {[1]=previous, [2]=next, [3]=first, [4]=last}

-- Store mean and variances, for each time step
-- 2D table of size: #actions * maxTimeIndex
meanYaw, meanRoll, meanPitch = {}, {}, {}
varYaw, varRoll, varPitch = {}, {}, {}

-- Container for training data
-- 2D tables of size: maxTimeIndex * trialIndex
dataYaw, dataRoll, dataPitch = {}, {}, {}
dataIndex = 1
trialIndex = 1

-- Stores the error, for each action, for each timestep
-- 1D tables of size: #actions * maxTimeIndex
errorYaw, errorRoll, errorPitch = {}, {}, {}

-- Stores total error over the current gesture time interval, for each action
-- 1D table of size: #actions
totalErr = {}

totalProb = {}

timeIndex = 1

intervalUnlock = 2000 --ms (time between first & second unlock gesture)

period = 100		--ms (multiple of 10)
maxTime = 1300	--ms
maxTimeIndex = 0

printCount = 0

function onForegroundWindowChange(app, title)
	local uppercaseApp = string.upper(app)
	return platform == "MacOS" and app == "com.microsoft.Powerpoint" or
	platform == "Windows" and (uppercaseApp == "POWERPNT.EXE" or uppercaseApp == "PPTVIEW.EXE")
end

function activeAppName()
	return "PowerPoint"
end

-- Effects
function doAction(a)
	if a == previous then
		myo.keyboard("up_arrow", "press")
	elseif a == next then
		myo.keyboard("down_arrow", "press")
	elseif a == first then
		myo.keyboard("home", "press")
	elseif a == last then
		myo.keyboard("end", "press")
	end
end

function insert1d(data, i, row)
	data[i] = row
end

function insert2d(data, i, j, x)
	if data[i] == nil then
		data[i] = {}
	end
	data[i][j] = x
end
	
function get2d(data, i, j)
	if data[i] ~= nil then
		if data[i][j] ~= nil then
			return data[i][j]
		else
			return 2
		end
	else
		return 1
	end
end
	
calledTimes = 0;
calledTimesUnlock = 0;  -- since first unlock gesture
-- Called every 10 milliseconds, as long as the script is active.
function onPeriodic()
	if (unlock_phase==1) then
		calledTimesUnlock = calledTimesUnlock + 1
		if (calledTimesUnlock > (intervalUnlock/10)) then
			unlock_phase = 0
			calledTimesUnlock = 0
			myo.debug("Please try unlocking again")
		end
	end
	
	if (calledTimes < (period/10)) then
		calledTimes = calledTimes + 1
		return
	end
	calledTimes = 0;

	if (centreYaw == 0) then
		return
	end
	
	local currentYaw = myo.getYaw()
	local currentRoll = myo.getRoll()
	local currentPitch = myo.getPitch()
	
	local deltaYaw = calculateDeltaRadians(currentYaw, centreYaw)
	local deltaRoll = calculateDeltaRadians(currentRoll, centreRoll)
	local deltaPitch = calculateDeltaRadians(currentPitch, centrePitch)
	
	
	if (train_state ~= no_train and record and dataIndex < maxTimeIndex) then
		insert2d(dataYaw, dataIndex, trialIndex, deltaYaw)
		insert2d(dataRoll, dataIndex, trialIndex, deltaRoll)
		insert2d(dataPitch, dataIndex, trialIndex, deltaPitch)
		dataIndex = dataIndex + 1
	elseif (train_state ~= no_train and record and dataIndex == maxTimeIndex) then
		myo.notifyUserAction()
		dataIndex = dataIndex + 1
	elseif train_state == no_train and timeIndex < maxTimeIndex then
		--local err = {} -- 1D table of size #actions
		local prob = {}
		for i, act in pairs(actions) do
		
			-- Compute error in Yaw, Roll, Pitch			
			-- insert2d(errorYaw, act, timeIndex, abs(deltaYaw - get2d(meanYaw, act, timeIndex) / get2d(varYaw, act, timeIndex)));
			-- insert2d(errorRoll, act, timeIndex, abs(deltaRoll - get2d(meanRoll, act, timeIndex) / get2d(varRoll, act, timeIndex)));
			-- insert2d(errorPitch, act, timeIndex, abs(deltaPitch - get2d(meanPitch, act, timeIndex) / get2d(varPitch, act, timeIndex)));
			
			-- Compute global error for this timestep, for each action
			-- err[act] = get2d(errorYaw, act, timeIndex)+get2d(errorRoll, act, timeIndex)+get2d(errorPitch, act, timeIndex);

			local mY, mR, mP = get2d(meanYaw, act, timeIndex), get2d(meanRoll, act, timeIndex), get2d(meanPitch, act, timeIndex)
			local vY, vR, vP = get2d(varYaw, act, timeIndex), get2d(varRoll, act, timeIndex), get2d(varPitch, act, timeIndex)
			local tY = 1 / (TWOPI*vY)
			local pY = tY*E^(-(deltaYaw-mY)^2/(2*vY))
			local tR = 1 / (TWOPI*vR)
			local pR = tR*E^(-(deltaRoll-mR)^2/(2*vR))
			local tP = 1 / (TWOPI*vP)
			local pP = tP*E^(-(deltaPitch-mP)^2/(2*vP))
			
			prob[act] = pY + pR + pP
			
			--[[
			myo.debug("++\n++\n++\n")
		
			myo.debug("yaw: " .. deltaYaw .. " roll: " .. deltaRoll .. " pitch: " .. deltaPitch)
			myo.debug("eRoll: " .. get2d(errorRoll, act, timeIndex))
			myo.debug("ePitch: " .. get2d(errorPitch, act, timeIndex))
		
			myo.debug("meanYaw: " .. get2d(meanYaw, act, timeIndex))
			myo.debug("meanRoll: " .. get2d(meanRoll, act, timeIndex))
			myo.debug("meanPitch: " .. get2d(meanPitch, act, timeIndex))
			
			myo.debug("varYaw = "..vY)
			myo.debug("varRoll = "..vR)
			myo.debug("varPitch = "..vP)
			
			myo.debug("eYaw: " .. get2d(errorYaw, act, timeIndex))
			myo.debug("eRoll: " .. get2d(errorRoll, act, timeIndex))
			myo.debug("ePitch: " .. get2d(errorPitch, act, timeIndex))
			--]]
					
			-- Compute the total error
			--[[
			if totalErr[act] == nil then
				totalErr[act] = err[act] 
			else 
				totalErr[act] = totalErr[act] + err[act] 
			end	
			--]]
			if totalProb[act] == nil then
				totalProb[act] = prob[act]
			else
				totalProb[act] = totalProb[act] + prob[act]
			end
		end
		timeIndex = timeIndex + 1
		
	elseif (timeIndex == maxTimeIndex) then
		myo.notifyUserAction()
		timeIndex = timeIndex + 1
		if (train_state == no_train) then
			local maxProb, bestAction = nil, nil
			myo.debug("Evaluating gesture...")
			local sum = ""
			for i, act in pairs(actions) do
				local currentProb = totalProb[act]
				sum = sum .. currentProb .."  "
				if (maxProb == nil) or (currentProb > maxProb) then
					maxProb = currentProb
					bestAction = act
					--myo.debug("new max prob = "..maxProb..", new best action = "..bestAction)
				end
			end
			-- myo.debug("changed gestures:"..changed)
			
			--myo.debug("++++++++++++++++++++++++++++++\n+++++++++++++++++++++++++++++")
			myo.debug("min error = "..maxProb..", best action = "..bestAction)
			myo.debug(sum)
			--[[
			myo.debug("++++++++++++++++++++++++++++++\n+++++++++++++++++++++++++++++")
			myo.debug("error per gesture")
			myo.debug("act1: " .. totalErr[1])
			myo.debug("act2: " .. totalErr[2])
			myo.debug("act3: " .. totalErr[3])
			myo.debug("act4: " .. totalErr[4])
			--]]
			doAction(bestAction)
		end -- Do not execute anything again until double tap
	end
	
	
	-- debug every 200 events
	printCount = printCount + 1
	if printCount >= 200 then
		--myo.debug("deltaYaw = " .. deltaYaw .. ", centreYaw = " .. centreYaw .. ", currentYaw = " .. currentYaw)
		--myo.debug("deltaRoll = " .. deltaRoll .. " currentRoll = " .. currentRoll)
		printCount = 0
	end
end



-- ================================= Utility =================================
function calculateDeltaRadians(current, centre)  
	local delta = current - centre

	if (delta > PI) then
		delta = delta - TWOPI
	elseif(delta < -PI) then
		delta = delta + TWOPI
	end
	return delta
end

unlock_phase = 0
unlocked = false
record = false
firstTrial = true
-- on fingersSpread center arm position
function onPoseEdge(pose, edge)  
	myo.debug("onPoseEdge: " .. pose .. ", " .. edge)
	if not initialized then
		init()
	end
	if (edge == "on" ) then
	
		-- Unlocking mechanism: Fist -> double tap
		-- Needed to begin training phase
		if train_state == no_train then
			if unlock_phase == 0  and pose == "fist" then
				unlock_phase = 1
			elseif unlock_phase == 1 and pose == "doubleTap" then
				unlock_phase = 0
				calledTimesUnlock = 0
				unlocked = true
				center()
				beginTraining()
			end
		end
		
		-- Training phase transitions
		-- Double Tap -> Begin recording for this gesture
		-- Fist -> Move to training phase for next gesture
		if unlocked then
			if pose == "fingersSpread" then
				-- Go to next trial for same gesture
				nextTrial()
			elseif pose == "fist" then 
				-- Go to next gesture training phase
				nextGesture()
			end
		else
			if pose == "fingersSpread" then
				timeIndex = 1
				myo.notifyUserAction()
			end
		end		
	end	
end

-- Helper functions
function beginTraining()
	firstTrial = true
	record = false
	train_state = previous
	myo.debug("\n==============================")
	myo.debug("Beginning training")
	myo.debug("Training gesture for >" .. "Previous Slide" .. "<. ")
end
function nextTrial()
	myo.notifyUserAction()
	myo.debug("\t.")
	dataIndex = 1
	if firstTrial then
		trialIndex = 1
		firstTrial = false
	else
		trialIndex = trialIndex + 1
	end
	record = true
end
function nextGesture()
	myo.notifyUserAction()
	myo.notifyUserAction()
	record = false
	firstTrial = true
	
	evaluateData()
	dataYaw, dataRoll, dataPitch = {}, {}, {}
	if train_state == previous then
		train_state = next
		myo.debug("Training gesture for >" .. "Next Slide" .. "<. ")
	elseif train_state == next then
		train_state = first
		myo.debug("Training gesture for >" .. "First Slide" .. "<. ")
	elseif train_state == first then
		train_state = last
		myo.debug("Training gesture for >" .. "Last Slide" .. "<. ")
	elseif train_state == last then
		endTraining()
	end
	
	
end
function endTraining() 
	train_state = no_train
	unlocked = false
	-- unlock_phase = 0
	-- record = false
	myo.debug("Training Ended")
	myo.debug("==============================\n")
	-- TODO: store means, variances to file for future use
end

-- Compute empirical mean and variance
function evaluateData()
	-- make a local copy of data
	local dY, dR, dP = copy(dataYaw), copy(dataRoll), copy(dataPitch)
	
	insert1d(meanYaw, train_state, computeMean(dY))
	insert1d(meanRoll, train_state, computeMean(dR))
	insert1d(meanPitch, train_state, computeMean(dP))
	
	insert1d(varYaw, train_state, computeVariance(dY, meanYaw[train_state]))
	insert1d(varRoll, train_state, computeVariance(dR, meanRoll[train_state]))
	insert1d(varPitch, train_state, computeVariance(dP, meanPitch[train_state]))
end

function copy(data)
	c = {}
	for i, row in pairs(data) do
		c[i] = {}
		for j, x in pairs(row) do
			c[i][j] = x
		end
	end
	return c
end

function computeMean(data) 
	local numData, numSamples = 0, 0
	local m = {}
	for dI, trials in pairs(data) do
		m[dI] = 0
		for tI, x in pairs(trials) do
			m[dI] = m[dI] + x
			numSamples = numSamples + 1
		end
		numData = numData + 1
	end
	numSamples = numSamples / numData
	for dI, sum in pairs(m) do
		m[dI] = sum / numSamples
	end
	return m
end

function computeVariance(data, mean)
	local numData, numSamples = 0, 0
	local v = {}
	for dI, trials in pairs(data) do
		v[dI] = 0
		if #trials == 1 then
			v[dI] = 0.002
		else
			for tI, x in pairs(trials) do
				v[dI] = v[dI] + (x - mean[dI])^2
				numSamples = numSamples + 1
			end
		end
		numData = numData + 1
	end
	numSamples = numSamples / numData
	if numSamples > 1 then
		for dI, sum in pairs(v) do
			v[dI] = sum / (numSamples - 1)
		end
	end
	return v
end

-- save current position of the arm
function center()  
	myo.debug("Centered")
	centreYaw = myo.getYaw()
	centreRoll = myo.getRoll()
	centrePitch = myo.getPitch()
	myo.vibrate("short")
end

function abs(x)
	if x < 0 then
		return -x
	else
		return x
	end
end

initialized = false
configPath = "./gestures.txt"

function init()
	local empty = {}
	for i, act in pairs(actions) do
		empty[act] = {}
		for k = 1 , maxTimeIndex, 1 do
			empty[act][k] = 0
		end
	end
	meanYaw, meanRoll, meanPitch = copy(empty), copy(empty), copy(empty)
	varYaw, varRoll, varPitch = copy(empty), copy(empty), copy(empty)
	maxTimeIndex = DIV(maxTime,period)
	initialized = true
	myo.debug("Initialization done")
end

function DIV(a,b)
    return (a - a % b) / b
end