Skip to content

Instantly share code, notes, and snippets.

@chaffeqa

chaffeqa/process_feed.rb

Created January 11, 2016 19:13
Show Gist options
  • Save chaffeqa/7d5da85042ce59c83a09 to your computer and use it in GitHub Desktop.
Save chaffeqa/7d5da85042ce59c83a09 to your computer and use it in GitHub Desktop.
Download ZIP
Feed changes
Raw
process_feed.rb
# peter tyrrell - 1/2016
################################################################
# ProcessFeed
#
# .parse
# ---------------------------------------------------------------
# This routine can accept four OPTIONAL parameters
# example calls:
# ProcessFeed.parse(sim_online_state:ProcessFeed::SIM_FORCE, sim_round:400, sim_state:ProcessFeed::SIM_IN_PROGRESS)
# ProcessFeed.parse(feed_uri:'https://feeds.pgatourhq.com/DDManager/DDClient.do?clientFile=SyndFullLeaderboard&action=Request&u=sportstechinc&p=Z17t0vF4!!&T_ID=R2015470&R_NUM=1', sim_online_state:ProcessFeed::SIM_FORCE)
#
# -----> sim_online_state
# can either be
# 'online' <= default - in which the feed will be pulled, determine if online is newer than DB and parse/populate if so
# 'force' - in which the script will attempt to get the data from the feed regardless of the feed's age
# 'offline' - the parser will pull from offline_feed.xml if present. This allows for more rigorous testing and modification of the feed
#
# -----> sim_round
# allows you set the round to whatever you like for scoring sim i.e. ProcessFeed.parse('offline', 400)
# keep in mind, that the round is using an artificial numbering system -- where 1 = 100, 2=200, 3=300, 301=301, 4=400 ...
#
# -----> sim_state
# can either be
# 'Official' <= default - use the final score and declare a winner
# 'In progress'- treat the round as if it was in progress (regardless of actual state), use artificial scoring and do not mark winner
#
# -----> feed_uri
# location of feed for PGA scoring sims
#
#
#
# .save_for_offline [depends on feed_uri constant or submitted as param]
# ---------------------------------------------------------------
# This routine grabs the latest feed and exports it into app/services/offline_feed.xml. This is useful for edge case testing
# and is paired with ProcessFeed.parse('offline')
#
#
#
#
# .populate_teams [depends on teams_feed_uri constant or submitted as param]
# ---------------------------------------------------------------
# This routine uses an alternate player feed to generate the teams table
#
################################################################
require 'nokogiri' # nokogiri for parsing
require 'net/http'
require 'json'
require 'open-uri'
class ProcessFeed
MY_DEBUG = true
# -- artificial round number as in 100,200,300,301,400,500,600,700
SIM_INCLUDE_ALL_ROUNDS = 1000
# -- a constant representing one of the following conditions
SIM_ONLINE = 'online' # possible parameters 'forced' - forces reload even if not changed, 'offline' if feed broken or want to use file
SIM_FORCE = 'forced'
# -- various states that the game/round can exist in
SIM_NONE = 'none' # no change to existing state
SIM_DONE = 'Official'
SIM_IN_PROGRESS = 'In progress'
LOCATION_OF_OFFLINE_XML = 'app/services/offline_feed.xml'
# received from PGA - no further documentation on various parameters
FEED_URI = 'https://feeds.pgatourhq.com/DDManager/DDClient.do?clientFile=SyndFullLeaderboard&action=Request&u=sportstechinc&p=Z17t0vF4!!&T_ID=R2015470&R_NUM=1'
TEAMS_FEED_URI = 'https://feeds.pgatourhq.com/DDManager/DDClient.do?clientFile=ConsSetup&action=Request&u=sportstechinc&p=Z17t0vF4!!&T_ID=R2015470&E_ID=1'
def self.save_for_offline(feed_uri:FEED_URI)
IO.copy_stream(feed_uri, LOCATION_OF_OFFLINE_XML)
end
def self.parse(sim_online_state:SIM_ONLINE, sim_round:SIM_INCLUDE_ALL_ROUNDS, sim_state:SIM_NONE, feed_uri:FEED_URI)
# mapping of seed to group match position
# following bracket setup as follows
#############################################################################################
#### s01/g101 m101,t m201,t m203,t m105,t s02/g109
#### s16/g102 m101,b m301,t m302,t m105,b s15/g110
####
#### s09/g103 m102,t m201,b 501 m203,b m106,t s10,g111
#### s08/g104 m102,b m401,t m401,b m106,b s07,g112
#### 502(3rd pl)
#### s05/g105 m103,t m202,t m204,t m107,t s06,g113
#### s12/g106 m103,b m301,b m302,b m107,b s11,g114
####
#### s13/g107 m104,t m202,b m205,b m108,t s14,g115
#### s04/g108 m104,b m108,b s03,g116
#############################################################################################
round_1_mapping = {
's1' => 'g_101',
's2' => 'g_109',
's3' => 'g_116',
's4' => 'g_108',
's5' => 'g_105',
's6' => 'g_113',
's7' => 'g_112',
's8' => 'g_104',
's9' => 'g_103',
's10' => 'g_111',
's11' => 'g_114',
's12' => 'g_106',
's13' => 'g_107',
's14' => 'g_115',
's15' => 'g_110',
's16' => 'g_102'
}
round_4_mapping = {
'g_101' => 'm_201',
'g_102' => 'm_201',
'g_103' => 'm_202',
'g_104' => 'm_202',
'g_105' => 'm_203',
'g_106' => 'm_203',
'g_107' => 'm_204',
'g_108' => 'm_204',
'g_109' => 'm_205',
'g_110' => 'm_205',
'g_111' => 'm_206',
'g_112' => 'm_206',
'g_113' => 'm_207',
'g_114' => 'm_207',
'g_115' => 'm_208',
'g_116' => 'm_208'
}
existing_sti_mapping = ''
# in case the feed is broken -- or the developer wants to do some deep synthetic testing, okay to use offline mode ProcessFeed.parse('offline')
if sim_online_state == 'online'
xml = Net::HTTP.get_response(URI.parse(feed_uri)).body
read_xml = Nokogiri::XML(xml)
else
begin
read_xml = File.open(LOCATION_OF_OFFLINE_XML) { |f| Nokogiri::XML(f) }
puts "---------- OFFLINE MODE --------------------" if MY_DEBUG
rescue => e
puts "---------- CREATING OFFLINE FEED --------------------"
save_for_offline
read_xml = File.open(LOCATION_OF_OFFLINE_XML) { |f| Nokogiri::XML(f) }
end
end
#Read in the Feed from the PGA
read_xml.search('Tourn').each do |tdata|
tournament = {
tournament_id: tdata.attr('T_ID'),
last_scored: tdata.attr('CurTime'),
name: tdata.attr('Name'),
front_cur_rnd: tdata.attr('CurRnd').to_i,
cur_rnd_state: tdata.attr('CurRndState'),
local_city: tdata.attr('localCity')
}
##############
# Very important: this exits the entire ProcessFeed.parse if the feed does not report that it has changed
# unless the manual 'offline' override is supplied as a parameter
#
m = Wcbracket::TournamentStatus.first
if m
if tournament[:last_scored] == m.last_scored and sim_online_state != SIM_FORCE
puts 'The feed has not changed - no action'
return
end
end
#
#
#############
if sim_round != SIM_INCLUDE_ALL_ROUNDS
#adjust rounds for back_end & front end game.json
case sim_round
when 100, 200, 300
tournament[:front_cur_rnd] = 1
tournament[:back_cur_rnd] = sim_round
when 301
tournament[:front_cur_rnd] = 1
tournament[:back_cur_rnd] = 301
else # 400,500,600,700
tournament[:back_cur_rnd] = sim_round
tournament[:front_cur_rnd] = (sim_round - 200)/100
end
else
#adjust rounds for back_end & front end game.json
case tournament[:front_cur_rnd]
when 1, 2, 3
tournament[:front_cur_rnd] = 1
tournament[:back_cur_rnd] = tournament[:front_cur_rnd] * 100
when 301
tournament[:front_cur_rnd] = 1
tournament[:back_cur_rnd] = 301
else # 4,5,6,7
tournament[:back_cur_rnd] = tournament[:front_cur_rnd] * 100
tournament[:front_cur_rnd] = tournament[:front_cur_rnd] -2
end
end
puts tournament.inspect if MY_DEBUG
m = Wcbracket::TournamentStatus.find_or_initialize_by(tournament_id: tournament[:tournament_id])
m.assign_attributes(tournament)
m.save if m.changed?
end
puts sim_round if MY_DEBUG
#### We will need to update the seed the first time this feed is live -- other times will be redundant
=begin
all_teams = Wcbracket::Team
=end
# sift through the XML and grab latest information
read_xml.search('Rounds Round').each do |round_data|
#round = []
round_data.search('Bracket').each do |bracket_data|
#bracket = []
bracket_data.search('Group').each do |group_data|
#group = []
group_data.search('Player').each do |player_data|
player = {
:round_num => round_data.attr('RoundNum').to_i,
# required to determine the matchup position for group stage
:pool_match_pos=> group_data.attr('PoolMatchPosition').to_i,
#:pool => group_data.attr('PoolNum').to_i, #useless value -- bracket based on seeding not pool number
# pool_winner is always 'f' on rounds 1 and 2
# on rounds 3 & 301, pool_winner indicates the player who will advance
# and always n/a on 4,5,6,7
:pool_winner => player_data.attr('PoolWinner').to_s,
#match_winner represents the current game [on that particular day] -- rounds 1,2,3,301,4,5,6,7
#on rounds 4,5,6,7 -- it represents the player that will advance
:match_winner => player_data.attr('MatchWinner').to_s,
:pool_wins => player_data.attr('PoolWins').to_i,
:pool_losses => player_data.attr('PoolLosses').to_i,
:pool_ties => player_data.attr('PoolTies').to_i, #TODO change to actual name -- added as place holder
:seed => player_data.attr('Seed').to_i,
:player_id => player_data.attr('PID').to_i,
:scoreboard_name => player_data.attr('FreezeName').to_s.strip,
:first => player_data.attr('Fname').to_s.strip,
:last => player_data.attr('Lname').to_s.strip
}
=begin
if player[:round_num] == 1
# this will update the seeds in the Wcbracket::Team table once per ProcessFeed run
add_seed = {
:id => player[:player_id],
:seed => player[:seed]
}
k = all_teams.find_or_initialize_by(id: add_seed[:player_id])
k.assign_attributes(add_seed)
k.save if k.changed?
end
=end
### There are too many conditions that occur at rounds 4,5,6,7
# that do not occur at rounds 1,2,3, or 301 - because of this, we multiply by 100 except for 301
if player[:round_num] != 301
player[:round_num] = player[:round_num] * 100
end
# at this point we will have round 100,200,300,301,400,500,600,700
###
if player[:round_num] > sim_round #ignore all future logic and destroy row if exists
# SIM ROUND REMOVE PREVIOUS DATA
m = Wcbracket::MatchMember.find_by(player_id: player[:player_id], round_num: player[:round_num])
if m
m.destroy
end
else
all_match_groups = Wcbracket::MatchMember
# PROCESS AS NORMAL - NON-SIM ROUND
# Determine score
# -- this can either come from FinalMatchScr in cases where match has ended
# or from a concatenation of 'CurMatchScr' and 'Thru'
if player_data.attr('Thru') == 'F'
player[:score] = player_data.attr('FinalMatchScr').to_s
else
player[:score] = player_data.attr('CurMatchScr').to_s + player_data.attr('Thru').to_s
end
#override score for sim round for the round the sim_round specifies
if sim_round == player[:round_num]
if sim_state == SIM_IN_PROGRESS
player[:score] = player_data.attr('CurMatchScr').to_s + ' thru 12' # + player_data.attr('Thru').to_s
player[:advancing] = ''
else
player[:score] = player_data.attr('FinalMatchScr').to_s
end
end
### LOCATE WHICH ROUND THE PERSON CAME FROM AND THEN CALCULATE WHICH STI_MATCH THEY ARE IN
### BEGIN CLEANUP OF USELESS/MISLEADING COLUMNS, ALSO DETERMINE PLAYS_INTO
#
#
# on round 100 only where seed < 17, map to new sti_mapping according to round_1_mapping
#
# Notes on cleanup:
#
# rounds 300 and 301 are determined by whether player won pool
# rounds 400 thru 700 determined by whether player won match
#
# Notes on plays into
# rounds 100,200,301 play into the same g_1xx
# round 300 plays into m_1xx, round 400 into m_2xx, round 500 into m3xx
# round 600 into m_301 winner, m_302 loser
puts " ----------------------------------- #{player[:round_num]}" if MY_DEBUG
previous_match = all_match_groups.find_by('player_id = ? and round_num = ?', player[:player_id], player[:round_num] - 100)
case player[:round_num]
when 100
if player[:seed] < 17
# this will re-initialize the sti_mapping value every time it encounters a new seed <17
m = 's' + player[:seed].to_s.strip
player[:sti_match] = round_1_mapping[m].strip
existing_sti_mapping = round_1_mapping[m].strip
puts "=======================" + existing_sti_mapping.inspect if MY_DEBUG
else
player[:sti_match] = existing_sti_mapping
end
player.delete(:pool_winner)
#player[:winner_plays_into] = player[:sti_match]
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
player[:adj_round] = 1
when 200
#puts 'ROUND 200 -------------------'
player[:sti_match] = all_match_groups.find_by('player_id = ? and round_num = ?', player[:player_id], 100).sti_match
player.delete(:pool_winner)
#player[:winner_plays_into] = player[:sti_match]
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
player[:adj_round] = 1
when 300
#puts 'ROUND 300 -------------------'
player[:sti_match] = all_match_groups.find_by('player_id = ? and round_num = ?', player[:player_id], 100).sti_match
if player[:pool_winner] == 'Yes'
player[:advancing] = 't'
end
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
player[:adj_round] = 1
player[:sti_match].last(3).to_i.even? ? player[:plays_to_t_or_b] = 'b' : player[:plays_to_t_or_b] = 't'
previous_match.sti_match.last(3).to_i.even? ? player[:match_pos_t_or_b] = 'b' : player[:match_pos_t_or_b] = 't'
puts '=====================> Winner ===============> ' + player[:winner_plays_into].inspect + '=====================> ' + player[:match_pos_t_or_b].inspect if MY_DEBUG
when 301
player[:sti_match] = all_match_groups.find_by('player_id = ? and round_num = ?', player[:player_id], 100).sti_match
player.delete(:score)
player.delete(:match_winner)
player.delete(:pool_wins)
player.delete(:pool_losses)
player.delete(:pool_ties)
player.delete(:plays_to_t_or_b)
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
#puts 'ROUND 301 -------------------' + match_num.inspect
player[:adj_round] = 1
when 400
#puts 'ROUND 400 -------------------'
player[:sti_match] = round_4_mapping[previous_match.sti_match]
player.delete(:pool_wins)
player.delete(:pool_losses)
player.delete(:pool_ties)
player.delete(:plays_to_t_or_b)
player.delete(:pool)
player.delete(:pool_winner)
if player[:match_winner] == 'Yes'
player[:advancing] = 't'
end
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
player[:adj_round] = 2
player[:sti_match].last(3).to_i.even? ? player[:plays_to_t_or_b] = 'b' : player[:plays_to_t_or_b] = 't'
previous_match.sti_match.last(3).to_i.even? ? player[:match_pos_t_or_b] = 'b' : player[:match_pos_t_or_b] = 't'
puts '=====================> Winner ===============> ' + player[:winner_plays_into].inspect + '=====================> ' + player[:match_pos_t_or_b].inspect if MY_DEBUG
when 500, 600
# puts 'ROUND -------------------' + player[:round_num].inspect if MY_DEBUG
player[:sti_match] = previous_match.winner_plays_into
player.delete(:pool_wins)
player.delete(:pool_losses)
player.delete(:pool_ties)
player.delete(:plays_to_t_or_b)
player.delete(:pool)
player.delete(:pool_winner)
if player[:match_winner] == 'Yes'
player[:advancing] = 't'
end
i_current_match = player[:sti_match].last(3).to_i
r = (i_current_match / 100).to_i
m = i_current_match % 100
match_num = (r + 1) * 100 + ((m + 1) / 2).floor
player[:winner_plays_into] = 'm_' + match_num.to_s
if player[:round_num] == 600
player[:loser_plays_into] = 'm_' + match_num.to_s.first(2) + '2'
end
player[:sti_match].last(3).to_i.even? ? player[:plays_to_t_or_b] = 'b' : player[:plays_to_t_or_b] = 't'
previous_match.sti_match.last(3).to_i.even? ? player[:match_pos_t_or_b] = 'b' : player[:match_pos_t_or_b] = 't'
puts '=====================> Winner ===============> ' + player[:winner_plays_into].inspect + '=====================> ' + player[:match_pos_t_or_b].inspect if MY_DEBUG
player[:adj_round] = (player[:round_num] - 200) / 100
when 700
if previous_match.advancing == 't'
player[:sti_match] = 'm_501'
previous_match.sti_match.last(3).to_i.even? ? player[:match_pos_t_or_b] = 'b' : player[:match_pos_t_or_b] = 't'
else
player[:sti_match] = 'm_502'
previous_match.sti_match.last(3).to_i.even? ? player[:match_pos_t_or_b] = 'b' : player[:match_pos_t_or_b] = 't'
end
player.delete(:pool_wins)
player.delete(:pool_losses)
player.delete(:pool_ties)
player.delete(:plays_to_t_or_b)
player.delete(:pool)
player.delete(:pool_winner)
if player[:match_winner] == 'Yes'
player[:advancing] = 't'
end
player[:adj_round] = 5
# player[:match_pos_t_or_b] = previous_match.plays_to_t_or_b
end
#
#
########## END LOCATING SPECIFIC MATCH PLAYER IS IN
########## END CLEAN UP AND PLAYS INTO
### Make values less pretty, but more consistent -- replacing Yes/No with t/f
#put boolean t/f into columns replacing Yes/No
if player[:pool_winner] == 'No'
player[:pool_winner] = 'f'
end
if player[:match_winner] == 'No'
player[:match_winner] = 'f'
end
if player[:pool_winner] == 'Yes'
player[:pool_winner] = 't'
end
if player[:match_winner] == 'Yes'
player[:match_winner] = 't'
end
puts player.inspect #if MY_DEBUG
# Add value to DB (if changed)
m = all_match_groups.find_or_initialize_by(player_id: player[:player_id], round_num: player[:round_num])
m.assign_attributes(player)
m.save if m.changed?
end
end
end
end
end
puts 'Feed complete - updating legacy table' if MY_DEBUG
###########################
# This area below re-populates the legacy table "Wcbracket::Matchup" with current information
# not elegant, but insures other processes dependant on that table will operate as expected
#
back_cur_rnd = Wcbracket::TournamentStatus.select(:back_cur_rnd).first.back_cur_rnd
##############
# DB requests
matchups = Wcbracket::Matchup.ordered.to_a
Wcbracket::MatchMember.select(:round_num)
.distinct
.each { |rn|
Wcbracket::MatchMember.select(:sti_match)
.distinct
.where(round_num: rn.round_num)
.each { |x|
####################################################
# Begin codification of which matches are
# included in game.json
#
# The following matrix references how the
# logic operates:
#
# round 100 200 300 301 400 500 600 700
#
# data
# 100 Y - - - - - - -
# 200 - Y - - - - - -
# 300 - - Y Y Y Y Y Y
# 301 - - - - - - - -
# 400 - - - - Y Y Y Y
# 500 - - - - - Y Y Y
# 600 - - - - - - Y Y
# 700 - - - - - - - Y
#
#
include_in_json = false
if back_cur_rnd == rn.round_num and rn.round_num != 301
include_in_json = true
end
if back_cur_rnd > 301
if rn.round_num == 301 or rn.round_num < 300
include_in_json = false
else
include_in_json = true
end
end
# exception handler for 301:
if back_cur_rnd == 301 and rn.round_num == 300
include_in_json = true
end
#
#
####################################################
# Logic for propogation
#
# The Matchup class is used to score entries
# All changes to that table are reflected in scoring
# Up to this point, alterations HAD NOT touched the bracket
####################################################
if include_in_json
specific_match = Wcbracket::MatchMember.where(sti_match: x.sti_match, round_num: rn.round_num).order(advancing: :desc).first
# Find the Matchup
matchup = matchups.detect {|m| m.uid == x.sti_match}
# Or build one if it doesnt exist!
if matchup.nil?
matchup = Wcbracket::Matchup.new(
uid: x.sti_match,
plays_in_to: specific_match.winner_plays_into,
loser_in_to: specific_match.loser_plays_into,
t_or_b: specific_match.plays_to_t_or_b,
round: specific_match.adj_round
)
end
# update the winner_id
if specific_match.advancing == 't'
matchup.winner_id = specific_match.player_id
end
# update the teams
group_team_ids = []
Wcbracket::MatchMember.where(sti_match: x.sti_match, round_num: rn.round_num).order(:seed).each_with_index { |ma, index|
if specific_match.adj_round > 1
if ma.match_pos_t_or_b == 't'
matchup.team_t = ma.player_id
else
matchup.team_b = ma.player_id
end
else
group_team_ids << ma.player_id.to_i
end
}
# make sure we dont update the ordering of the group_stage_teams, since we want to keep whatever
# we hard set as the ordering onf the frontend
previous_group_stage_team_ids = matchup.group_stage_team_ids.to_s.split(",").map(&:to_i)
if specific_match.adj_round == 1 and previous_group_stage_team_ids.sort != group_team_ids.sort
matchup.group_stage_team_ids = group_team_ids.join(",")
end
# persist to the db
matchup.save! if matchup.changed?
end
}
}
puts 'Feed parsed and loaded'
# Write to the bottom of the console to remind developer that we are in a simulation and not using the actual feed
if MY_DEBUG and !sim_online_state
puts "\n\n---------- OFFLINE MODE --------------------\n\n"
end
m = Wcbracket::TournamentStatus.first
return "Backend round set to: <b> #{m.back_cur_rnd}</b><br>Frontend round equivalent is:<b> #{m.front_cur_rnd}</b> <br> State of game is: <b>#{sim_state}</b>"
end
def self.create_teams(teams_feed_uri:TEAMS_FEED_URI)
xml = Net::HTTP.get_response(URI.parse(teams_feed_uri)).body
read_xml = Nokogiri::XML(xml)
all_teams = Wcbracket::Team
read_xml.search('field plr').each do |team_data|
player = {
:id => team_data.attr('id').to_i,
:name => team_data.xpath('name').attr('ShortName').to_s + '. ' + team_data.xpath('name').attr('last').to_s,
:abbrev => team_data.xpath('name').attr('freeze').to_s,
:flag => team_data.xpath('info').attr('country').to_s
#still need
#:seed will need to come from alternate feed
#:is_eliminated -- it is okay to set this as null?
}
puts player if MY_DEBUG
# Add value to DB (if changed)
k = all_teams.find_or_initialize_by(id: player[:id])
k.assign_attributes(player)
k.save if k.changed?
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment