require 'csv'
require 'nokogiri'
require 'open-uri'
require 'pry'

urls = File.read('urls.txt').split("\n")

CSV.open('scraped_data.csv', 'wb') do |csv|
  urls.each do |url|
    html = URI.open(url).read
    doc = Nokogiri::HTML(html)
    puts(url)
    median = doc.xpath("//*[contains(text(), 'Median')]").first.parent.children.last.children.last.to_s
    for_sale = doc.xpath("//*[contains(text(), 'For Sale')]")[1].parent.children.last.children.last.to_s

    csv << [url, median, for_sale]
  rescue StandardError
    csv << [url]
  end
end