EoS Scraper

gistfile1.js

var Dict = function(obj) {
    this.items = {};
    if (typeof obj != 'undefined')
        this.setFromObj(obj);
};
Dict.prototype.setFromObj = function(obj) {
    for (var key in obj)
        if (Object.prototype.hasOwnProperty.call(obj, key))
            this.set(key, obj[key]);
};
Dict.prototype.has = function(key) {
    if (Object.prototype.hasOwnProperty.call(this.items, key))
        return true;
}
Dict.prototype.get = function(key) {
    if (this.has(key))
        return this.items[key];
};
Dict.prototype.delete = function(key) {
    var item = this.get(key);
    delete this.items[key];
    return item;
};
Dict.prototype.set = function(key, value) {
    this.items[key] = value;
    return value;
};
Dict.prototype.setDefault = function(key, value) {
    if (this.has(key))
        return this.items[key];
    return this.set(key, value);
};
Dict.prototype.toJSON = function() {
    return this.items;
};

var productDb = new Dict({
"Action Figure":  {basePrice: 1200},
"Air Conditioner":  {basePrice: 100000},
"Almonds":  {basePrice: 200},
"Aluminum":  {basePrice: 3000},
"Apple":  {basePrice: 50},
"Apple Cider":  {basePrice: 300},
"Apple Juice":  {basePrice: 250},
"Apple Juice Concentrate":  {basePrice: 125, nostore: true},
"Apple Pie":  {basePrice: 500},
"Apple Sorbet":  {basePrice: 150},
"Aquarium":  {basePrice: 15000},
"Artificial Leather":  {basePrice: 1000},
"Artificial Vanilla Extract":  {basePrice: 200},
"Athletic Jacket":  {basePrice: 2500},
"Bacon":  {basePrice: 1000},
"Balloons":  {basePrice: 250},
"Banana":  {basePrice: 50},
"Banana Chips":  {basePrice: 400},
"Banana Cream Pie":  {basePrice: 600},
"Banana Juice Concentrate":  {basePrice: 125, nostore: true},
"Banana Sorbet":  {basePrice: 150},
"Banana Split":  {basePrice: 600},
"Barley":  {basePrice: 50},
"Baseball":  {basePrice: 2500},
"Baseball Bat":  {basePrice: 6500},
"Basketball":  {basePrice: 1500},
"Batter":  {basePrice: 100},
"Bauxite":  {basePrice: 1500, nostore: true},
"Bed":  {basePrice: 35000},
"Beef (Roast)":  {basePrice: 500},
"Beef (Steak)":  {basePrice: 800},
"Beef Patty":  {basePrice: 700},
"Beef Sausage":  {basePrice: 250},
"Beef Snack Sticks":  {basePrice: 1250},
"Beer":  {basePrice: 300},
"Belt":  {basePrice: 1800},
"Bicycle":  {basePrice: 10000},
"Bicycle Helmet":  {basePrice: 2000},
"Black Licorice":  {basePrice: 250},
"Blanket":  {basePrice: 1500},
"Blueberry":  {basePrice: 100},
"Blueberry Cheesecake":  {basePrice: 600},
"Blueberry Muffin":  {basePrice: 200},
"Blueberry Pie":  {basePrice: 500},
"Bonsai":  {basePrice: 1000},
"Bonsai Plant":  {basePrice: 300},
"Bonsai Pot":  {basePrice: 300},
"Bottled Cappuccino":  {basePrice: 400},
"Bottled Water":  {basePrice: 200},
"Bow Set":  {basePrice: 2500},
"Bow Tie":  {basePrice: 1300},
"Bread":  {basePrice: 200},
"Bread Roll":  {basePrice: 100},
"Breakfast Cereal":  {basePrice: 250},
"Broccoli":  {basePrice: 50},
"Buffalo Wings":  {basePrice: 800},
"Buns":  {basePrice: 200},
"Business Jet":  {basePrice: 200000000},
"Business Jet Body":  {basePrice: 90000000, nostore: true},
"Business Jet Wing":  {basePrice: 25000000, nostore: true},
"Butter":  {basePrice: 200},
"Caffeine":  {basePrice: 50},
"Can":  {basePrice: 5},
"Candy":  {basePrice: 100},
"Canned Cherries":  {basePrice: 250},
"Canned Coffee":  {basePrice: 200},
"Canned Mango":  {basePrice: 250},
"Canned Oranges":  {basePrice: 250},
"Canned Pineapple":  {basePrice: 250},
"Cap":  {basePrice: 1200},
"Cappuccino":  {basePrice: 200},
"Car":  {basePrice: 1150000},
"Car Body":  {basePrice: 650000},
"Caramel":  {basePrice: 100},
"Caramel Candy":  {basePrice: 300},
"Caramel Ice Cream":  {basePrice: 250},
"Carbon Fiber":  {basePrice: 20000},
"Carpet":  {basePrice: 1000},
"Carrot":  {basePrice: 50},
"Casual Dress":  {basePrice: 3000},
"Cattle":  {basePrice: 80000, nostore: true},
"Cement":  {basePrice: 500},
"Ceramic Pot":  {basePrice: 500},
"Ceramic Tile":  {basePrice: 1000},
"CFRP":  {basePrice: 30000},
"Chair":  {basePrice: 3000},
"Charcoal":  {basePrice: 600, nostore: true},
"Cheese":  {basePrice: 100},
"Cheeseburger":  {basePrice: 300},
"Chemicals":  {basePrice: 100},
"Cherry":  {basePrice: 50},
"Cherry Cheesecake":  {basePrice: 600},
"Cherry Cola":  {basePrice: 300},
"Cherry Cola Concentrate":  {basePrice: 150, nostore: true},
"Cherry Gelatin":  {basePrice: 350},
"Cherry Juice Concentrate":  {basePrice: 125, nostore: true},
"Cherry Sorbet":  {basePrice: 150},
"Chicken":  {basePrice: 600, nostore: true},
"Chicken Breast":  {basePrice: 450},
"Chicken Breast Patty":  {basePrice: 750},
"Chicken Burger":  {basePrice: 260},
"Chicken Drumsticks":  {basePrice: 150},
"Chicken Nuggets":  {basePrice: 1000},
"Chicken Wings":  {basePrice: 150},
"Chocolate Bars":  {basePrice: 500},
"Chocolate Cake":  {basePrice: 1250},
"Chocolate Chip Cookies":  {basePrice: 250},
"Chocolate Chips":  {basePrice: 100},
"Chocolate Ice Cream":  {basePrice: 250},
"Chocolate Milk":  {basePrice: 300},
"Chocolate Truffles":  {basePrice: 700},
"Clay":  {basePrice: 500, nostore: true},
"Clock":  {basePrice: 1000, nostore: true},
"Coal":  {basePrice: 800, nostore: true},
"Cocoa Beans":  {basePrice: 100},
"Cocoa Butter":  {basePrice: 200},
"Cocoa Powder":  {basePrice: 100},
"Coconut":  {basePrice: 200},
"Coconut Juice":  {basePrice: 250},
"Coconut Juice Concentrate":  {basePrice: 125, nostore: true},
"Coconut Sorbet":  {basePrice: 150},
"Coffee Beans":  {basePrice: 50},
"Cola":  {basePrice: 300},
"Cola Concentrate":  {basePrice: 150, nostore: true},
"Comforter":  {basePrice: 4000},
"Common Beans":  {basePrice: 50},
"Conventional Oven":  {basePrice: 35000},
"Cookie Dough":  {basePrice: 100},
"Cookies N Cream Ice Cream":  {basePrice: 250},
"Cooking Oil":  {basePrice: 500},
"Copper":  {basePrice: 8000},
"Copper Ore":  {basePrice: 4000, nostore: true},
"Corn":  {basePrice: 50},
"Corn Flakes":  {basePrice: 250},
"Cotton":  {basePrice: 200},
"Cotton Thread":  {basePrice: 500},
"Country Fried Steak":  {basePrice: 750},
"CPU":  {basePrice: 5000},
"Cream":  {basePrice: 100},
"Cruiser Body":  {basePrice: 10000000, nostore: true},
"Crystal Earrings":  {basePrice: 10000},
"Crystal Pendant":  {basePrice: 15000},
"Designer Belt":  {basePrice: 18000},
"Designer Dress":  {basePrice: 120000},
"Designer Hand Bag":  {basePrice: 100000},
"Designer Shoes":  {basePrice: 80000},
"Desktop Computer":  {basePrice: 50000},
"Diamond":  {basePrice: 500000, nostore: true},
"Diamond Earrings":  {basePrice: 50000},
"Diamond Necklace":  {basePrice: 100000},
"Diamond Ring":  {basePrice: 150000},
"Diamond Watch":  {basePrice: 150000},
"Digital Camera":  {basePrice: 30000},
"Dog House":  {basePrice: 3000},
"Doll":  {basePrice: 1200},
"Donut":  {basePrice: 100},
"Dress Shirt":  {basePrice: 4000},
"Dress Shoes":  {basePrice: 6000},
"Dress Suit":  {basePrice: 10000},
"Dresser":  {basePrice: 12500},
"Dryer":  {basePrice: 50000},
"DVD Player":  {basePrice: 5000},
"Egg":  {basePrice: 25},
"Electricity":  {basePrice: 10, nostore: true},
"Energy Drink":  {basePrice: 300},
"Engine":  {basePrice: 250000, nostore: true},
"Espresso":  {basePrice: 150},
"Executive Chair":  {basePrice: 6000},
"Fan":  {basePrice: 2000},
"Feather":  {basePrice: 1000, nostore: true},
"Fiberglass":  {basePrice: 600},
"Fishing Rod":  {basePrice: 2000},
"Flax Seeds":  {basePrice: 300},
"Flight Management System":  {basePrice: 20000000, nostore: true},
"Floor Lamp":  {basePrice: 2000},
"Flour":  {basePrice: 100},
"Football":  {basePrice: 1500},
"Fried Drumsticks":  {basePrice: 650},
"Frozen Beef":  {basePrice: 700},
"Frozen Chicken":  {basePrice: 1250},
"Frozen Pizza":  {basePrice: 420},
"Frozen Pork":  {basePrice: 600},
"Fruit Salad":  {basePrice: 400},
"Fur":  {basePrice: 10000, nostore: true},
"Fur Coat":  {basePrice: 25000},
"Fur Hat":  {basePrice: 10000},
"Gaming Console":  {basePrice: 20000},
"Gasoline":  {basePrice: 1000},
"Gelatin":  {basePrice: 100},
"Glass":  {basePrice: 100},
"Glass Bottle":  {basePrice: 20},
"Gloves":  {basePrice: 500},
"Gold":  {basePrice: 30000},
"Gold Bracelet":  {basePrice: 60000},
"Gold Necklace":  {basePrice: 20000},
"Gold Ore":  {basePrice: 20000, nostore: true},
"Gold Watch":  {basePrice: 40000},
"Golf Ball":  {basePrice: 90},
"Golf Club Set":  {basePrice: 15000},
"Golf Gloves":  {basePrice: 1200},
"Golf Stand Bag":  {basePrice: 4000},
"Gown":  {basePrice: 7000},
"Granola Bar":  {basePrice: 300},
"Grape Juice":  {basePrice: 250},
"Grape Juice Concentrate":  {basePrice: 125, nostore: true},
"Grape Sorbet":  {basePrice: 150},
"Grapes":  {basePrice: 100},
"Grilled Chicken Breast":  {basePrice: 900},
"Grilled Steak":  {basePrice: 1000},
"Ground Beef":  {basePrice: 350},
"Ground Pork":  {basePrice: 350},
"Halite":  {basePrice: 100, nostore: true},
"Ham":  {basePrice: 2000},
"Ham Sandwich":  {basePrice: 250},
"Hamburger":  {basePrice: 250},
"Hand Bag":  {basePrice: 3000},
"Hat":  {basePrice: 1800},
"Helicopter":  {basePrice: 80000000},
"Helicopter Body":  {basePrice: 50000000, nostore: true},
"Helicopter Rotor":  {basePrice: 1000000, nostore: true},
"Honey":  {basePrice: 200},
"Hot Chocolate":  {basePrice: 270},
"Hot Dog":  {basePrice: 200},
"Integrated Circuit":  {basePrice: 1000},
"Iron Ore":  {basePrice: 300, nostore: true},
"Jeans":  {basePrice: 4000},
"Jet Ski":  {basePrice: 900000},
"Jet Ski Body":  {basePrice: 120000, nostore: true},
"Khaki Pants":  {basePrice: 3000},
"Laptop Computer":  {basePrice: 60000},
"Leather":  {basePrice: 8000, nostore: true},
"Leather Gloves":  {basePrice: 2000},
"Leather Jacket":  {basePrice: 18000},
"Lemon":  {basePrice: 50},
"Lemon Gelatin":  {basePrice: 350},
"Lemon Juice Concentrate":  {basePrice: 125, nostore: true},
"Lemon Pie":  {basePrice: 500},
"Lemon Sorbet":  {basePrice: 150},
"Lemon-Lime":  {basePrice: 300},
"Lemon-Lime Concentrate":  {basePrice: 150, nostore: true},
"Lemonade":  {basePrice: 250},
"Lettuce":  {basePrice: 50},
"Licorice Roots":  {basePrice: 100},
"Light Bulb":  {basePrice: 100},
"Lime":  {basePrice: 50},
"Lime Gelatin":  {basePrice: 350},
"Lime Juice Concentrate":  {basePrice: 125, nostore: true},
"Lime Sorbet":  {basePrice: 150},
"Limeade":  {basePrice: 250},
"Limestone":  {basePrice: 150, nostore: true},
"Linen Thread":  {basePrice: 600},
"Lumber":  {basePrice: 5000, nostore: true},
"Malt Vinegar":  {basePrice: 50},
"Mango":  {basePrice: 100},
"Mango Juice":  {basePrice: 250},
"Mango Juice Concentrate":  {basePrice: 125, nostore: true},
"Mango Sorbet":  {basePrice: 150},
"Mattress":  {basePrice: 8000},
"Mayonnaise":  {basePrice: 300},
"Microprocessor":  {basePrice: 200},
"Microwave Oven":  {basePrice: 8000},
"Milk":  {basePrice: 50},
"Milk Chocolate Bar":  {basePrice: 550},
"Milk in a Carton":  {basePrice: 200},
"Milk in a Jug":  {basePrice: 250},
"Mineral":  {basePrice: 50, nostore: true},
"Mirror":  {basePrice: 12500},
"Monitor":  {basePrice: 10000},
"Motherboard":  {basePrice: 15000},
"Motorcycle":  {basePrice: 700000},
"Motorcycle Body":  {basePrice: 400000},
"Mustard":  {basePrice: 300},
"Mustard Seeds":  {basePrice: 100},
"Natural Gas":  {basePrice: 100, nostore: true},
"Neapolitan Ice Cream":  {basePrice: 250},
"Necktie":  {basePrice: 1200},
"Nightstand":  {basePrice: 4000},
"Nylon":  {basePrice: 200},
"Nylon Thread":  {basePrice: 400},
"Oat":  {basePrice: 50},
"Oatmeal":  {basePrice: 100},
"Oatmeal Raisin Cookie":  {basePrice: 250},
"Orange":  {basePrice: 50},
"Orange Gelatin":  {basePrice: 350},
"Orange Juice":  {basePrice: 250},
"Orange Juice Concentrate":  {basePrice: 125, nostore: true},
"Orange Sorbet":  {basePrice: 150},
"Paper":  {basePrice: 200},
"Paper Carton":  {basePrice: 10},
"Paper Napkin":  {basePrice: 100},
"Paper Towel":  {basePrice: 200},
"Passenger Jet":  {basePrice: 3000000000},
"Passenger Jet Body":  {basePrice: 1600000000, nostore: true},
"Passenger Jet Wing":  {basePrice: 300000000, nostore: true},
"Pastry Dough":  {basePrice: 100},
"Peanut Butter":  {basePrice: 200},
"Peanuts":  {basePrice: 50},
"Pepper":  {basePrice: 500},
"Personal Jet":  {basePrice: 120000000},
"Personal Jet Body":  {basePrice: 50000000, nostore: true},
"Personal Jet Wing":  {basePrice: 15000000, nostore: true},
"Petroleum":  {basePrice: 5000},
"Pig":  {basePrice: 7000, nostore: true},
"Pillow":  {basePrice: 1500},
"Piña Colada": {basePrice: 400},
"Pineapple":  {basePrice: 100},
"Pineapple Gelatin":  {basePrice: 350},
"Pineapple Juice":  {basePrice: 250},
"Pineapple Juice Concentrate":  {basePrice: 125, nostore: true},
"Pineapple Sorbet":  {basePrice: 150},
"Pizza":  {basePrice: 800},
"Plastic":  {basePrice: 500},
"Plastic Bottle":  {basePrice: 10},
"Plastic Jar":  {basePrice: 10},
"Plastic Jug":  {basePrice: 20},
"Polyester":  {basePrice: 200},
"Polyester Thread":  {basePrice: 350},
"Pork":  {basePrice: 400},
"Pork Chop":  {basePrice: 500},
"Pork Sausage":  {basePrice: 250},
"Pork Skin":  {basePrice: 60, nostore: true},
"Portable Heater":  {basePrice: 6000},
"Pot Roast":  {basePrice: 650},
"Potato":  {basePrice: 50},
"Power Boat":  {basePrice: 2500000},
"Power Boat Body":  {basePrice: 1000000, nostore: true},
"Power Cruiser":  {basePrice: 18000000},
"Printer":  {basePrice: 12000},
"Propeller":  {basePrice: 100000, nostore: true},
"Pumpkin":  {basePrice: 400},
"Pumpkin Pie":  {basePrice: 500},
"Quartz":  {basePrice: 500, nostore: true},
"Radio":  {basePrice: 2000},
"Raisin":  {basePrice: 200},
"Remote Controlled Car":  {basePrice: 3500},
"Rice":  {basePrice: 50},
"Roast Beef":  {basePrice: 750},
"Roast Beef Sandwich":  {basePrice: 250},
"Roasted Pork Chop":  {basePrice: 800},
"Root Beer":  {basePrice: 300},
"Root Beer Concentrate":  {basePrice: 150, nostore: true},
"Rotisserie Chicken":  {basePrice: 900},
"Row Boat":  {basePrice: 70000},
"Rubber":  {basePrice: 500},
"Rubber Duck":  {basePrice: 500},
"Rubik's Cube":  {basePrice: 800},
"Rug":  {basePrice: 5000},
"Rum":  {basePrice: 200},
"Sailboat":  {basePrice: 800000},
"Sails":  {basePrice: 150000, nostore: true},
"Salami":  {basePrice: 750},
"Salt":  {basePrice: 200},
"Sapphire":  {basePrice: 2000, nostore: true},
"Scanner":  {basePrice: 12000},
"Scarf":  {basePrice: 1800},
"Seats":  {basePrice: 30000},
"Sheet":  {basePrice: 1500},
"Shorts":  {basePrice: 1500},
"Side-By-Side Refrigerator":  {basePrice: 100000},
"Silica Sand":  {basePrice: 100, nostore: true},
"Silicon":  {basePrice: 1000},
"Silk":  {basePrice: 500, nostore: true},
"Silk Thread":  {basePrice: 1000},
"Silver":  {basePrice: 10000},
"Silver Bracelet":  {basePrice: 25000},
"Silver Necklace":  {basePrice: 10000},
"Silver Ore":  {basePrice: 5000, nostore: true},
"Skirt":  {basePrice: 1500},
"Sleeping Bag":  {basePrice: 3000},
"Snorkel":  {basePrice: 1500},
"Soccer Ball":  {basePrice: 1800},
"Socks":  {basePrice: 1000},
"Sport Shoes":  {basePrice: 3000},
"Sport Watch":  {basePrice: 1500},
"Steak Taco":  {basePrice: 350},
"Steel":  {basePrice: 5000},
"Stockings":  {basePrice: 1200},
"Strawberry":  {basePrice: 50},
"Strawberry Banana Cake":  {basePrice: 1250},
"Strawberry Banana Smoothie":  {basePrice: 500},
"Strawberry Gelatin":  {basePrice: 350},
"Strawberry Ice Cream":  {basePrice: 250},
"Strawberry Juice Concentrate":  {basePrice: 125, nostore: true},
"Strawberry Milk":  {basePrice: 300},
"Strawberry Pie":  {basePrice: 500},
"Strawberry Sorbet":  {basePrice: 150},
"Stuffed Animal":  {basePrice: 1000},
"Sugar":  {basePrice: 200},
"Sugar Cane":  {basePrice: 150},
"Sunglasses":  {basePrice: 1800},
"Surfboard":  {basePrice: 20000},
"SUV":  {basePrice: 1200000},
"SUV Body":  {basePrice: 700000},
"Sweater":  {basePrice: 2000},
"Swim Fin":  {basePrice: 2500},
"Swim Goggle":  {basePrice: 1500},
"Swim Suit":  {basePrice: 6000},
"T-Shirt":  {basePrice: 1000},
"Table":  {basePrice: 15000},
"Table Lamp":  {basePrice: 1000},
"Tennis Ball":  {basePrice: 150},
"Tennis Racquet":  {basePrice: 2000},
"Tiramisu":  {basePrice: 1250},
"Titanium":  {basePrice: 20000},
"Titanium Alloy":  {basePrice: 30000},
"Titanium Ore":  {basePrice: 3000, nostore: true},
"Toilet":  {basePrice: 15000},
"Toilet Bowl":  {basePrice: 5000},
"Toilet Tank":  {basePrice: 2500},
"Toilet Tissue":  {basePrice: 1000},
"Tomato":  {basePrice: 70},
"Tomato Ketchup":  {basePrice: 200},
"Top-Freezer Refrigerator":  {basePrice: 50000},
"Tortilla":  {basePrice: 200},
"Tortilla Chips":  {basePrice: 400},
"Towel":  {basePrice: 700},
"Toy Keyboard":  {basePrice: 2500},
"Truck":  {basePrice: 1400000},
"Truck Body":  {basePrice: 850000},
"Tungsten Ore":  {basePrice: 12000, nostore: true},
"Tungsten Powder":  {basePrice: 250},
"Turbine Engine":  {basePrice: 1000000, nostore: true},
"TV":  {basePrice: 30000},
"Van":  {basePrice: 1350000},
"Van Body":  {basePrice: 750000},
"Vanilla Beans":  {basePrice: 900},
"Vanilla Cake":  {basePrice: 1250},
"Vanilla Extract":  {basePrice: 1000},
"Vanilla Ice Cream":  {basePrice: 250},
"Vanilla Milk":  {basePrice: 300},
"Vinyl":  {basePrice: 250},
"Vinyl Thread":  {basePrice: 500},
"Wallet":  {basePrice: 1600},
"Washer":  {basePrice: 42000},
"Water":  {basePrice: 10, nostore: true},
"Water Gun":  {basePrice: 1500},
"Watermelon":  {basePrice: 400},
"Watermelon Juice":  {basePrice: 250},
"Watermelon Juice Concentrate":  {basePrice: 125, nostore: true},
"Watermelon Sorbet":  {basePrice: 150},
"Wedding Dress":  {basePrice: 10000},
"Wheat":  {basePrice: 50},
"Wheel":  {basePrice: 8000},
"Whipped Cream":  {basePrice: 100},
"White Vinegar":  {basePrice: 100},
"Whole Chicken":  {basePrice: 1000},
"Wig":  {basePrice: 1500},
"Window AC":  {basePrice: 15000},
"Wine":  {basePrice: 1000},
"Wood Floor":  {basePrice: 2000},
"Wool":  {basePrice: 400, nostore: true},
"Wool Coat":  {basePrice: 11000},
"Wool Yarn":  {basePrice: 900},
"Yogurt":  {basePrice: 200},
});
var productDbUpdated = new Date(1900, 1, 1, 0, 0, 0, 0);

function updateProductDatabase() {
    if (window.localStorage) {
        var db = window.localStorage.getItem('productDb');
        var updated = window.localStorage.getItem('productDbUpdated');
        var now = new Date();
        if (db != null) {
            productDb = new Dict(JSON.parse(db));
        }
        if (updated != null) {
            productDbUpdated = new Date(updated);
        }
        // two hours
        if (now - productDbUpdated > (1000 * 60 * 60 * 2)) {
            htmlToDom('http://www.ratjoy.com/eos/tools/product_list.php', scrapeProductList);
            htmlToDom('http://www.ratjoy.com/eos/tools/product_stats_data.php', scrapeProductStats);
        }
    }
}

function saveProductDb(update) {
    if (window.localStorage) {
        if (update)
            productDbUpdated = new Date();
        window.localStorage.setItem('productDb', JSON.stringify(productDb));
        window.localStorage.setItem('productDbUpdated', productDbUpdated);
    }
}

function htmlToDom(url, callback) {
    var i = 0, frame;
    do {
        var id = 'htmlToDomFrame-' + i;
        i++;
    } while (frame = document.getElementById(id));
    
    frame = document.createElement("iframe");
    frame.id = id;
    frame.name = id;
    frame.width = 0;
    frame.height = 0;
    frame.style.visibility = 'hidden';

    document.body.appendChild(frame);
    frame.addEventListener("load", function (event) {
        var doc = this.contentDocument;
        if (doc.location.href == "about:blank") return;

        callback(doc);
        
        setTimeout(function (){
            frame.parentNode.removeChild(frame);
        }, 10);
    }, true);
    frame.src = url; 
}

function scrapeTable(doc, search, namekey) {
    namekey = namekey || 'name';
    
    var i, j;
    var name, val;
    var table = doc.getElementsByTagName('table')[0];
    var thead = table.getElementsByTagName('thead')[0];

    var cols = thead.getElementsByTagName('td');
    var found = {};
    var nameIdx = null;
    
    for (i = 0; i < cols.length; i++) {
        name = cols[i].textContent;
        val = search[name];
        if (typeof val != 'undefined') {
            if (val == namekey)
                nameIdx = i;
            else
                found[i] = val;
        }
    }
    if (nameIdx == null) {
        return
    }
    
//    var tbody = table.getElementsByTagName('tbody')[0];
    var rows = table.getElementsByTagName('tbody'); //tbody.getElementsByTagName('tr');
    var product;
    for (i = 0; i < rows.length; i++) {
        cols = rows[i].getElementsByTagName('td');
        product = productDb.setDefault(cols[nameIdx].textContent, {});
        for (j in found)
            product[found[j]] = cols[j].textContent;
    }
}

function scrapeProductList(doc) {
    scrapeTable(doc, {
        'Product' : 'name',
        'Base Value' : 'basePrice',
    });
    saveProductDb(true);
}

function scrapeProductStats(doc) {
    scrapeTable(doc, {
        'Product' : 'name',
        'Avg Store Price' : 'avgPrice',
        'Avg Store Quality' : 'avgQuality',
        'Units Sold' : 'unitsSold',
        'Demand' : 'demand',
    });
    saveProductDb(true);
}

updateProductDatabase();