Last active
February 19, 2020 18:51
-
-
Save Tokariew/394c20270166c737dac3327f49b36534 to your computer and use it in GitHub Desktop.
Download weather data from wunderground site using new API
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import os | |
from concurrent.futures import ThreadPoolExecutor | |
from datetime import date, timedelta | |
from time import sleep | |
import requests | |
API_KEY = '' # fill your API key | |
STATION_ID = '' # pws of intertest | |
UNITS = 'm' | |
FORMAT = 'json' | |
URL = 'https://api.weather.com/v2/pws/history/all?stationId={}&format={}&units={}&date={}&apiKey={}' | |
START_DATE = date(2018, 6, 1) | |
END_DATE = date.today() | |
def flat_json(x): | |
def flat_dict(x): | |
out = dict() | |
for key, val in x.items(): | |
if type(val) == dict: | |
for key2, val2 in val.items(): | |
out[key2] = val2 | |
else: | |
out[key] = val | |
return out | |
if type(x) == dict: | |
return flat_dict(x) | |
elif type(x) == list: | |
ret = [] | |
for item in x: | |
ret.append(flat_dict(item)) | |
return ret | |
def download(day): | |
url = URL.format(STATION_ID, FORMAT, UNITS, day.strftime("%Y%m%d"), | |
API_KEY) | |
file_name = f'{day.strftime("%Y%m%d")}.json' | |
r = requests.get(url) | |
if r.status_code != 200: | |
print(f'Skipping {file_name}, status: {r.status_code}') | |
return | |
response = r.content | |
js = json.loads(response)['observations'] | |
js = flat_json(js) | |
with open(file_name, 'w') as file: | |
json.dump(js, file, indent=2) | |
delay = r.elapsed.total_seconds() | |
sleep(delay) | |
def daterange(start_date, end_date): | |
for n in range(int((end_date - start_date).days)): | |
yield start_date + timedelta(n) | |
with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: | |
for day in executor.map(download, daterange(START_DATE, END_DATE)): | |
pass |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment