Skip to content

Instantly share code, notes, and snippets.

@JorgeMartinezG

JorgeMartinezG/acled.ipynb

Last active November 20, 2020 19:24
Show Gist options
  • Save JorgeMartinezG/a469eb10a7d38fe7f42138d697aa3828 to your computer and use it in GitHub Desktop.
Save JorgeMartinezG/a469eb10a7d38fe7f42138d697aa3828 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
acled.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ACLED CSV example"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import pandas as pd\n",
"import csv\n",
"\n",
"from arcgis.gis import GIS"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Fetch data from ACLED API\n",
"\n",
"To read API documentation, check this link: https://developer.acleddata.com/rehd/cms/views/acled_api/documents/API-User-Guide.pdf"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"params = {\n",
" 'key': 'h3IYuBzWZ7S9j',\n",
" 'email': '[email protected]',\n",
" 'iso': 4, # AFG.\n",
" 'event_date': '2020-10-01 | 2020-11-01',\n",
" 'event_date_where': 'BETWEEN',\n",
" 'page': 1\n",
"}\n",
"\n",
"API_URL = 'https://api.acleddata.com/acled/read'\n",
"len_data = -1\n",
"\n",
"csv_file = []\n",
"while len_data != 0:\n",
" data = requests.get(API_URL, params=params).json()['data']\n",
" csv_file.extend(data)\n",
" len_data = len(data)\n",
" params['page'] = params['page'] + 1\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Process data (group by, etc.)\n",
"\n",
"For this example, a simple group by location. Count number of incidents."
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>data_id</th>\n",
" <th>iso</th>\n",
" <th>event_id_cnty</th>\n",
" <th>event_id_no_cnty</th>\n",
" <th>event_date</th>\n",
" <th>year</th>\n",
" <th>time_precision</th>\n",
" <th>event_type</th>\n",
" <th>sub_event_type</th>\n",
" <th>actor1</th>\n",
" <th>...</th>\n",
" <th>location</th>\n",
" <th>latitude</th>\n",
" <th>longitude</th>\n",
" <th>geo_precision</th>\n",
" <th>source</th>\n",
" <th>source_scale</th>\n",
" <th>notes</th>\n",
" <th>fatalities</th>\n",
" <th>timestamp</th>\n",
" <th>iso3</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>7296148</td>\n",
" <td>4</td>\n",
" <td>AFG48891</td>\n",
" <td>48891</td>\n",
" <td>2020-11-01</td>\n",
" <td>2020</td>\n",
" <td>2</td>\n",
" <td>Explosions/Remote violence</td>\n",
" <td>Remote explosive/landmine/IED</td>\n",
" <td>Taliban</td>\n",
" <td>...</td>\n",
" <td>Qalandar</td>\n",
" <td>33.5151</td>\n",
" <td>69.6760</td>\n",
" <td>2</td>\n",
" <td>Voice of Jihad</td>\n",
" <td>Other</td>\n",
" <td>Around 1 November 2020 (as reported), 2 Afghan...</td>\n",
" <td>0</td>\n",
" <td>1604952033</td>\n",
" <td>AFG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>7296125</td>\n",
" <td>4</td>\n",
" <td>AFG48880</td>\n",
" <td>48880</td>\n",
" <td>2020-11-01</td>\n",
" <td>2020</td>\n",
" <td>2</td>\n",
" <td>Explosions/Remote violence</td>\n",
" <td>Remote explosive/landmine/IED</td>\n",
" <td>Taliban</td>\n",
" <td>...</td>\n",
" <td>Kaga</td>\n",
" <td>34.2409</td>\n",
" <td>70.1827</td>\n",
" <td>2</td>\n",
" <td>Voice of Jihad</td>\n",
" <td>Other</td>\n",
" <td>Around 1 November 2020 (as reported), 3 Afghan...</td>\n",
" <td>0</td>\n",
" <td>1604952033</td>\n",
" <td>AFG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>7296123</td>\n",
" <td>4</td>\n",
" <td>AFG48893</td>\n",
" <td>48893</td>\n",
" <td>2020-11-01</td>\n",
" <td>2020</td>\n",
" <td>2</td>\n",
" <td>Battles</td>\n",
" <td>Armed clash</td>\n",
" <td>Taliban</td>\n",
" <td>...</td>\n",
" <td>Sholgara</td>\n",
" <td>36.3041</td>\n",
" <td>66.8573</td>\n",
" <td>2</td>\n",
" <td>Voice of Jihad</td>\n",
" <td>Other</td>\n",
" <td>Around 1 November 2020 (as reported), 1 Afghan...</td>\n",
" <td>0</td>\n",
" <td>1604952033</td>\n",
" <td>AFG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>7296122</td>\n",
" <td>4</td>\n",
" <td>AFG48892</td>\n",
" <td>48892</td>\n",
" <td>2020-11-01</td>\n",
" <td>2020</td>\n",
" <td>2</td>\n",
" <td>Battles</td>\n",
" <td>Armed clash</td>\n",
" <td>Taliban</td>\n",
" <td>...</td>\n",
" <td>Qarabagh</td>\n",
" <td>34.8529</td>\n",
" <td>69.1706</td>\n",
" <td>2</td>\n",
" <td>Voice of Jihad</td>\n",
" <td>Other</td>\n",
" <td>Around 1 November 2020 (as reported), 5 Afghan...</td>\n",
" <td>0</td>\n",
" <td>1604952033</td>\n",
" <td>AFG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>7296118</td>\n",
" <td>4</td>\n",
" <td>AFG48888</td>\n",
" <td>48888</td>\n",
" <td>2020-11-01</td>\n",
" <td>2020</td>\n",
" <td>1</td>\n",
" <td>Explosions/Remote violence</td>\n",
" <td>Remote explosive/landmine/IED</td>\n",
" <td>Taliban</td>\n",
" <td>...</td>\n",
" <td>Nawa-e Barakzayi</td>\n",
" <td>31.3918</td>\n",
" <td>64.3190</td>\n",
" <td>2</td>\n",
" <td>Voice of Jihad</td>\n",
" <td>Other</td>\n",
" <td>On 1 November 2020, an Afghan military armored...</td>\n",
" <td>0</td>\n",
" <td>1604952033</td>\n",
" <td>AFG</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 31 columns</p>\n",
"</div>"
],
"text/plain": [
" data_id iso event_id_cnty event_id_no_cnty event_date year \\\n",
"0 7296148 4 AFG48891 48891 2020-11-01 2020 \n",
"1 7296125 4 AFG48880 48880 2020-11-01 2020 \n",
"2 7296123 4 AFG48893 48893 2020-11-01 2020 \n",
"3 7296122 4 AFG48892 48892 2020-11-01 2020 \n",
"4 7296118 4 AFG48888 48888 2020-11-01 2020 \n",
"\n",
" time_precision event_type sub_event_type \\\n",
"0 2 Explosions/Remote violence Remote explosive/landmine/IED \n",
"1 2 Explosions/Remote violence Remote explosive/landmine/IED \n",
"2 2 Battles Armed clash \n",
"3 2 Battles Armed clash \n",
"4 1 Explosions/Remote violence Remote explosive/landmine/IED \n",
"\n",
" actor1 ... location latitude longitude geo_precision \\\n",
"0 Taliban ... Qalandar 33.5151 69.6760 2 \n",
"1 Taliban ... Kaga 34.2409 70.1827 2 \n",
"2 Taliban ... Sholgara 36.3041 66.8573 2 \n",
"3 Taliban ... Qarabagh 34.8529 69.1706 2 \n",
"4 Taliban ... Nawa-e Barakzayi 31.3918 64.3190 2 \n",
"\n",
" source source_scale \\\n",
"0 Voice of Jihad Other \n",
"1 Voice of Jihad Other \n",
"2 Voice of Jihad Other \n",
"3 Voice of Jihad Other \n",
"4 Voice of Jihad Other \n",
"\n",
" notes fatalities timestamp \\\n",
"0 Around 1 November 2020 (as reported), 2 Afghan... 0 1604952033 \n",
"1 Around 1 November 2020 (as reported), 3 Afghan... 0 1604952033 \n",
"2 Around 1 November 2020 (as reported), 1 Afghan... 0 1604952033 \n",
"3 Around 1 November 2020 (as reported), 5 Afghan... 0 1604952033 \n",
"4 On 1 November 2020, an Afghan military armored... 0 1604952033 \n",
"\n",
" iso3 \n",
"0 AFG \n",
"1 AFG \n",
"2 AFG \n",
"3 AFG \n",
"4 AFG \n",
"\n",
"[5 rows x 31 columns]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.DataFrame(csv_file)\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>location</th>\n",
" <th>latitude</th>\n",
" <th>longitude</th>\n",
" <th>count</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Adraskan</td>\n",
" <td>33.6457</td>\n",
" <td>62.2696</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Alasay</td>\n",
" <td>34.8958</td>\n",
" <td>69.7208</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Ali Abad</td>\n",
" <td>36.5211</td>\n",
" <td>68.8998</td>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Alingar</td>\n",
" <td>34.8352</td>\n",
" <td>70.3607</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Alishing</td>\n",
" <td>34.7038</td>\n",
" <td>70.1537</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" location latitude longitude count\n",
"0 Adraskan 33.6457 62.2696 2\n",
"1 Alasay 34.8958 69.7208 2\n",
"2 Ali Abad 36.5211 68.8998 6\n",
"3 Alingar 34.8352 70.3607 1\n",
"4 Alishing 34.7038 70.1537 1"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"location_count = df.groupby(['location','latitude', 'longitude']).size().reset_index(name=\"count\")\n",
"location_count.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Save data to CSV format"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"FILENAME = 'count_incidents'\n",
"OUTPUT = f'/Users/jorge/Desktop/{FILENAME}.csv'\n",
"\n",
"with open(OUTPUT, \"w\", newline=\"\") as output_file:\n",
" keys = csv_file[0].keys()\n",
" dict_writer = csv.DictWriter(output_file, keys)\n",
" dict_writer.writeheader()\n",
" dict_writer.writerows(csv_file)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Upload data to Arcgis Online"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ARCGIS_USER = ''\n",
"ARCGIS_PW = ''\n",
"ARCGIS_URL = 'https://unwfp.maps.arcgis.com/'\n",
"\n",
"gis = GIS(ARCGIS_URL, ARCGIS_USER, ARCGIS_PW)\n",
"items = gis.content.search(f\"title:{FILENAME} type:CSV owner:{ARCGIS_USER}\")\n",
"\n",
"item_params = dict(title=FILENAME)\n",
"overwrite = False\n",
"if len(items) > 0:\n",
" print(\"Updating file\")\n",
" item = items[0]\n",
" item.update(item_params, data=OUTPUT)\n",
" overwrite = True\n",
"else:\n",
" print(\"Uploading file\")\n",
" item = gis.content.add(item_params, data=OUTPUT)\n",
"\n",
"publish_params = dict(\n",
" type=\"csv\",\n",
" locationType=\"coordinates\",\n",
" latitudeFieldName=\"latitude\",\n",
" longitudeFieldName=\"longitude\",\n",
")\n",
"item.publish(publish_parameters=publish_params, overwrite=overwrite)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.2"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment