append.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ACLED CSV example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "\n",
    "from datetime import date\n",
    "from arcgis.gis import GIS\n",
    "from json import dump"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch data from ACLED API\n",
    "\n",
    "To read API documentation, check this link: https://developer.acleddata.com/rehd/cms/views/acled_api/documents/API-User-Guide.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'key': '',\n",
    "    'email': '',\n",
    "    'iso': 4, # AFG.\n",
    "    'event_date': f'2020-10-01 | {date.today().isoformat()}',\n",
    "    'event_date_where': 'BETWEEN',\n",
    "    'page': 1\n",
    "}\n",
    "\n",
    "API_URL = 'https://api.acleddata.com/acled/read'\n",
    "len_data = -1\n",
    "\n",
    "csv_file = []\n",
    "while len_data != 0:\n",
    "    data = requests.get(API_URL, params=params).json()['data']\n",
    "    csv_file.append(data)\n",
    "    len_data = len(data)\n",
    "    params['page'] = params['page'] + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save first block of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_feature(row):\n",
    "    row['data_id'] = int(row['data_id'])\n",
    "    return {\n",
    "        'type': 'Feature',\n",
    "        'properties': row,\n",
    "        'geometry': {\n",
    "            'type': 'Point',\n",
    "            'coordinates': [float(row.get('longitude')),\n",
    "                            float(row.get('latitude'))]\n",
    "        }\n",
    "    }\n",
    "\n",
    "\n",
    "def save_to_geojson(data, path):\n",
    "    features = [make_feature(f) for f in data]\n",
    "    collection = {\n",
    "        'type': 'FeatureCollection',\n",
    "        'features': features,\n",
    "    }\n",
    "    \n",
    "    with open(path, \"w\") as output_file:\n",
    "        dump(collection, output_file)\n",
    "\n",
    "ARCGIS_USER = ''\n",
    "ARCGIS_PW = ''\n",
    "ARCGIS_URL = ''\n",
    "\n",
    "FILENAME = 'afg_incidents'\n",
    "OUTPUT = f'/tmp/{FILENAME}.geojson'\n",
    "\n",
    "save_to_geojson(csv_file[0], OUTPUT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload data to Arcgis Online"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"item_container\" style=\"height: auto; overflow: hidden; border: 1px solid #cfcfcf; border-radius: 2px; background: #f6fafa; line-height: 1.21429em; padding: 10px;\">\n",
       "                    <div class=\"item_left\" style=\"width: 210px; float: left;\">\n",
       "                       <a href='https://UNWFP.maps.arcgis.com/home/item.html?id=aef4b7a04fba4cc88296f0bb4db6e608' target='_blank'>\n",
       "                        <img src='http://static.arcgis.com/images/desktopapp.png' class=\"itemThumbnail\">\n",
       "                       </a>\n",
       "                    </div>\n",
       "\n",
       "                    <div class=\"item_right\"     style=\"float: none; width: auto; overflow: hidden;\">\n",
       "                        <a href='https://UNWFP.maps.arcgis.com/home/item.html?id=aef4b7a04fba4cc88296f0bb4db6e608' target='_blank'><b>afg_incidents</b>\n",
       "                        </a>\n",
       "                        <br/><img src='https://UNWFP.maps.arcgis.com/home/js/jsapi/esri/css/images/item_type_icons/featureshosted16.png' style=\"vertical-align:middle;\">Feature Layer Collection by jorge.martinezgomez_UNWFP\n",
       "                        <br/>Last Modified: June 15, 2021\n",
       "                        <br/>0 comments, 0 views\n",
       "                    </div>\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<Item title:\"afg_incidents\" type:Feature Layer Collection owner:jorge.martinezgomez_UNWFP>"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gis = GIS(ARCGIS_URL, ARCGIS_USER, ARCGIS_PW)\n",
    "# Create and publish feature layer.\n",
    "item = gis.content.add({'title': FILENAME, 'type': 'GeoJson'}, data=OUTPUT)\n",
    "item.publish()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Append data to layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items = gis.content.search(f'title: {FILENAME} type: Feature Service owner: {ARCGIS_USER}')\n",
    "feature_service = items[0]\n",
    "\n",
    "lyr = feature_service.layers[0]\n",
    "\n",
    "TEMP_OUTPUT = f\"/tmp/{FILENAME}_temp.geojson\"\n",
    "save_to_geojson(csv_file[2], TEMP_OUTPUT)\n",
    "\n",
    "# Creating temporal item.\n",
    "temp_item = gis.content.add({'title': f\"{FILENAME}_temp\", 'type': 'GeoJson'}, data=TEMP_OUTPUT)\n",
    "lyr.append(item_id=temp_item.id, upload_format='geojson', upsert=False)\n",
    "\n",
    "# delete item.\n",
    "temp_item.delete()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}