acled.py

import requests
import csv
import logging
import os

from arcgis.gis import GIS
from datetime import date
from dotenv import load_dotenv

logger = logging.getLogger()
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)

FILENAME = "incidents_raw"
TEMP_PATH = f"/tmp/{FILENAME}.csv"


def fetch():
    ACLED_KEY = os.getenv("ACLED_KEY")
    ACLED_EMAIL = os.getenv("ACLED_EMAIL")
    ISO_COUNTRIES = os.getenv("ISO_COUNTRIES").split(",")
    START_DATE = os.getenv("START_DATE")

    logger.info("Fetching data from acled API")

    params = dict(
        key=ACLED_KEY,
        email=ACLED_EMAIL,
        iso="|".join(ISO_COUNTRIES),
        event_date=f"{START_DATE}|{date.today().isoformat()}",
        event_date_where="BETWEEN",
        page=1,
    )

    if len(ISO_COUNTRIES) > 1:
        params["iso_where"] = "="

    ACLED_API_URL = os.getenv("ACLED_API_URL")

    len_data = -1

    csv_file = []
    while len_data != 0:
        data = requests.get(ACLED_API_URL, params=params).json()["data"]
        csv_file.extend(data)
        len_data = len(data)
        params["page"] = params["page"] + 1

    return csv_file

def upload_arcgis():
    ARCGIS_USER = os.getenv("ARCGIS_USER")
    ARCGIS_PW = os.getenv("ARCGIS_PW")
    ARCGIS_URL = os.getenv("ARCGIS_URL")

    gis = GIS(ARCGIS_URL, ARCGIS_USER, ARCGIS_PW)
    items = gis.content.search(f"title:{FILENAME} type:CSV owner:{ARCGIS_USER}")

    item_params = dict(title=FILENAME)
    overwrite = False
    if len(items) > 0:
        logger.info("Updating file in Arcgis")
        item = items[0]
        item.update(item_params, data=TEMP_PATH)
        overwrite = True
    else:
        logger.info("Uploading file to Arcgis")
        item = gis.content.add(item_params, data=TEMP_PATH)

    item.share(everyone=True)

    logger.info("Publishing layer")
    publish_params = dict(
        name=FILENAME,
        type="csv",
        locationType="coordinates",
        latitudeFieldName="latitude",
        longitudeFieldName="longitude",
    )
    item = item.publish(publish_parameters=publish_params, overwrite=overwrite)
    item.share(everyone=True)


def main():
    folder = os.path.dirname(__file__)
    load_dotenv(os.path.join(folder, "config.txt"))

    data = fetch()

    keys = data[0].keys()
    with open(TEMP_PATH, "w", newline="") as output_file:
        dict_writer = csv.DictWriter(output_file, keys)
        dict_writer.writeheader()
        dict_writer.writerows(data)

    upload_arcgis()


if __name__ == "__main__":
    main()

config.txt

ACLED_KEY=""
ACLED_EMAIL="[email protected]"
ACLED_API_URL="https://api.acleddata.com/acled/read"
ISO_COUNTRIES="51,31"
START_DATE="2020-08-01"
ARCGIS_USER="jorge.martinezgomez_UNWFP"
ARCGIS_PW=""
ARCGIS_URL="https://unwfp.maps.arcgis.com/"

requirements.txt

# pip install -r --no-deps. Arcgis comes with a bunch of unneeded dependencies
arcgis==1.8.2
certifi==2020.6.20
cffi==1.14.3
chardet==3.0.4
cryptography==3.1.1
idna==2.10
ntlm-auth==1.5.0
pycparser==2.20
requests==2.24.0
requests-ntlm==1.1.0
requests-toolbelt==0.9.1
six==1.15.0
urllib3==1.25.10
python-dateutil==2.8.1
pytz==2020.4
six==1.15.0