Use clojure.core.async to Implement Http Client Async

clojure-async.clj

;; lein repl

(ns test (:require [clojure.core.async :as async]
                   [clj-http.client :as client]))

;; define test data
(def urls ["http://www.google.com" "http://www.google1.com" "http://255.255.33.44:8001/"])

;; use async/go to build a async http client
(defn fetch-async
  [url]
  (let [ret (async/promise-chan)]
    (async/go
      (try
        (let [res (client/get url {:socket-timeout 500 
                                   :connection-timeout 500})]
          (async/put! ret {:result :success :msg (:status res)}))
      (catch Exception e (async/put! ret {:result :error :msg (.getMessage e)}))))
  ret))

;; use async/go to build a async http client but disable apache http client retry feature
(defn fetch-async-without-retry
  [url]
  (let [ret (async/promise-chan)]
    (async/go
      (try
        (let [res (client/get url {:socket-timeout 500 
                                   :connection-timeout 1000
                                   :retry-handler (fn [ex try-count http-context]
                                                    (if (> try-count 0) false true))})]
          (async/put! ret {:result :success :msg (:status res)}))
      (catch Exception e (async/put! ret {:result :error :msg (.getMessage e)}))))
  ret))

;; test fetach a domain url
(time (async/<!! (fetch-async "http://www.google1.com")))

;; test fetach an ip url
(time (async/<!! (fetch-async "http://255.255.33.44:8001/")))

;; block main thread to get all response data
(let [res-promise (map #(fetch-async %) urls)]
    (map #(async/<!! %) res-promise))

;; metric the execute time
(let [res-promise (map #(fetch-async %) urls)]
    (map #(time (async/<!! %)) res-promise))

(time (let [res-promise (map #(fetch-async %) urls)]
    (map #(async/<!! %) res-promise)))

(let [res-promise (map #(fetch-async %) urls)]
    (time (map #(async/<!! %) res-promise)))

lein-profiles.clj

;; ~/.lein/profiles.clj
{
  :user {
          :plugins [[lein-midje "3.2.1"]
                    [lein-shell "0.5.0"]]
          :dependencies [[org.clojure/clojure "1.10.3"]
                    [rhizome "0.2.9"]
                    [org.clojure/core.match "1.0.0"]
                    [buddy/buddy-sign "3.1.0"]
                    [clj-http "3.12.2"]
                    [org.clojure/core.async "1.5.648"]
                    [environ "1.1.0"]
                    [cheshire "5.10.0"]
                    [instaparse "1.4.10"]]
        }
}