gistfile1.clj

(ns user
  (:require [clojure.java.io :as io]
            [datomic.api :as d]))

(defn order-schema []
  (read-string (slurp (io/resource "schema.edn"))))

(defn start-datomic! [uri schema]
  (d/create-database uri)
  (let [conn (d/connect uri)]
    (when-not (seq (d/q '[:find ?e :where [?e :db/ident :orders/primary-key]]
                        (d/db conn)))
      @(d/transact conn schema))
    conn))

(def orders-uri "datomic:dev://localhost:4334/drogalis")

(defn orders-db []
  (start-datomic! orders-uri (order-schema)))

(defn create-order-tx
  ([n]
     (create-order-tx (d/tempid :orders) n))
  ([tempid n]
     [{:db/id                   tempid
       :orders/primary-key      n
       :orders/customer-id      n
       :orders/timestamp        (java.sql.Timestamp/valueOf "2009-01-01 04:32:33")
       :orders/shipping-address "xxx"
       :orders/total-amount     (java.math.BigDecimal. 0.00)}]))

(defn partition-order-txs
  [txs]
  (->> txs
       (partition-all 200)
       (map #(apply concat %))))

(defn aws-insert
  [conn]
  (doseq [n (range 100001)]
    @(d/transact-async conn (create-order-tx n))))

(defn aws-batch-insert
  [conn]
  (doseq [tx (->> 100001
                  range
                  (map create-order-tx)
                  partition-order-txs)]
    @(d/transact-async conn tx)))

(defn aws-batch-pipeline-insert
  [conn]
  (doseq [tx-res (->> 100001
                      range
                      (map create-order-tx)
                      partition-order-txs
                      (pmap #(d/transact-async conn %)))]
    @tx-res))

(comment

  (def conn (orders-db))

  (time
   (aws-insert conn))

  ;; "Elapsed time: 200907.803 msecs"

  (time
   (aws-batch-insert conn))

  ;; "Elapsed time: 15996.735 msecs"

  (time
   (aws-batch-pipeline-insert conn))

  ;; "Elapsed time: 9892.244 msecs"

  )