gistfile1.txt

rm /tmp/bulk
for doc in {1..10000}; do
    echo '{"index":{}}' >> /tmp/bulk
    echo '{"@timestamp": '$(($idx * 10000 + $doc))', "field": "foo'$(($doc % 10))'"}' >> /tmp/bulk
done

for idx in {1..10}; do
    curl -uelastic:password -XDELETE localhost:9200/test$idx

    echo '{
        "settings": {
            "index.mapping.total_fields.limit": 10000
        },
        "mappings": {
            "properties": {
                "@timestamp": { "type": "date" },
                "field": { "type": "keyword" }
            }
        }
    }' > /tmp/idx
    curl -uelastic:password -XPUT -HContent-Type:application/json localhost:9200/test$idx --data @/tmp/idx

    echo
    for docs in {1..100}; do
        echo -n "$idx $docs: "
        curl -s -uelastic:password -XPOST -HContent-Type:application/json "localhost:9200/test$idx/_bulk?refresh&pretty" --data-binary @/tmp/bulk | tee /tmp/bulk_result | grep error
        curl -s -uelastic:password -XPOST -HContent-Type:application/json "localhost:9200/test$idx/_flush" | tee /tmp/flush_out > /dev/null
    done
done

curl -s -uelastic:password -XPOST -HContent-Type:application/json 'localhost:9200/_query?pretty' -d'{
    "query": "FROM * | STATS SUM(LENGTH(field))",
    "profile": true
}' | jq -c '[.profile.drivers[].operators[] | {process_nanos: .status.process_nanos, operator}] | sort_by(.process_nanos)[]'



while true; do
    curl -s -uelastic:password -XPOST -HContent-Type:application/json 'localhost:9200/_query?pretty' -d'{
        "query": "FROM * | STATS SUM(LENGTH(field))"
    }' | jq .took
done


# With
# 28
# {"process_nanos":null,"operator":"OutputOperator[columns = [SUM(LENGTH(field))]]"}
# {"process_nanos":4875,"operator":"ProjectOperator[projection = [0]]"}
# {"process_nanos":5619,"operator":"EvalOperator[evaluator=Attribute[channel=1]]"}
# {"process_nanos":5821,"operator":"EvalOperator[evaluator=Attribute[channel=1]]"}
# {"process_nanos":5875,"operator":"EvalOperator[evaluator=Attribute[channel=1]]"}
# {"process_nanos":6267,"operator":"EvalOperator[evaluator=Attribute[channel=1]]"}
# {"process_nanos":3580358,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 21845, remainingDocs = 2147482135]"}
# {"process_nanos":3782061,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 21845, remainingDocs = 2147467632]"}
# {"process_nanos":3921976,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 21845, remainingDocs = 2147473647]"}
# {"process_nanos":3930299,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 21845, remainingDocs = 2147469147]"}
# {"process_nanos":12085140,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":12245074,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":12306936,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":12553149,"operator":"ValuesSourceReaderOperator[fields = [field]]"}

# Without
# 48
# {"process_nanos":null,"operator":"OutputOperator[columns = [SUM(LENGTH(field))]]"}
# {"process_nanos":1782,"operator":"ProjectOperator[projection = [0]]"}
# {"process_nanos":3189869,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 4519, remainingDocs = 2147483326]"}
# {"process_nanos":3319246,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 4519, remainingDocs = 2147479306]"}
# {"process_nanos":3501836,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 4519, remainingDocs = 2147482685]"}
# {"process_nanos":3673315,"operator":"LuceneSourceOperator[shards = [test10:0,test1:0,test2:0,test3:0,test4:0,test5:0,test6:0,test7:0,test8:0,test9:0], maxPageSize = 4519, remainingDocs = 2147480939]"}
# {"process_nanos":10562797,"operator":"EvalOperator[evaluator=LengthEvaluator[val=Attribute[channel=1]]]"}
# {"process_nanos":10621623,"operator":"EvalOperator[evaluator=LengthEvaluator[val=Attribute[channel=1]]]"}
# {"process_nanos":11202374,"operator":"EvalOperator[evaluator=LengthEvaluator[val=Attribute[channel=1]]]"}
# {"process_nanos":11243582,"operator":"EvalOperator[evaluator=LengthEvaluator[val=Attribute[channel=1]]]"}
# {"process_nanos":14911219,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":14978035,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":15560394,"operator":"ValuesSourceReaderOperator[fields = [field]]"}
# {"process_nanos":15854270,"operator":"ValuesSourceReaderOperator[fields = [field]]"}


#             Load        Eval
# Without   13684208    11339296
#    With   11444592        5192